Merge branch 'main' into nexucis/merge-back

.github/CODEOWNERS

@@ -1,6 +1,6 @@
 /web/ui @juliusv
 /web/ui/module @juliusv @nexucis
-/storage/remote @csmarchbanks @cstyan @bwplotka @tomwilkie
+/storage/remote @cstyan @bwplotka @tomwilkie
 /storage/remote/otlptranslator @gouthamve @jesusvazquez
 /discovery/kubernetes @brancz
 /tsdb @jesusvazquez

.github/workflows/buf.yml

@@ -23,7 +23,7 @@ jobs:
         with:
           input: 'prompb'
           against: 'https://github.com/prometheus/prometheus.git#branch=main,ref=HEAD~1,subdir=prompb'
-      - uses: bufbuild/buf-push-action@342fc4cdcf29115a01cf12a2c6dd6aac68dc51e1 # v1.1.1
+      - uses: bufbuild/buf-push-action@a654ff18effe4641ebea4a4ce242c49800728459 # v1.1.1
         with:
           input: 'prompb'
           buf_token: ${{ secrets.BUF_TOKEN }}

.github/workflows/ci.yml

@@ -151,6 +151,7 @@ jobs:
         uses: golangci/golangci-lint-action@3a919529898de77ec3da873e3063ca4b10e7f5cc # v3.7.0
         with:
           args: --verbose
+          # Make sure to sync this with Makefile.common and scripts/golangci-lint.yml.
           version: v1.55.2
   fuzzing:
     uses: ./.github/workflows/fuzzing.yml
@@ -196,7 +197,7 @@ jobs:
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - uses: prometheus/promci@3cb0c3871f223bd5ce1226995bd52ffb314798b6 # v0.1.0
       - name: Install nodejs
-        uses: actions/setup-node@5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d # v3.8.1
+        uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4.0.1
         with:
           node-version-file: "web/ui/.nvmrc"
           registry-url: "https://registry.npmjs.org"

.github/workflows/codeql-analysis.yml

@@ -30,12 +30,12 @@ jobs:
           go-version: 1.21.x
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@407ffafae6a767df3e0230c3df91b6443ae8df75 # v2.22.8
+        uses: github/codeql-action/init@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # v3.22.12
         with:
           languages: ${{ matrix.language }}
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@407ffafae6a767df3e0230c3df91b6443ae8df75 # v2.22.8
+        uses: github/codeql-action/autobuild@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # v3.22.12
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@407ffafae6a767df3e0230c3df91b6443ae8df75 # v2.22.8
+        uses: github/codeql-action/analyze@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # v3.22.12

.github/workflows/fuzzing.yml

@@ -21,7 +21,7 @@ jobs:
           fuzz-seconds: 600
           dry-run: false
       - name: Upload Crash
-        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
+        uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
         if: failure() && steps.build.outcome == 'success'
         with:
           name: artifacts

.github/workflows/scorecards.yml

@@ -37,7 +37,7 @@ jobs:
       # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
       # format to the repository Actions tab.
       - name: "Upload artifact"
-        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # tag=v3.1.3
+        uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # tag=v4.0.0
         with:
           name: SARIF file
           path: results.sarif
@@ -45,6 +45,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@407ffafae6a767df3e0230c3df91b6443ae8df75 # tag=v2.22.8
+        uses: github/codeql-action/upload-sarif@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # tag=v3.22.12
         with:
           sarif_file: results.sarif

.golangci.yml

@@ -36,13 +36,9 @@ issues:
     - path: _test.go
       linters:
         - errcheck
-    - path: tsdb/
+    - path: "tsdb/head_wal.go"
       linters:
         - errorlint
-    - path: tsdb/
-      text: "import 'github.com/pkg/errors' is not allowed"
-      linters:
-        - depguard
     - linters:
         - godot
       source: "^// ==="

MAINTAINERS.md

@@ -9,7 +9,7 @@ Julien Pivotto (<roidelapluie@prometheus.io> / @roidelapluie) and Levi Harrison
 * `documentation`
   * `prometheus-mixin`: Matthias Loibl (<mail@matthiasloibl.com> / @metalmatze)
 * `storage`
-  * `remote`: Chris Marchbanks (<csmarchbanks@gmail.com> / @csmarchbanks), Callum Styan (<callumstyan@gmail.com> / @cstyan), Bartłomiej Płotka (<bwplotka@gmail.com> / @bwplotka), Tom Wilkie (<tom.wilkie@gmail.com> / @tomwilkie)
+  * `remote`: Callum Styan (<callumstyan@gmail.com> / @cstyan), Bartłomiej Płotka (<bwplotka@gmail.com> / @bwplotka), Tom Wilkie (<tom.wilkie@gmail.com> / @tomwilkie)
 * `tsdb`: Ganesh Vernekar (<ganesh@grafana.com> / @codesome), Bartłomiej Płotka (<bwplotka@gmail.com> / @bwplotka), Jesús Vázquez (<jesus.vazquez@grafana.com> / @jesusvazquez)
   * `agent`: Robert Fratto (<robert.fratto@grafana.com> / @rfratto)
 * `web`

README.md

@@ -14,7 +14,7 @@ examples and guides.</p>
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/486/badge)](https://bestpractices.coreinfrastructure.org/projects/486)
 [![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/prometheus/prometheus)
 [![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/prometheus.svg)](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:prometheus)
-[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/prometheus/prometheus/badge)](https://api.securityscorecards.dev/projects/github.com/prometheus/prometheus)
+[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/prometheus/prometheus/badge)](https://securityscorecards.dev/viewer/?uri=github.com/prometheus/prometheus)
 
 </div>
 

RELEASE.md

@@ -54,7 +54,8 @@ Release cadence of first pre-releases being cut is 6 weeks.
 | v2.47          | 2023-08-23                                 | Bryan Boreham (GitHub: @bboreham)           |
 | v2.48          | 2023-10-04                                 | Levi Harrison (GitHub: @LeviHarrison)       |
 | v2.49          | 2023-12-05                                 | Bartek Plotka (GitHub: @bwplotka)           |
-| v2.50          | 2024-01-16                                 | **searching for volunteer**                 |
+| v2.50          | 2024-01-16                                 | Augustin Husson (GitHub: @nexucis)          |
+| v2.51          | 2024-02-13                                 | **searching for volunteer**                 |
 
 If you are interested in volunteering please create a pull request against the [prometheus/prometheus](https://github.com/prometheus/prometheus) repository and propose yourself for the release series of your choice.
 

cmd/prometheus/main.go

@@ -33,6 +33,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/KimMachineGun/automemlimit/memlimit"
 	"github.com/alecthomas/kingpin/v2"
 	"github.com/alecthomas/units"
 	"github.com/go-kit/log"
@@ -147,13 +148,15 @@ type flagConfig struct {
 	queryMaxSamples     int
 	RemoteFlushDeadline model.Duration
 
-	featureList []string
+	featureList   []string
+	memlimitRatio float64
 	// These options are extracted from featureList
 	// for ease of use.
 	enableExpandExternalLabels bool
 	enableNewSDManager         bool
 	enablePerStepStats         bool
 	enableAutoGOMAXPROCS       bool
+	enableAutoGOMEMLIMIT       bool
 
 	prometheusURL   string
 	corsRegexString string
@@ -197,6 +200,9 @@ func (c *flagConfig) setFeatureListOptions(logger log.Logger) error {
 			case "auto-gomaxprocs":
 				c.enableAutoGOMAXPROCS = true
 				level.Info(logger).Log("msg", "Automatically set GOMAXPROCS to match Linux container CPU quota")
+			case "auto-gomemlimit":
+				c.enableAutoGOMEMLIMIT = true
+				level.Info(logger).Log("msg", "Automatically set GOMEMLIMIT to match Linux container or system memory limit")
 			case "no-default-scrape-port":
 				c.scrape.NoDefaultPort = true
 				level.Info(logger).Log("msg", "No default port will be appended to scrape targets' addresses.")
@@ -206,9 +212,15 @@ func (c *flagConfig) setFeatureListOptions(logger log.Logger) error {
 			case "native-histograms":
 				c.tsdb.EnableNativeHistograms = true
 				// Change relevant global variables. Hacky, but it's hard to pass a new option or default to unmarshallers.
-				config.DefaultConfig.GlobalConfig.ScrapeProtocols = config.DefaultNativeHistogramScrapeProtocols
+				config.DefaultConfig.GlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
-				config.DefaultGlobalConfig.ScrapeProtocols = config.DefaultNativeHistogramScrapeProtocols
+				config.DefaultGlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
 				level.Info(logger).Log("msg", "Experimental native histogram support enabled. Changed default scrape_protocols to prefer PrometheusProto format.", "global.scrape_protocols", fmt.Sprintf("%v", config.DefaultGlobalConfig.ScrapeProtocols))
+			case "created-timestamp-zero-ingestion":
+				c.scrape.EnableCreatedTimestampZeroIngestion = true
+				// Change relevant global variables. Hacky, but it's hard to pass a new option or default to unmarshallers.
+				config.DefaultConfig.GlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
+				config.DefaultGlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
+				level.Info(logger).Log("msg", "Experimental created timestamp zero ingestion enabled. Changed default scrape_protocols to prefer PrometheusProto format.", "global.scrape_protocols", fmt.Sprintf("%v", config.DefaultGlobalConfig.ScrapeProtocols))
 			case "":
 				continue
 			case "promql-at-modifier", "promql-negative-offset":
@@ -256,6 +268,9 @@ func main() {
 	a.Flag("web.listen-address", "Address to listen on for UI, API, and telemetry.").
 		Default("0.0.0.0:9090").StringVar(&cfg.web.ListenAddress)
 
+	a.Flag("auto-gomemlimit.ratio", "The ratio of reserved GOMEMLIMIT memory to the detected maximum container or system memory").
+		Default("0.9").FloatVar(&cfg.memlimitRatio)
+
 	webConfig := a.Flag(
 		"web.config.file",
 		"[EXPERIMENTAL] Path to configuration file that can enable TLS or authentication.",
@@ -423,7 +438,7 @@ func main() {
 	a.Flag("scrape.discovery-reload-interval", "Interval used by scrape manager to throttle target groups updates.").
 		Hidden().Default("5s").SetValue(&cfg.scrape.DiscoveryReloadInterval)
 
-	a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: agent, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-at-modifier, promql-negative-offset, promql-per-step-stats, promql-experimental-functions, remote-write-receiver (DEPRECATED), extra-scrape-metrics, new-service-discovery-manager, auto-gomaxprocs, no-default-scrape-port, native-histograms, otlp-write-receiver. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details.").
+	a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: agent, auto-gomemlimit, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-at-modifier, promql-negative-offset, promql-per-step-stats, promql-experimental-functions, remote-write-receiver (DEPRECATED), extra-scrape-metrics, new-service-discovery-manager, auto-gomaxprocs, no-default-scrape-port, native-histograms, otlp-write-receiver. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details.").
 		Default("").StringsVar(&cfg.featureList)
 
 	promlogflag.AddFlags(a, &cfg.promlogConfig)
@@ -461,6 +476,11 @@ func main() {
 		os.Exit(3)
 	}
 
+	if cfg.memlimitRatio <= 0.0 || cfg.memlimitRatio > 1.0 {
+		fmt.Fprintf(os.Stderr, "--auto-gomemlimit.ratio must be greater than 0 and less than or equal to 1.")
+		os.Exit(1)
+	}
+
 	localStoragePath := cfg.serverStoragePath
 	if agentMode {
 		localStoragePath = cfg.agentStoragePath
@@ -614,14 +634,59 @@ func main() {
 		discoveryManagerNotify  discoveryManager
 	)
 
+	// Kubernetes client metrics are used by Kubernetes SD.
+	// They are registered here in the main function, because SD mechanisms
+	// can only register metrics specific to a SD instance.
+	// Kubernetes client metrics are the same for the whole process -
+	// they are not specific to an SD instance.
+	err = discovery.RegisterK8sClientMetricsWithPrometheus(prometheus.DefaultRegisterer)
+	if err != nil {
+		level.Error(logger).Log("msg", "failed to register Kubernetes client metrics", "err", err)
+		os.Exit(1)
+	}
+
+	sdMetrics, err := discovery.CreateAndRegisterSDMetrics(prometheus.DefaultRegisterer)
+	if err != nil {
+		level.Error(logger).Log("msg", "failed to register service discovery metrics", "err", err)
+		os.Exit(1)
+	}
+
 	if cfg.enableNewSDManager {
-		discovery.RegisterMetrics()
+		{
-		discoveryManagerScrape = discovery.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), discovery.Name("scrape"))
+			discMgr := discovery.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, sdMetrics, discovery.Name("scrape"))
-		discoveryManagerNotify = discovery.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), discovery.Name("notify"))
+			if discMgr == nil {
+				level.Error(logger).Log("msg", "failed to create a discovery manager scrape")
+				os.Exit(1)
+			}
+			discoveryManagerScrape = discMgr
+		}
+
+		{
+			discMgr := discovery.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, sdMetrics, discovery.Name("notify"))
+			if discMgr == nil {
+				level.Error(logger).Log("msg", "failed to create a discovery manager notify")
+				os.Exit(1)
+			}
+			discoveryManagerNotify = discMgr
+		}
 	} else {
-		legacymanager.RegisterMetrics()
+		{
-		discoveryManagerScrape = legacymanager.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), legacymanager.Name("scrape"))
+			discMgr := legacymanager.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, sdMetrics, legacymanager.Name("scrape"))
-		discoveryManagerNotify = legacymanager.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), legacymanager.Name("notify"))
+			if discMgr == nil {
+				level.Error(logger).Log("msg", "failed to create a discovery manager scrape")
+				os.Exit(1)
+			}
+			discoveryManagerScrape = discMgr
+		}
+
+		{
+			discMgr := legacymanager.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, sdMetrics, legacymanager.Name("notify"))
+			if discMgr == nil {
+				level.Error(logger).Log("msg", "failed to create a discovery manager notify")
+				os.Exit(1)
+			}
+			discoveryManagerNotify = discMgr
+		}
 	}
 
 	scrapeManager, err := scrape.NewManager(
@@ -651,6 +716,20 @@ func main() {
 		}
 	}
 
+	if cfg.enableAutoGOMEMLIMIT {
+		if _, err := memlimit.SetGoMemLimitWithOpts(
+			memlimit.WithRatio(cfg.memlimitRatio),
+			memlimit.WithProvider(
+				memlimit.ApplyFallback(
+					memlimit.FromCgroup,
+					memlimit.FromSystem,
+				),
+			),
+		); err != nil {
+			level.Warn(logger).Log("component", "automemlimit", "msg", "Failed to set GOMEMLIMIT automatically", "err", err)
+		}
+	}
+
 	if !agentMode {
 		opts := promql.EngineOpts{
 			Logger:                   log.With(logger, "component", "query engine"),
@@ -1449,6 +1528,10 @@ func (n notReadyAppender) UpdateMetadata(ref storage.SeriesRef, l labels.Labels,
 	return 0, tsdb.ErrNotReady
 }
 
+func (n notReadyAppender) AppendCTZeroSample(ref storage.SeriesRef, l labels.Labels, t, ct int64) (storage.SeriesRef, error) {
+	return 0, tsdb.ErrNotReady
+}
+
 func (n notReadyAppender) Commit() error { return tsdb.ErrNotReady }
 
 func (n notReadyAppender) Rollback() error { return tsdb.ErrNotReady }
@@ -1587,7 +1670,6 @@ func (opts tsdbOptions) ToTSDBOptions() tsdb.Options {
 		RetentionDuration:              int64(time.Duration(opts.RetentionDuration) / time.Millisecond),
 		MaxBytes:                       int64(opts.MaxBytes),
 		NoLockfile:                     opts.NoLockfile,
-		AllowOverlappingCompaction:     true,
 		WALCompression:                 wlog.ParseCompressionType(opts.WALCompression, opts.WALCompressionType),
 		HeadChunksWriteQueueSize:       opts.HeadChunksWriteQueueSize,
 		SamplesPerChunk:                opts.SamplesPerChunk,
@@ -1599,6 +1681,7 @@ func (opts tsdbOptions) ToTSDBOptions() tsdb.Options {
 		EnableMemorySnapshotOnShutdown: opts.EnableMemorySnapshotOnShutdown,
 		EnableNativeHistograms:         opts.EnableNativeHistograms,
 		OutOfOrderTimeWindow:           opts.OutOfOrderTimeWindow,
+		EnableOverlappingCompaction:    true,
 	}
 }
 

cmd/prometheus/main_unix_test.go

@@ -12,7 +12,6 @@
 // limitations under the License.
 //
 //go:build !windows
-// +build !windows
 
 package main
 

cmd/promtool/analyze.go

@@ -0,0 +1,370 @@
+// Copyright 2023 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"net/url"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
+	"github.com/prometheus/common/model"
+
+	"github.com/prometheus/prometheus/model/labels"
+)
+
+var (
+	errNotNativeHistogram = fmt.Errorf("not a native histogram")
+	errNotEnoughData      = fmt.Errorf("not enough data")
+
+	outputHeader = `Bucket stats for each histogram series over time
+------------------------------------------------
+First the min, avg, and max number of populated buckets, followed by the total
+number of buckets (only if different from the max number of populated buckets
+which is typical for classic but not native histograms).`
+	outputFooter = `Aggregated bucket stats
+-----------------------
+Each line shows min/avg/max over the series above.`
+)
+
+type QueryAnalyzeConfig struct {
+	metricType string
+	duration   time.Duration
+	time       string
+	matchers   []string
+}
+
+// run retrieves metrics that look like conventional histograms (i.e. have _bucket
+// suffixes) or native histograms, depending on metricType flag.
+func (c *QueryAnalyzeConfig) run(url *url.URL, roundtripper http.RoundTripper) error {
+	if c.metricType != "histogram" {
+		return fmt.Errorf("analyze type is %s, must be 'histogram'", c.metricType)
+	}
+
+	ctx := context.Background()
+
+	api, err := newAPI(url, roundtripper, nil)
+	if err != nil {
+		return err
+	}
+
+	var endTime time.Time
+	if c.time != "" {
+		endTime, err = parseTime(c.time)
+		if err != nil {
+			return fmt.Errorf("error parsing time '%s': %w", c.time, err)
+		}
+	} else {
+		endTime = time.Now()
+	}
+
+	return c.getStatsFromMetrics(ctx, api, endTime, os.Stdout, c.matchers)
+}
+
+func (c *QueryAnalyzeConfig) getStatsFromMetrics(ctx context.Context, api v1.API, endTime time.Time, out io.Writer, matchers []string) error {
+	fmt.Fprintf(out, "%s\n\n", outputHeader)
+	metastatsNative := newMetaStatistics()
+	metastatsClassic := newMetaStatistics()
+	for _, matcher := range matchers {
+		seriesSel := seriesSelector(matcher, c.duration)
+		matrix, err := querySamples(ctx, api, seriesSel, endTime)
+		if err != nil {
+			return err
+		}
+
+		matrices := make(map[string]model.Matrix)
+		for _, series := range matrix {
+			// We do not handle mixed types. If there are float values, we assume it is a
+			// classic histogram, otherwise we assume it is a native histogram, and we
+			// ignore series with errors if they do not match the expected type.
+			if len(series.Values) == 0 {
+				stats, err := calcNativeBucketStatistics(series)
+				if err != nil {
+					if errors.Is(err, errNotNativeHistogram) || errors.Is(err, errNotEnoughData) {
+						continue
+					}
+					return err
+				}
+				fmt.Fprintf(out, "- %s (native): %v\n", series.Metric, *stats)
+				metastatsNative.update(stats)
+			} else {
+				lbs := model.LabelSet(series.Metric).Clone()
+				if _, ok := lbs["le"]; !ok {
+					continue
+				}
+				metricName := string(lbs[labels.MetricName])
+				if !strings.HasSuffix(metricName, "_bucket") {
+					continue
+				}
+				delete(lbs, labels.MetricName)
+				delete(lbs, "le")
+				key := formatSeriesName(metricName, lbs)
+				matrices[key] = append(matrices[key], series)
+			}
+		}
+
+		for key, matrix := range matrices {
+			stats, err := calcClassicBucketStatistics(matrix)
+			if err != nil {
+				if errors.Is(err, errNotEnoughData) {
+					continue
+				}
+				return err
+			}
+			fmt.Fprintf(out, "- %s (classic): %v\n", key, *stats)
+			metastatsClassic.update(stats)
+		}
+	}
+	fmt.Fprintf(out, "\n%s\n", outputFooter)
+	if metastatsNative.Count() > 0 {
+		fmt.Fprintf(out, "\nNative %s\n", metastatsNative)
+	}
+	if metastatsClassic.Count() > 0 {
+		fmt.Fprintf(out, "\nClassic %s\n", metastatsClassic)
+	}
+	return nil
+}
+
+func seriesSelector(metricName string, duration time.Duration) string {
+	builder := strings.Builder{}
+	builder.WriteString(metricName)
+	builder.WriteRune('[')
+	builder.WriteString(duration.String())
+	builder.WriteRune(']')
+	return builder.String()
+}
+
+func formatSeriesName(metricName string, lbs model.LabelSet) string {
+	builder := strings.Builder{}
+	builder.WriteString(metricName)
+	builder.WriteString(lbs.String())
+	return builder.String()
+}
+
+func querySamples(ctx context.Context, api v1.API, query string, end time.Time) (model.Matrix, error) {
+	values, _, err := api.Query(ctx, query, end)
+	if err != nil {
+		return nil, err
+	}
+
+	matrix, ok := values.(model.Matrix)
+	if !ok {
+		return nil, fmt.Errorf("query of buckets resulted in non-Matrix")
+	}
+
+	return matrix, nil
+}
+
+// minPop/avgPop/maxPop is for the number of populated (non-zero) buckets.
+// total is the total number of buckets across all samples in the series,
+// populated or not.
+type statistics struct {
+	minPop, maxPop, total int
+	avgPop                float64
+}
+
+func (s statistics) String() string {
+	if s.maxPop == s.total {
+		return fmt.Sprintf("%d/%.3f/%d", s.minPop, s.avgPop, s.maxPop)
+	}
+	return fmt.Sprintf("%d/%.3f/%d/%d", s.minPop, s.avgPop, s.maxPop, s.total)
+}
+
+func calcClassicBucketStatistics(matrix model.Matrix) (*statistics, error) {
+	numBuckets := len(matrix)
+
+	stats := &statistics{
+		minPop: math.MaxInt,
+		total:  numBuckets,
+	}
+
+	if numBuckets == 0 || len(matrix[0].Values) < 2 {
+		return stats, errNotEnoughData
+	}
+
+	numSamples := len(matrix[0].Values)
+
+	sortMatrix(matrix)
+
+	totalPop := 0
+	for timeIdx := 0; timeIdx < numSamples; timeIdx++ {
+		curr, err := getBucketCountsAtTime(matrix, numBuckets, timeIdx)
+		if err != nil {
+			return stats, err
+		}
+		countPop := 0
+		for _, b := range curr {
+			if b != 0 {
+				countPop++
+			}
+		}
+
+		totalPop += countPop
+		if stats.minPop > countPop {
+			stats.minPop = countPop
+		}
+		if stats.maxPop < countPop {
+			stats.maxPop = countPop
+		}
+	}
+	stats.avgPop = float64(totalPop) / float64(numSamples)
+	return stats, nil
+}
+
+func sortMatrix(matrix model.Matrix) {
+	sort.SliceStable(matrix, func(i, j int) bool {
+		return getLe(matrix[i]) < getLe(matrix[j])
+	})
+}
+
+func getLe(series *model.SampleStream) float64 {
+	lbs := model.LabelSet(series.Metric)
+	le, _ := strconv.ParseFloat(string(lbs["le"]), 64)
+	return le
+}
+
+func getBucketCountsAtTime(matrix model.Matrix, numBuckets, timeIdx int) ([]int, error) {
+	counts := make([]int, numBuckets)
+	if timeIdx >= len(matrix[0].Values) {
+		// Just return zeroes instead of erroring out so we can get partial results.
+		return counts, nil
+	}
+	counts[0] = int(matrix[0].Values[timeIdx].Value)
+	for i, bucket := range matrix[1:] {
+		if timeIdx >= len(bucket.Values) {
+			// Just return zeroes instead of erroring out so we can get partial results.
+			return counts, nil
+		}
+		curr := bucket.Values[timeIdx]
+		prev := matrix[i].Values[timeIdx]
+		// Assume the results are nicely aligned.
+		if curr.Timestamp != prev.Timestamp {
+			return counts, fmt.Errorf("matrix result is not time aligned")
+		}
+		counts[i+1] = int(curr.Value - prev.Value)
+	}
+	return counts, nil
+}
+
+type bucketBounds struct {
+	boundaries   int32
+	upper, lower float64
+}
+
+func makeBucketBounds(b *model.HistogramBucket) bucketBounds {
+	return bucketBounds{
+		boundaries: b.Boundaries,
+		upper:      float64(b.Upper),
+		lower:      float64(b.Lower),
+	}
+}
+
+func calcNativeBucketStatistics(series *model.SampleStream) (*statistics, error) {
+	stats := &statistics{
+		minPop: math.MaxInt,
+	}
+
+	overall := make(map[bucketBounds]struct{})
+	totalPop := 0
+	if len(series.Histograms) == 0 {
+		return nil, errNotNativeHistogram
+	}
+	if len(series.Histograms) == 1 {
+		return nil, errNotEnoughData
+	}
+	for _, histogram := range series.Histograms {
+		for _, bucket := range histogram.Histogram.Buckets {
+			bb := makeBucketBounds(bucket)
+			overall[bb] = struct{}{}
+		}
+		countPop := len(histogram.Histogram.Buckets)
+
+		totalPop += countPop
+		if stats.minPop > countPop {
+			stats.minPop = countPop
+		}
+		if stats.maxPop < countPop {
+			stats.maxPop = countPop
+		}
+	}
+	stats.avgPop = float64(totalPop) / float64(len(series.Histograms))
+	stats.total = len(overall)
+	return stats, nil
+}
+
+type distribution struct {
+	min, max, count int
+	avg             float64
+}
+
+func newDistribution() distribution {
+	return distribution{
+		min: math.MaxInt,
+	}
+}
+
+func (d *distribution) update(num int) {
+	if d.min > num {
+		d.min = num
+	}
+	if d.max < num {
+		d.max = num
+	}
+	d.count++
+	d.avg += float64(num)/float64(d.count) - d.avg/float64(d.count)
+}
+
+func (d distribution) String() string {
+	return fmt.Sprintf("%d/%.3f/%d", d.min, d.avg, d.max)
+}
+
+type metaStatistics struct {
+	minPop, avgPop, maxPop, total distribution
+}
+
+func newMetaStatistics() *metaStatistics {
+	return &metaStatistics{
+		minPop: newDistribution(),
+		avgPop: newDistribution(),
+		maxPop: newDistribution(),
+		total:  newDistribution(),
+	}
+}
+
+func (ms metaStatistics) Count() int {
+	return ms.minPop.count
+}
+
+func (ms metaStatistics) String() string {
+	if ms.maxPop == ms.total {
+		return fmt.Sprintf("histogram series (%d in total):\n- min populated: %v\n- avg populated: %v\n- max populated: %v", ms.Count(), ms.minPop, ms.avgPop, ms.maxPop)
+	}
+	return fmt.Sprintf("histogram series (%d in total):\n- min populated: %v\n- avg populated: %v\n- max populated: %v\n- total: %v", ms.Count(), ms.minPop, ms.avgPop, ms.maxPop, ms.total)
+}
+
+func (ms *metaStatistics) update(s *statistics) {
+	ms.minPop.update(s.minPop)
+	ms.avgPop.update(int(s.avgPop))
+	ms.maxPop.update(s.maxPop)
+	ms.total.update(s.total)
+}

cmd/promtool/analyze_test.go

@@ -0,0 +1,170 @@
+// Copyright 2023 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/common/model"
+)
+
+var (
+	exampleMatrix = model.Matrix{
+		&model.SampleStream{
+			Metric: model.Metric{
+				"le": "+Inf",
+			},
+			Values: []model.SamplePair{
+				{
+					Value:     31,
+					Timestamp: 100,
+				},
+				{
+					Value:     32,
+					Timestamp: 200,
+				},
+				{
+					Value:     40,
+					Timestamp: 300,
+				},
+			},
+		},
+		&model.SampleStream{
+			Metric: model.Metric{
+				"le": "0.5",
+			},
+			Values: []model.SamplePair{
+				{
+					Value:     10,
+					Timestamp: 100,
+				},
+				{
+					Value:     11,
+					Timestamp: 200,
+				},
+				{
+					Value:     11,
+					Timestamp: 300,
+				},
+			},
+		},
+		&model.SampleStream{
+			Metric: model.Metric{
+				"le": "10",
+			},
+			Values: []model.SamplePair{
+				{
+					Value:     30,
+					Timestamp: 100,
+				},
+				{
+					Value:     31,
+					Timestamp: 200,
+				},
+				{
+					Value:     37,
+					Timestamp: 300,
+				},
+			},
+		},
+		&model.SampleStream{
+			Metric: model.Metric{
+				"le": "2",
+			},
+			Values: []model.SamplePair{
+				{
+					Value:     25,
+					Timestamp: 100,
+				},
+				{
+					Value:     26,
+					Timestamp: 200,
+				},
+				{
+					Value:     27,
+					Timestamp: 300,
+				},
+			},
+		},
+	}
+	exampleMatrixLength = len(exampleMatrix)
+)
+
+func init() {
+	sortMatrix(exampleMatrix)
+}
+
+func TestGetBucketCountsAtTime(t *testing.T) {
+	cases := []struct {
+		matrix   model.Matrix
+		length   int
+		timeIdx  int
+		expected []int
+	}{
+		{
+			exampleMatrix,
+			exampleMatrixLength,
+			0,
+			[]int{10, 15, 5, 1},
+		},
+		{
+			exampleMatrix,
+			exampleMatrixLength,
+			1,
+			[]int{11, 15, 5, 1},
+		},
+		{
+			exampleMatrix,
+			exampleMatrixLength,
+			2,
+			[]int{11, 16, 10, 3},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(fmt.Sprintf("exampleMatrix@%d", c.timeIdx), func(t *testing.T) {
+			res, err := getBucketCountsAtTime(c.matrix, c.length, c.timeIdx)
+			require.NoError(t, err)
+			require.Equal(t, c.expected, res)
+		})
+	}
+}
+
+func TestCalcClassicBucketStatistics(t *testing.T) {
+	cases := []struct {
+		matrix   model.Matrix
+		expected *statistics
+	}{
+		{
+			exampleMatrix,
+			&statistics{
+				minPop: 4,
+				avgPop: 4,
+				maxPop: 4,
+				total:  4,
+			},
+		},
+	}
+
+	for i, c := range cases {
+		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
+			res, err := calcClassicBucketStatistics(c.matrix)
+			require.NoError(t, err)
+			require.Equal(t, c.expected, res)
+		})
+	}
+}

cmd/promtool/main.go

@@ -35,8 +35,7 @@ import (
 	"github.com/go-kit/log"
 	"github.com/google/pprof/profile"
 	"github.com/prometheus/client_golang/api"
-	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
+	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/prometheus/client_golang/prometheus/testutil/promlint"
 	config_util "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
@@ -184,6 +183,14 @@ func main() {
 	queryLabelsEnd := queryLabelsCmd.Flag("end", "End time (RFC3339 or Unix timestamp).").String()
 	queryLabelsMatch := queryLabelsCmd.Flag("match", "Series selector. Can be specified multiple times.").Strings()
 
+	queryAnalyzeCfg := &QueryAnalyzeConfig{}
+	queryAnalyzeCmd := queryCmd.Command("analyze", "Run queries against your Prometheus to analyze the usage pattern of certain metrics.")
+	queryAnalyzeCmd.Flag("server", "Prometheus server to query.").Required().URLVar(&serverURL)
+	queryAnalyzeCmd.Flag("type", "Type of metric: histogram.").Required().StringVar(&queryAnalyzeCfg.metricType)
+	queryAnalyzeCmd.Flag("duration", "Time frame to analyze.").Default("1h").DurationVar(&queryAnalyzeCfg.duration)
+	queryAnalyzeCmd.Flag("time", "Query time (RFC3339 or Unix timestamp), defaults to now.").StringVar(&queryAnalyzeCfg.time)
+	queryAnalyzeCmd.Flag("match", "Series selector. Can be specified multiple times.").Required().StringsVar(&queryAnalyzeCfg.matchers)
+
 	pushCmd := app.Command("push", "Push to a Prometheus server.")
 	pushCmd.Flag("http.config.file", "HTTP client configuration file for promtool to connect to Prometheus.").PlaceHolder("<filename>").ExistingFileVar(&httpConfigFilePath)
 	pushMetricsCmd := pushCmd.Command("metrics", "Push metrics to a prometheus remote write (for testing purpose only).")
@@ -203,6 +210,7 @@ func main() {
 		"test-rule-file",
 		"The unit test file.",
 	).Required().ExistingFiles()
+	testRulesDiff := testRulesCmd.Flag("diff", "[Experimental] Print colored differential output between expected & received output.").Default("false").Bool()
 
 	defaultDBPath := "data/"
 	tsdbCmd := app.Command("tsdb", "Run tsdb commands.")
@@ -229,7 +237,7 @@ func main() {
 	dumpPath := tsdbDumpCmd.Arg("db path", "Database path (default is "+defaultDBPath+").").Default(defaultDBPath).String()
 	dumpMinTime := tsdbDumpCmd.Flag("min-time", "Minimum timestamp to dump.").Default(strconv.FormatInt(math.MinInt64, 10)).Int64()
 	dumpMaxTime := tsdbDumpCmd.Flag("max-time", "Maximum timestamp to dump.").Default(strconv.FormatInt(math.MaxInt64, 10)).Int64()
-	dumpMatch := tsdbDumpCmd.Flag("match", "Series selector.").Default("{__name__=~'(?s:.*)'}").String()
+	dumpMatch := tsdbDumpCmd.Flag("match", "Series selector. Can be specified multiple times.").Default("{__name__=~'(?s:.*)'}").Strings()
 
 	importCmd := tsdbCmd.Command("create-blocks-from", "[Experimental] Import samples from input and produce TSDB blocks. Please refer to the storage docs for more details.")
 	importHumanReadable := importCmd.Flag("human-readable", "Print human readable values.").Short('r').Bool()
@@ -317,7 +325,7 @@ func main() {
 
 	switch parsedCmd {
 	case sdCheckCmd.FullCommand():
-		os.Exit(CheckSD(*sdConfigFile, *sdJobName, *sdTimeout, noDefaultScrapePort))
+		os.Exit(CheckSD(*sdConfigFile, *sdJobName, *sdTimeout, noDefaultScrapePort, prometheus.DefaultRegisterer))
 
 	case checkConfigCmd.FullCommand():
 		os.Exit(CheckConfig(*agentMode, *checkConfigSyntaxOnly, newLintConfig(*checkConfigLint, *checkConfigLintFatal), *configFiles...))
@@ -368,6 +376,7 @@ func main() {
 				EnableNegativeOffset: true,
 			},
 			*testRulesRun,
+			*testRulesDiff,
 			*testRulesFiles...),
 		)
 
@@ -389,6 +398,9 @@ func main() {
 	case importRulesCmd.FullCommand():
 		os.Exit(checkErr(importRules(serverURL, httpRoundTripper, *importRulesStart, *importRulesEnd, *importRulesOutputDir, *importRulesEvalInterval, *maxBlockDuration, *importRulesFiles...)))
 
+	case queryAnalyzeCmd.FullCommand():
+		os.Exit(checkErr(queryAnalyzeCfg.run(serverURL, httpRoundTripper)))
+
 	case documentationCmd.FullCommand():
 		os.Exit(checkErr(documentcli.GenerateMarkdown(app.Model(), os.Stdout)))
 
@@ -996,246 +1008,6 @@ func checkMetricsExtended(r io.Reader) ([]metricStat, int, error) {
 	return stats, total, nil
 }
 
-// QueryInstant performs an instant query against a Prometheus server.
-func QueryInstant(url *url.URL, roundTripper http.RoundTripper, query, evalTime string, p printer) int {
-	if url.Scheme == "" {
-		url.Scheme = "http"
-	}
-	config := api.Config{
-		Address:      url.String(),
-		RoundTripper: roundTripper,
-	}
-
-	// Create new client.
-	c, err := api.NewClient(config)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, "error creating API client:", err)
-		return failureExitCode
-	}
-
-	eTime := time.Now()
-	if evalTime != "" {
-		eTime, err = parseTime(evalTime)
-		if err != nil {
-			fmt.Fprintln(os.Stderr, "error parsing evaluation time:", err)
-			return failureExitCode
-		}
-	}
-
-	// Run query against client.
-	api := v1.NewAPI(c)
-
-	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
-	val, _, err := api.Query(ctx, query, eTime) // Ignoring warnings for now.
-	cancel()
-	if err != nil {
-		return handleAPIError(err)
-	}
-
-	p.printValue(val)
-
-	return successExitCode
-}
-
-// QueryRange performs a range query against a Prometheus server.
-func QueryRange(url *url.URL, roundTripper http.RoundTripper, headers map[string]string, query, start, end string, step time.Duration, p printer) int {
-	if url.Scheme == "" {
-		url.Scheme = "http"
-	}
-	config := api.Config{
-		Address:      url.String(),
-		RoundTripper: roundTripper,
-	}
-
-	if len(headers) > 0 {
-		config.RoundTripper = promhttp.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
-			for key, value := range headers {
-				req.Header.Add(key, value)
-			}
-			return roundTripper.RoundTrip(req)
-		})
-	}
-
-	// Create new client.
-	c, err := api.NewClient(config)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, "error creating API client:", err)
-		return failureExitCode
-	}
-
-	var stime, etime time.Time
-
-	if end == "" {
-		etime = time.Now()
-	} else {
-		etime, err = parseTime(end)
-		if err != nil {
-			fmt.Fprintln(os.Stderr, "error parsing end time:", err)
-			return failureExitCode
-		}
-	}
-
-	if start == "" {
-		stime = etime.Add(-5 * time.Minute)
-	} else {
-		stime, err = parseTime(start)
-		if err != nil {
-			fmt.Fprintln(os.Stderr, "error parsing start time:", err)
-			return failureExitCode
-		}
-	}
-
-	if !stime.Before(etime) {
-		fmt.Fprintln(os.Stderr, "start time is not before end time")
-		return failureExitCode
-	}
-
-	if step == 0 {
-		resolution := math.Max(math.Floor(etime.Sub(stime).Seconds()/250), 1)
-		// Convert seconds to nanoseconds such that time.Duration parses correctly.
-		step = time.Duration(resolution) * time.Second
-	}
-
-	// Run query against client.
-	api := v1.NewAPI(c)
-	r := v1.Range{Start: stime, End: etime, Step: step}
-	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
-	val, _, err := api.QueryRange(ctx, query, r) // Ignoring warnings for now.
-	cancel()
-
-	if err != nil {
-		return handleAPIError(err)
-	}
-
-	p.printValue(val)
-	return successExitCode
-}
-
-// QuerySeries queries for a series against a Prometheus server.
-func QuerySeries(url *url.URL, roundTripper http.RoundTripper, matchers []string, start, end string, p printer) int {
-	if url.Scheme == "" {
-		url.Scheme = "http"
-	}
-	config := api.Config{
-		Address:      url.String(),
-		RoundTripper: roundTripper,
-	}
-
-	// Create new client.
-	c, err := api.NewClient(config)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, "error creating API client:", err)
-		return failureExitCode
-	}
-
-	stime, etime, err := parseStartTimeAndEndTime(start, end)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		return failureExitCode
-	}
-
-	// Run query against client.
-	api := v1.NewAPI(c)
-	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
-	val, _, err := api.Series(ctx, matchers, stime, etime) // Ignoring warnings for now.
-	cancel()
-
-	if err != nil {
-		return handleAPIError(err)
-	}
-
-	p.printSeries(val)
-	return successExitCode
-}
-
-// QueryLabels queries for label values against a Prometheus server.
-func QueryLabels(url *url.URL, roundTripper http.RoundTripper, matchers []string, name, start, end string, p printer) int {
-	if url.Scheme == "" {
-		url.Scheme = "http"
-	}
-	config := api.Config{
-		Address:      url.String(),
-		RoundTripper: roundTripper,
-	}
-
-	// Create new client.
-	c, err := api.NewClient(config)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, "error creating API client:", err)
-		return failureExitCode
-	}
-
-	stime, etime, err := parseStartTimeAndEndTime(start, end)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		return failureExitCode
-	}
-
-	// Run query against client.
-	api := v1.NewAPI(c)
-	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
-	val, warn, err := api.LabelValues(ctx, name, matchers, stime, etime)
-	cancel()
-
-	for _, v := range warn {
-		fmt.Fprintln(os.Stderr, "query warning:", v)
-	}
-	if err != nil {
-		return handleAPIError(err)
-	}
-
-	p.printLabelValues(val)
-	return successExitCode
-}
-
-func handleAPIError(err error) int {
-	var apiErr *v1.Error
-	if errors.As(err, &apiErr) && apiErr.Detail != "" {
-		fmt.Fprintf(os.Stderr, "query error: %v (detail: %s)\n", apiErr, strings.TrimSpace(apiErr.Detail))
-	} else {
-		fmt.Fprintln(os.Stderr, "query error:", err)
-	}
-
-	return failureExitCode
-}
-
-func parseStartTimeAndEndTime(start, end string) (time.Time, time.Time, error) {
-	var (
-		minTime = time.Now().Add(-9999 * time.Hour)
-		maxTime = time.Now().Add(9999 * time.Hour)
-		err     error
-	)
-
-	stime := minTime
-	etime := maxTime
-
-	if start != "" {
-		stime, err = parseTime(start)
-		if err != nil {
-			return stime, etime, fmt.Errorf("error parsing start time: %w", err)
-		}
-	}
-
-	if end != "" {
-		etime, err = parseTime(end)
-		if err != nil {
-			return stime, etime, fmt.Errorf("error parsing end time: %w", err)
-		}
-	}
-	return stime, etime, nil
-}
-
-func parseTime(s string) (time.Time, error) {
-	if t, err := strconv.ParseFloat(s, 64); err == nil {
-		s, ns := math.Modf(t)
-		return time.Unix(int64(s), int64(ns*float64(time.Second))).UTC(), nil
-	}
-	if t, err := time.Parse(time.RFC3339Nano, s); err == nil {
-		return t, nil
-	}
-	return time.Time{}, fmt.Errorf("cannot parse %q to a valid timestamp", s)
-}
-
 type endpointsGroup struct {
 	urlToFilename map[string]string
 	postProcess   func(b []byte) ([]byte, error)
@@ -1389,15 +1161,12 @@ func importRules(url *url.URL, roundTripper http.RoundTripper, start, end, outpu
 		evalInterval:     evalInterval,
 		maxBlockDuration: maxBlockDuration,
 	}
-	client, err := api.NewClient(api.Config{
+	api, err := newAPI(url, roundTripper, nil)
-		Address:      url.String(),
-		RoundTripper: roundTripper,
-	})
 	if err != nil {
 		return fmt.Errorf("new api client error: %w", err)
 	}
 
-	ruleImporter := newRuleImporter(log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr)), cfg, v1.NewAPI(client))
+	ruleImporter := newRuleImporter(log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr)), cfg, api)
 	errs := ruleImporter.loadGroups(ctx, files)
 	for _, err := range errs {
 		if err != nil {

cmd/promtool/query.go

@@ -0,0 +1,251 @@
+// Copyright 2023 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math"
+	"net/http"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/prometheus/client_golang/api"
+	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
+	_ "github.com/prometheus/prometheus/plugins" // Register plugins.
+)
+
+func newAPI(url *url.URL, roundTripper http.RoundTripper, headers map[string]string) (v1.API, error) {
+	if url.Scheme == "" {
+		url.Scheme = "http"
+	}
+	config := api.Config{
+		Address:      url.String(),
+		RoundTripper: roundTripper,
+	}
+
+	if len(headers) > 0 {
+		config.RoundTripper = promhttp.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
+			for key, value := range headers {
+				req.Header.Add(key, value)
+			}
+			return roundTripper.RoundTrip(req)
+		})
+	}
+
+	// Create new client.
+	client, err := api.NewClient(config)
+	if err != nil {
+		return nil, err
+	}
+
+	api := v1.NewAPI(client)
+	return api, nil
+}
+
+// QueryInstant performs an instant query against a Prometheus server.
+func QueryInstant(url *url.URL, roundTripper http.RoundTripper, query, evalTime string, p printer) int {
+	api, err := newAPI(url, roundTripper, nil)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "error creating API client:", err)
+		return failureExitCode
+	}
+
+	eTime := time.Now()
+	if evalTime != "" {
+		eTime, err = parseTime(evalTime)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, "error parsing evaluation time:", err)
+			return failureExitCode
+		}
+	}
+
+	// Run query against client.
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	val, _, err := api.Query(ctx, query, eTime) // Ignoring warnings for now.
+	cancel()
+	if err != nil {
+		return handleAPIError(err)
+	}
+
+	p.printValue(val)
+
+	return successExitCode
+}
+
+// QueryRange performs a range query against a Prometheus server.
+func QueryRange(url *url.URL, roundTripper http.RoundTripper, headers map[string]string, query, start, end string, step time.Duration, p printer) int {
+	api, err := newAPI(url, roundTripper, headers)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "error creating API client:", err)
+		return failureExitCode
+	}
+
+	var stime, etime time.Time
+
+	if end == "" {
+		etime = time.Now()
+	} else {
+		etime, err = parseTime(end)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, "error parsing end time:", err)
+			return failureExitCode
+		}
+	}
+
+	if start == "" {
+		stime = etime.Add(-5 * time.Minute)
+	} else {
+		stime, err = parseTime(start)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, "error parsing start time:", err)
+			return failureExitCode
+		}
+	}
+
+	if !stime.Before(etime) {
+		fmt.Fprintln(os.Stderr, "start time is not before end time")
+		return failureExitCode
+	}
+
+	if step == 0 {
+		resolution := math.Max(math.Floor(etime.Sub(stime).Seconds()/250), 1)
+		// Convert seconds to nanoseconds such that time.Duration parses correctly.
+		step = time.Duration(resolution) * time.Second
+	}
+
+	// Run query against client.
+	r := v1.Range{Start: stime, End: etime, Step: step}
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	val, _, err := api.QueryRange(ctx, query, r) // Ignoring warnings for now.
+	cancel()
+
+	if err != nil {
+		return handleAPIError(err)
+	}
+
+	p.printValue(val)
+	return successExitCode
+}
+
+// QuerySeries queries for a series against a Prometheus server.
+func QuerySeries(url *url.URL, roundTripper http.RoundTripper, matchers []string, start, end string, p printer) int {
+	api, err := newAPI(url, roundTripper, nil)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "error creating API client:", err)
+		return failureExitCode
+	}
+
+	stime, etime, err := parseStartTimeAndEndTime(start, end)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		return failureExitCode
+	}
+
+	// Run query against client.
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	val, _, err := api.Series(ctx, matchers, stime, etime) // Ignoring warnings for now.
+	cancel()
+
+	if err != nil {
+		return handleAPIError(err)
+	}
+
+	p.printSeries(val)
+	return successExitCode
+}
+
+// QueryLabels queries for label values against a Prometheus server.
+func QueryLabels(url *url.URL, roundTripper http.RoundTripper, matchers []string, name, start, end string, p printer) int {
+	api, err := newAPI(url, roundTripper, nil)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "error creating API client:", err)
+		return failureExitCode
+	}
+
+	stime, etime, err := parseStartTimeAndEndTime(start, end)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		return failureExitCode
+	}
+
+	// Run query against client.
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	val, warn, err := api.LabelValues(ctx, name, matchers, stime, etime)
+	cancel()
+
+	for _, v := range warn {
+		fmt.Fprintln(os.Stderr, "query warning:", v)
+	}
+	if err != nil {
+		return handleAPIError(err)
+	}
+
+	p.printLabelValues(val)
+	return successExitCode
+}
+
+func handleAPIError(err error) int {
+	var apiErr *v1.Error
+	if errors.As(err, &apiErr) && apiErr.Detail != "" {
+		fmt.Fprintf(os.Stderr, "query error: %v (detail: %s)\n", apiErr, strings.TrimSpace(apiErr.Detail))
+	} else {
+		fmt.Fprintln(os.Stderr, "query error:", err)
+	}
+
+	return failureExitCode
+}
+
+func parseStartTimeAndEndTime(start, end string) (time.Time, time.Time, error) {
+	var (
+		minTime = time.Now().Add(-9999 * time.Hour)
+		maxTime = time.Now().Add(9999 * time.Hour)
+		err     error
+	)
+
+	stime := minTime
+	etime := maxTime
+
+	if start != "" {
+		stime, err = parseTime(start)
+		if err != nil {
+			return stime, etime, fmt.Errorf("error parsing start time: %w", err)
+		}
+	}
+
+	if end != "" {
+		etime, err = parseTime(end)
+		if err != nil {
+			return stime, etime, fmt.Errorf("error parsing end time: %w", err)
+		}
+	}
+	return stime, etime, nil
+}
+
+func parseTime(s string) (time.Time, error) {
+	if t, err := strconv.ParseFloat(s, 64); err == nil {
+		s, ns := math.Modf(t)
+		return time.Unix(int64(s), int64(ns*float64(time.Second))).UTC(), nil
+	}
+	if t, err := time.Parse(time.RFC3339Nano, s); err == nil {
+		return t, nil
+	}
+	return time.Time{}, fmt.Errorf("cannot parse %q to a valid timestamp", s)
+}

cmd/promtool/sd.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/discovery"
@@ -37,7 +38,7 @@ type sdCheckResult struct {
 }
 
 // CheckSD performs service discovery for the given job name and reports the results.
-func CheckSD(sdConfigFiles, sdJobName string, sdTimeout time.Duration, noDefaultScrapePort bool) int {
+func CheckSD(sdConfigFiles, sdJobName string, sdTimeout time.Duration, noDefaultScrapePort bool, registerer prometheus.Registerer) int {
 	logger := log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr))
 
 	cfg, err := config.LoadFile(sdConfigFiles, false, false, logger)
@@ -77,12 +78,25 @@ func CheckSD(sdConfigFiles, sdJobName string, sdTimeout time.Duration, noDefault
 	defer cancel()
 
 	for _, cfg := range scrapeConfig.ServiceDiscoveryConfigs {
-		d, err := cfg.NewDiscoverer(discovery.DiscovererOptions{Logger: logger})
+		reg := prometheus.NewRegistry()
+		refreshMetrics := discovery.NewRefreshMetrics(reg)
+		metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+		err := metrics.Register()
+		if err != nil {
+			fmt.Fprintln(os.Stderr, "Could not register service discovery metrics", err)
+			return failureExitCode
+		}
+
+		d, err := cfg.NewDiscoverer(discovery.DiscovererOptions{Logger: logger, Metrics: metrics})
 		if err != nil {
 			fmt.Fprintln(os.Stderr, "Could not create new discoverer", err)
 			return failureExitCode
 		}
-		go d.Run(ctx, targetGroupChan)
+		go func() {
+			d.Run(ctx, targetGroupChan)
+			metrics.Unregister()
+			refreshMetrics.Unregister()
+		}()
 	}
 
 	var targetGroups []*targetgroup.Group

cmd/promtool/testdata/dump-test-1.prom

@@ -0,0 +1,15 @@
+{__name__="heavy_metric", foo="bar"} 5 0
+{__name__="heavy_metric", foo="bar"} 4 60000
+{__name__="heavy_metric", foo="bar"} 3 120000
+{__name__="heavy_metric", foo="bar"} 2 180000
+{__name__="heavy_metric", foo="bar"} 1 240000
+{__name__="heavy_metric", foo="foo"} 5 0
+{__name__="heavy_metric", foo="foo"} 4 60000
+{__name__="heavy_metric", foo="foo"} 3 120000
+{__name__="heavy_metric", foo="foo"} 2 180000
+{__name__="heavy_metric", foo="foo"} 1 240000
+{__name__="metric", baz="abc", foo="bar"} 1 0
+{__name__="metric", baz="abc", foo="bar"} 2 60000
+{__name__="metric", baz="abc", foo="bar"} 3 120000
+{__name__="metric", baz="abc", foo="bar"} 4 180000
+{__name__="metric", baz="abc", foo="bar"} 5 240000

cmd/promtool/testdata/dump-test-2.prom

@@ -0,0 +1,10 @@
+{__name__="heavy_metric", foo="foo"} 5 0
+{__name__="heavy_metric", foo="foo"} 4 60000
+{__name__="heavy_metric", foo="foo"} 3 120000
+{__name__="heavy_metric", foo="foo"} 2 180000
+{__name__="heavy_metric", foo="foo"} 1 240000
+{__name__="metric", baz="abc", foo="bar"} 1 0
+{__name__="metric", baz="abc", foo="bar"} 2 60000
+{__name__="metric", baz="abc", foo="bar"} 3 120000
+{__name__="metric", baz="abc", foo="bar"} 4 180000
+{__name__="metric", baz="abc", foo="bar"} 5 240000

cmd/promtool/testdata/dump-test-3.prom

@@ -0,0 +1,2 @@
+{__name__="metric", baz="abc", foo="bar"} 2 60000
+{__name__="metric", baz="abc", foo="bar"} 3 120000

cmd/promtool/tsdb.go

@@ -667,7 +667,7 @@ func analyzeCompaction(ctx context.Context, block tsdb.BlockReader, indexr tsdb.
 				it := fhchk.Iterator(nil)
 				bucketCount := 0
 				for it.Next() == chunkenc.ValFloatHistogram {
-					_, f := it.AtFloatHistogram()
+					_, f := it.AtFloatHistogram(nil)
 					bucketCount += len(f.PositiveBuckets)
 					bucketCount += len(f.NegativeBuckets)
 				}
@@ -682,7 +682,7 @@ func analyzeCompaction(ctx context.Context, block tsdb.BlockReader, indexr tsdb.
 				it := hchk.Iterator(nil)
 				bucketCount := 0
 				for it.Next() == chunkenc.ValHistogram {
-					_, f := it.AtHistogram()
+					_, f := it.AtHistogram(nil)
 					bucketCount += len(f.PositiveBuckets)
 					bucketCount += len(f.NegativeBuckets)
 				}
@@ -706,7 +706,7 @@ func analyzeCompaction(ctx context.Context, block tsdb.BlockReader, indexr tsdb.
 	return nil
 }
 
-func dumpSamples(ctx context.Context, path string, mint, maxt int64, match string) (err error) {
+func dumpSamples(ctx context.Context, path string, mint, maxt int64, match []string) (err error) {
 	db, err := tsdb.OpenDBReadOnly(path, nil)
 	if err != nil {
 		return err
@@ -720,11 +720,21 @@ func dumpSamples(ctx context.Context, path string, mint, maxt int64, match strin
 	}
 	defer q.Close()
 
-	matchers, err := parser.ParseMetricSelector(match)
+	matcherSets, err := parser.ParseMetricSelectors(match)
 	if err != nil {
 		return err
 	}
-	ss := q.Select(ctx, false, nil, matchers...)
+
+	var ss storage.SeriesSet
+	if len(matcherSets) > 1 {
+		var sets []storage.SeriesSet
+		for _, mset := range matcherSets {
+			sets = append(sets, q.Select(ctx, true, nil, mset...))
+		}
+		ss = storage.NewMergeSeriesSet(sets, storage.ChainedSeriesMerge)
+	} else {
+		ss = q.Select(ctx, false, nil, matcherSets[0]...)
+	}
 
 	for ss.Next() {
 		series := ss.At()
@@ -735,11 +745,11 @@ func dumpSamples(ctx context.Context, path string, mint, maxt int64, match strin
 			fmt.Printf("%s %g %d\n", lbs, val, ts)
 		}
 		for it.Next() == chunkenc.ValFloatHistogram {
-			ts, fh := it.AtFloatHistogram()
+			ts, fh := it.AtFloatHistogram(nil)
 			fmt.Printf("%s %s %d\n", lbs, fh.String(), ts)
 		}
 		for it.Next() == chunkenc.ValHistogram {
-			ts, h := it.AtHistogram()
+			ts, h := it.AtHistogram(nil)
 			fmt.Printf("%s %s %d\n", lbs, h.String(), ts)
 		}
 		if it.Err() != nil {

cmd/promtool/tsdb_test.go

@@ -14,9 +14,18 @@
 package main
 
 import (
+	"bytes"
+	"context"
+	"io"
+	"math"
+	"os"
+	"runtime"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/prometheus/promql"
 )
 
 func TestGenerateBucket(t *testing.T) {
@@ -41,3 +50,101 @@ func TestGenerateBucket(t *testing.T) {
 		require.Equal(t, tc.step, step)
 	}
 }
+
+// getDumpedSamples dumps samples and returns them.
+func getDumpedSamples(t *testing.T, path string, mint, maxt int64, match []string) string {
+	t.Helper()
+
+	oldStdout := os.Stdout
+	r, w, _ := os.Pipe()
+	os.Stdout = w
+
+	err := dumpSamples(
+		context.Background(),
+		path,
+		mint,
+		maxt,
+		match,
+	)
+	require.NoError(t, err)
+
+	w.Close()
+	os.Stdout = oldStdout
+
+	var buf bytes.Buffer
+	io.Copy(&buf, r)
+	return buf.String()
+}
+
+func TestTSDBDump(t *testing.T) {
+	storage := promql.LoadedStorage(t, `
+		load 1m
+			metric{foo="bar", baz="abc"} 1 2 3 4 5
+			heavy_metric{foo="bar"} 5 4 3 2 1
+			heavy_metric{foo="foo"} 5 4 3 2 1
+	`)
+
+	tests := []struct {
+		name         string
+		mint         int64
+		maxt         int64
+		match        []string
+		expectedDump string
+	}{
+		{
+			name:         "default match",
+			mint:         math.MinInt64,
+			maxt:         math.MaxInt64,
+			match:        []string{"{__name__=~'(?s:.*)'}"},
+			expectedDump: "testdata/dump-test-1.prom",
+		},
+		{
+			name:         "same matcher twice",
+			mint:         math.MinInt64,
+			maxt:         math.MaxInt64,
+			match:        []string{"{foo=~'.+'}", "{foo=~'.+'}"},
+			expectedDump: "testdata/dump-test-1.prom",
+		},
+		{
+			name:         "no duplication",
+			mint:         math.MinInt64,
+			maxt:         math.MaxInt64,
+			match:        []string{"{__name__=~'(?s:.*)'}", "{baz='abc'}"},
+			expectedDump: "testdata/dump-test-1.prom",
+		},
+		{
+			name:         "well merged",
+			mint:         math.MinInt64,
+			maxt:         math.MaxInt64,
+			match:        []string{"{__name__='heavy_metric'}", "{baz='abc'}"},
+			expectedDump: "testdata/dump-test-1.prom",
+		},
+		{
+			name:         "multi matchers",
+			mint:         math.MinInt64,
+			maxt:         math.MaxInt64,
+			match:        []string{"{__name__='heavy_metric',foo='foo'}", "{__name__='metric'}"},
+			expectedDump: "testdata/dump-test-2.prom",
+		},
+		{
+			name:         "with reduced mint and maxt",
+			mint:         int64(60000),
+			maxt:         int64(120000),
+			match:        []string{"{__name__='metric'}"},
+			expectedDump: "testdata/dump-test-3.prom",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			dumpedMetrics := getDumpedSamples(t, storage.Dir(), tt.mint, tt.maxt, tt.match)
+			expectedMetrics, err := os.ReadFile(tt.expectedDump)
+			require.NoError(t, err)
+			if strings.Contains(runtime.GOOS, "windows") {
+				// We use "/n" while dumping on windows as well.
+				expectedMetrics = bytes.ReplaceAll(expectedMetrics, []byte("\r\n"), []byte("\n"))
+			}
+			// even though in case of one matcher samples are not sorted, the order in the cases above should stay the same.
+			require.Equal(t, string(expectedMetrics), dumpedMetrics)
+		})
+	}
+}

cmd/promtool/unittest.go

@@ -15,6 +15,7 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
@@ -27,6 +28,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/grafana/regexp"
+	"github.com/nsf/jsondiff"
 	"github.com/prometheus/common/model"
 	"gopkg.in/yaml.v2"
 
@@ -40,7 +42,7 @@ import (
 
 // RulesUnitTest does unit testing of rules based on the unit testing files provided.
 // More info about the file format can be found in the docs.
-func RulesUnitTest(queryOpts promql.LazyLoaderOpts, runStrings []string, files ...string) int {
+func RulesUnitTest(queryOpts promql.LazyLoaderOpts, runStrings []string, diffFlag bool, files ...string) int {
 	failed := false
 
 	var run *regexp.Regexp
@@ -49,7 +51,7 @@ func RulesUnitTest(queryOpts promql.LazyLoaderOpts, runStrings []string, files .
 	}
 
 	for _, f := range files {
-		if errs := ruleUnitTest(f, queryOpts, run); errs != nil {
+		if errs := ruleUnitTest(f, queryOpts, run, diffFlag); errs != nil {
 			fmt.Fprintln(os.Stderr, "  FAILED:")
 			for _, e := range errs {
 				fmt.Fprintln(os.Stderr, e.Error())
@@ -67,7 +69,7 @@ func RulesUnitTest(queryOpts promql.LazyLoaderOpts, runStrings []string, files .
 	return successExitCode
 }
 
-func ruleUnitTest(filename string, queryOpts promql.LazyLoaderOpts, run *regexp.Regexp) []error {
+func ruleUnitTest(filename string, queryOpts promql.LazyLoaderOpts, run *regexp.Regexp, diffFlag bool) []error {
 	fmt.Println("Unit Testing: ", filename)
 
 	b, err := os.ReadFile(filename)
@@ -109,7 +111,7 @@ func ruleUnitTest(filename string, queryOpts promql.LazyLoaderOpts, run *regexp.
 		if t.Interval == 0 {
 			t.Interval = unitTestInp.EvaluationInterval
 		}
-		ers := t.test(evalInterval, groupOrderMap, queryOpts, unitTestInp.RuleFiles...)
+		ers := t.test(evalInterval, groupOrderMap, queryOpts, diffFlag, unitTestInp.RuleFiles...)
 		if ers != nil {
 			errs = append(errs, ers...)
 		}
@@ -173,7 +175,7 @@ type testGroup struct {
 }
 
 // test performs the unit tests.
-func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, queryOpts promql.LazyLoaderOpts, ruleFiles ...string) []error {
+func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, queryOpts promql.LazyLoaderOpts, diffFlag bool, ruleFiles ...string) []error {
 	// Setup testing suite.
 	suite, err := promql.NewLazyLoader(nil, tg.seriesLoadingString(), queryOpts)
 	if err != nil {
@@ -345,8 +347,44 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
 					}
 					expString := indentLines(expAlerts.String(), "            ")
 					gotString := indentLines(gotAlerts.String(), "            ")
-					errs = append(errs, fmt.Errorf("%s    alertname: %s, time: %s, \n        exp:%v, \n        got:%v",
+					if diffFlag {
-						testName, testcase.Alertname, testcase.EvalTime.String(), expString, gotString))
+						// If empty, populates an empty value
+						if gotAlerts.Len() == 0 {
+							gotAlerts = append(gotAlerts, labelAndAnnotation{
+								Labels:      labels.Labels{},
+								Annotations: labels.Labels{},
+							})
+						}
+						// If empty, populates an empty value
+						if expAlerts.Len() == 0 {
+							expAlerts = append(expAlerts, labelAndAnnotation{
+								Labels:      labels.Labels{},
+								Annotations: labels.Labels{},
+							})
+						}
+
+						diffOpts := jsondiff.DefaultConsoleOptions()
+						expAlertsJSON, err := json.Marshal(expAlerts)
+						if err != nil {
+							errs = append(errs, fmt.Errorf("error marshaling expected %s alert: [%s]", tg.TestGroupName, err.Error()))
+							continue
+						}
+
+						gotAlertsJSON, err := json.Marshal(gotAlerts)
+						if err != nil {
+							errs = append(errs, fmt.Errorf("error marshaling received %s alert: [%s]", tg.TestGroupName, err.Error()))
+							continue
+						}
+
+						res, diff := jsondiff.Compare(expAlertsJSON, gotAlertsJSON, &diffOpts)
+						if res != jsondiff.FullMatch {
+							errs = append(errs, fmt.Errorf("%s    alertname: %s, time: %s, \n        diff: %v",
+								testName, testcase.Alertname, testcase.EvalTime.String(), indentLines(diff, "            ")))
+						}
+					} else {
+						errs = append(errs, fmt.Errorf("%s    alertname: %s, time: %s, \n        exp:%v, \n        got:%v",
+							testName, testcase.Alertname, testcase.EvalTime.String(), expString, gotString))
+					}
 				}
 			}
 

cmd/promtool/unittest_test.go

@@ -125,7 +125,7 @@ func TestRulesUnitTest(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := RulesUnitTest(tt.queryOpts, nil, tt.args.files...); got != tt.want {
+			if got := RulesUnitTest(tt.queryOpts, nil, false, tt.args.files...); got != tt.want {
 				t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want)
 			}
 		})
@@ -178,7 +178,7 @@ func TestRulesUnitTestRun(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := RulesUnitTest(tt.queryOpts, tt.args.run, tt.args.files...); got != tt.want {
+			if got := RulesUnitTest(tt.queryOpts, tt.args.run, false, tt.args.files...); got != tt.want {
 				t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want)
 			}
 		})

config/config.go

@@ -454,12 +454,19 @@ var (
 		OpenMetricsText1_0_0: "application/openmetrics-text;version=1.0.0",
 	}
 
+	// DefaultScrapeProtocols is the set of scrape protocols that will be proposed
+	// to scrape target, ordered by priority.
 	DefaultScrapeProtocols = []ScrapeProtocol{
 		OpenMetricsText1_0_0,
 		OpenMetricsText0_0_1,
 		PrometheusText0_0_4,
 	}
-	DefaultNativeHistogramScrapeProtocols = []ScrapeProtocol{
+
+	// DefaultProtoFirstScrapeProtocols is like DefaultScrapeProtocols, but it
+	// favors protobuf Prometheus exposition format.
+	// Used by default for certain feature-flags like
+	// "native-histograms" and "created-timestamp-zero-ingestion".
+	DefaultProtoFirstScrapeProtocols = []ScrapeProtocol{
 		PrometheusProto,
 		OpenMetricsText1_0_0,
 		OpenMetricsText0_0_1,
@@ -603,9 +610,12 @@ type ScrapeConfig struct {
 	// More than this label value length post metric-relabeling will cause the
 	// scrape to fail. 0 means no limit.
 	LabelValueLengthLimit uint `yaml:"label_value_length_limit,omitempty"`
-	// More than this many buckets in a native histogram will cause the scrape to
+	// If there are more than this many buckets in a native histogram,
-	// fail.
+	// buckets will be merged to stay within the limit.
 	NativeHistogramBucketLimit uint `yaml:"native_histogram_bucket_limit,omitempty"`
+	// If the growth factor of one bucket to the next is smaller than this,
+	// buckets will be merged to increase the factor sufficiently.
+	NativeHistogramMinBucketFactor float64 `yaml:"native_histogram_min_bucket_factor,omitempty"`
 	// Keep no more than this many dropped targets per job.
 	// 0 means no limit.
 	KeepDroppedTargets uint `yaml:"keep_dropped_targets,omitempty"`
@@ -1117,6 +1127,9 @@ type QueueConfig struct {
 	MinBackoff       model.Duration `yaml:"min_backoff,omitempty"`
 	MaxBackoff       model.Duration `yaml:"max_backoff,omitempty"`
 	RetryOnRateLimit bool           `yaml:"retry_on_http_429,omitempty"`
+
+	// Samples older than the limit will be dropped.
+	SampleAgeLimit model.Duration `yaml:"sample_age_limit,omitempty"`
 }
 
 // MetadataConfig is the configuration for sending metadata to remote

config/config_default_test.go

@@ -12,7 +12,6 @@
 // limitations under the License.
 
 //go:build !windows
-// +build !windows
 
 package config
 

config/config_test.go

@@ -568,6 +568,7 @@ var expectedConf = &Config{
 			ServiceDiscoveryConfigs: discovery.Configs{
 				&xds.KumaSDConfig{
 					Server:           "http://kuma-control-plane.kuma-system.svc:5676",
+					ClientID:         "main-prometheus",
 					HTTPClientConfig: config.DefaultHTTPClientConfig,
 					RefreshInterval:  model.Duration(15 * time.Second),
 					FetchTimeout:     model.Duration(2 * time.Minute),

config/testdata/conf.good.yml

@@ -221,6 +221,7 @@ scrape_configs:
 
     kuma_sd_configs:
       - server: http://kuma-control-plane.kuma-system.svc:5676
+        client_id: main-prometheus
 
   - job_name: service-marathon
     marathon_sd_configs:

config/testdata/roundtrip.good.yml

@@ -108,6 +108,7 @@ scrape_configs:
 
     kuma_sd_configs:
       - server: http://kuma-control-plane.kuma-system.svc:5676
+        client_id: main-prometheus
 
     marathon_sd_configs:
       - servers:

consoles/node-cpu.html

@@ -47,7 +47,7 @@
   <script>
   new PromConsole.Graph({
     node: document.querySelector("#cpuGraph"),
-    expr: "sum by (mode)(irate(node_cpu_seconds_total{job='node',instance='{{ .Params.instance }}',mode!='idle'}[5m]))",
+    expr: "sum by (mode)(irate(node_cpu_seconds_total{job='node',instance='{{ .Params.instance }}',mode!='idle',mode!='iowait',mode!='steal'}[5m]))",
     renderer: 'area',
     max: {{ with printf "count(count by (cpu)(node_cpu_seconds_total{job='node',instance='%s'}))" .Params.instance | query }}{{ . | first | value }}{{ else}}undefined{{end}},
     yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,

consoles/node-overview.html

@@ -69,7 +69,7 @@
   <script>
   new PromConsole.Graph({
     node: document.querySelector("#cpuGraph"),
-    expr: "sum by (mode)(irate(node_cpu_seconds_total{job='node',instance='{{ .Params.instance }}',mode!='idle'}[5m]))",
+    expr: "sum by (mode)(irate(node_cpu_seconds_total{job='node',instance='{{ .Params.instance }}',mode!='idle',mode!='iowait',mode!='steal'}[5m]))",
     renderer: 'area',
     max: {{ with printf "count(count by (cpu)(node_cpu_seconds_total{job='node',instance='%s'}))" .Params.instance | query }}{{ . | first | value }}{{ else}}undefined{{end}},
     yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,

consoles/node.html

@@ -21,7 +21,7 @@
 <tr>
   <td><a href="node-overview.html?instance={{ .Labels.instance }}">{{ reReplaceAll "(.*?://)([^:/]+?)(:\\d+)?/.*" "$2" .Labels.instance }}</a></td>
   <td{{ if eq (. | value) 1.0 }}>Yes{{ else }} class="alert-danger">No{{ end }}</td>
-  <td>{{ template "prom_query_drilldown" (args (printf "100 * (1 - avg by(instance)(irate(node_cpu_seconds_total{job='node',mode='idle',instance='%s'}[5m])))" .Labels.instance) "%" "printf.1f") }}</td>
+  <td>{{ template "prom_query_drilldown" (args (printf "100 * (1 - avg by(instance) (sum without(mode) (irate(node_cpu_seconds_total{job='node',mode=~'idle|iowait|steal',instance='%s'}[5m]))))" .Labels.instance) "%" "printf.1f") }}</td>
   <td>{{ template "prom_query_drilldown" (args (printf "node_memory_MemFree_bytes{job='node',instance='%s'} + node_memory_Cached_bytes{job='node',instance='%s'} + node_memory_Buffers_bytes{job='node',instance='%s'}" .Labels.instance .Labels.instance .Labels.instance) "B" "humanize1024") }}</td>
 </tr>
 {{ else }}

discovery/README.md

@@ -234,6 +234,11 @@ type Config interface {
 
 type DiscovererOptions struct {
 	Logger log.Logger
+
+	// A registerer for the Discoverer's metrics.
+	Registerer prometheus.Registerer
+	
+	HTTPClientOptions []config.HTTPClientOption
 }
 ```
 

discovery/aws/ec2.go

@@ -30,6 +30,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -96,12 +97,19 @@ type EC2SDConfig struct {
 	HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*EC2SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &ec2Metrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the EC2 Config.
 func (*EC2SDConfig) Name() string { return "ec2" }
 
 // NewDiscoverer returns a Discoverer for the EC2 Config.
 func (c *EC2SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewEC2Discovery(c, opts.Logger), nil
+	return NewEC2Discovery(c, opts.Logger, opts.Metrics)
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface for the EC2 Config.
@@ -147,7 +155,12 @@ type EC2Discovery struct {
 }
 
 // NewEC2Discovery returns a new EC2Discovery which periodically refreshes its targets.
-func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger) *EC2Discovery {
+func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*EC2Discovery, error) {
+	m, ok := metrics.(*ec2Metrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -156,12 +169,15 @@ func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger) *EC2Discovery {
 		cfg:    conf,
 	}
 	d.Discovery = refresh.NewDiscovery(
-		logger,
+		refresh.Options{
-		"ec2",
+			Logger:              logger,
-		time.Duration(d.cfg.RefreshInterval),
+			Mech:                "ec2",
-		d.refresh,
+			Interval:            time.Duration(d.cfg.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
+		},
 	)
-	return d
+	return d, nil
 }
 
 func (d *EC2Discovery) ec2Client(context.Context) (*ec2.EC2, error) {

discovery/aws/lightsail.go

@@ -29,6 +29,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/lightsail"
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -79,12 +80,19 @@ type LightsailSDConfig struct {
 	HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*LightsailSDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &lightsailMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Lightsail Config.
 func (*LightsailSDConfig) Name() string { return "lightsail" }
 
 // NewDiscoverer returns a Discoverer for the Lightsail Config.
 func (c *LightsailSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewLightsailDiscovery(c, opts.Logger), nil
+	return NewLightsailDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface for the Lightsail Config.
@@ -121,20 +129,29 @@ type LightsailDiscovery struct {
 }
 
 // NewLightsailDiscovery returns a new LightsailDiscovery which periodically refreshes its targets.
-func NewLightsailDiscovery(conf *LightsailSDConfig, logger log.Logger) *LightsailDiscovery {
+func NewLightsailDiscovery(conf *LightsailSDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*LightsailDiscovery, error) {
+	m, ok := metrics.(*lightsailMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
+
 	d := &LightsailDiscovery{
 		cfg: conf,
 	}
 	d.Discovery = refresh.NewDiscovery(
-		logger,
+		refresh.Options{
-		"lightsail",
+			Logger:              logger,
-		time.Duration(d.cfg.RefreshInterval),
+			Mech:                "lightsail",
-		d.refresh,
+			Interval:            time.Duration(d.cfg.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
+		},
 	)
-	return d
+	return d, nil
 }
 
 func (d *LightsailDiscovery) lightsailClient() (*lightsail.Lightsail, error) {

discovery/aws/metrics_ec2.go

@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aws
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+type ec2Metrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+var _ discovery.DiscovererMetrics = (*ec2Metrics)(nil)
+
+// Register implements discovery.DiscovererMetrics.
+func (m *ec2Metrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *ec2Metrics) Unregister() {}

discovery/aws/metrics_lightsail.go

@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aws
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+type lightsailMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+var _ discovery.DiscovererMetrics = (*lightsailMetrics)(nil)
+
+// Register implements discovery.DiscovererMetrics.
+func (m *lightsailMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *lightsailMetrics) Unregister() {}

discovery/azure/azure.go

@@ -30,8 +30,8 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
 	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
-	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5"
-	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4"
 	cache "github.com/Code-Hex/go-generics-cache"
 	"github.com/Code-Hex/go-generics-cache/policy/lru"
 	"github.com/go-kit/log"
@@ -79,17 +79,6 @@ var (
 		AuthenticationMethod: authMethodOAuth,
 		HTTPClientConfig:     config_util.DefaultHTTPClientConfig,
 	}
-
-	failuresCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_azure_failures_total",
-			Help: "Number of Azure service discovery refresh failures.",
-		})
-	cacheHitCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_azure_cache_hit_total",
-			Help: "Number of cache hit during refresh.",
-		})
 )
 
 var environments = map[string]cloud.Configuration{
@@ -114,8 +103,6 @@ func CloudConfigurationFromName(name string) (cloud.Configuration, error) {
 
 func init() {
 	discovery.RegisterConfig(&SDConfig{})
-	prometheus.MustRegister(failuresCount)
-	prometheus.MustRegister(cacheHitCount)
 }
 
 // SDConfig is the configuration for Azure based service discovery.
@@ -133,12 +120,17 @@ type SDConfig struct {
 	HTTPClientConfig config_util.HTTPClientConfig `yaml:",inline"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return newDiscovererMetrics(reg, rmi)
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "azure" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger), nil
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 func validateAuthParam(param, name string) error {
@@ -181,33 +173,43 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 
 type Discovery struct {
 	*refresh.Discovery
-	logger log.Logger
+	logger  log.Logger
-	cfg    *SDConfig
+	cfg     *SDConfig
-	port   int
+	port    int
-	cache  *cache.Cache[string, *armnetwork.Interface]
+	cache   *cache.Cache[string, *armnetwork.Interface]
+	metrics *azureMetrics
 }
 
 // NewDiscovery returns a new AzureDiscovery which periodically refreshes its targets.
-func NewDiscovery(cfg *SDConfig, logger log.Logger) *Discovery {
+func NewDiscovery(cfg *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*azureMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
 	l := cache.New(cache.AsLRU[string, *armnetwork.Interface](lru.WithCapacity(5000)))
 	d := &Discovery{
-		cfg:    cfg,
+		cfg:     cfg,
-		port:   cfg.Port,
+		port:    cfg.Port,
-		logger: logger,
+		logger:  logger,
-		cache:  l,
+		cache:   l,
+		metrics: m,
 	}
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
+		refresh.Options{
-		"azure",
+			Logger:              logger,
-		time.Duration(cfg.RefreshInterval),
+			Mech:                "azure",
-		d.refresh,
+			Interval:            time.Duration(cfg.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
+		},
 	)
 
-	return d
+	return d, nil
 }
 
 // azureClient represents multiple Azure Resource Manager providers.
@@ -330,14 +332,14 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 
 	client, err := createAzureClient(*d.cfg)
 	if err != nil {
-		failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, fmt.Errorf("could not create Azure client: %w", err)
 	}
 	client.logger = d.logger
 
 	machines, err := client.getVMs(ctx, d.cfg.ResourceGroup)
 	if err != nil {
-		failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, fmt.Errorf("could not get virtual machines: %w", err)
 	}
 
@@ -346,14 +348,14 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 	// Load the vms managed by scale sets.
 	scaleSets, err := client.getScaleSets(ctx, d.cfg.ResourceGroup)
 	if err != nil {
-		failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, fmt.Errorf("could not get virtual machine scale sets: %w", err)
 	}
 
 	for _, scaleSet := range scaleSets {
 		scaleSetVms, err := client.getScaleSetVMs(ctx, scaleSet)
 		if err != nil {
-			failuresCount.Inc()
+			d.metrics.failuresCount.Inc()
 			return nil, fmt.Errorf("could not get virtual machine scale set vms: %w", err)
 		}
 		machines = append(machines, scaleSetVms...)
@@ -404,18 +406,18 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 				var networkInterface *armnetwork.Interface
 				if v, ok := d.getFromCache(nicID); ok {
 					networkInterface = v
-					cacheHitCount.Add(1)
+					d.metrics.cacheHitCount.Add(1)
 				} else {
 					if vm.ScaleSet == "" {
 						networkInterface, err = client.getVMNetworkInterfaceByID(ctx, nicID)
-						if err != nil {
+					} else {
-							if errors.Is(err, errorNotFound) {
+						networkInterface, err = client.getVMScaleSetVMNetworkInterfaceByID(ctx, nicID, vm.ScaleSet, vm.InstanceID)
-								level.Warn(d.logger).Log("msg", "Network interface does not exist", "name", nicID, "err", err)
+					}
-							} else {
+					if err != nil {
-								ch <- target{labelSet: nil, err: err}
+						if errors.Is(err, errorNotFound) {
-							}
+							level.Warn(d.logger).Log("msg", "Network interface does not exist", "name", nicID, "err", err)
-							// Get out of this routine because we cannot continue without a network interface.
+						} else {
-							return
+							ch <- target{labelSet: nil, err: err}
 						}
 						d.addToCache(nicID, networkInterface)
 					} else {
@@ -477,7 +479,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 	var tg targetgroup.Group
 	for tgt := range ch {
 		if tgt.err != nil {
-			failuresCount.Inc()
+			d.metrics.failuresCount.Inc()
 			return nil, fmt.Errorf("unable to complete Azure service discovery: %w", tgt.err)
 		}
 		if tgt.labelSet != nil {

discovery/azure/azure_test.go

@@ -17,7 +17,7 @@ import (
 	"testing"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/arm"
-	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
 )

discovery/azure/metrics.go

@@ -0,0 +1,64 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package azure
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*azureMetrics)(nil)
+
+type azureMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+
+	failuresCount prometheus.Counter
+	cacheHitCount prometheus.Counter
+
+	metricRegisterer discovery.MetricRegisterer
+}
+
+func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	m := &azureMetrics{
+		refreshMetrics: rmi,
+		failuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_azure_failures_total",
+				Help: "Number of Azure service discovery refresh failures.",
+			}),
+		cacheHitCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_azure_cache_hit_total",
+				Help: "Number of cache hit during refresh.",
+			}),
+	}
+
+	m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
+		m.failuresCount,
+		m.cacheHitCount,
+	})
+
+	return m
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *azureMetrics) Register() error {
+	return m.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *azureMetrics) Unregister() {
+	m.metricRegisterer.UnregisterMetrics()
+}

discovery/consul/consul.go

@@ -71,41 +71,18 @@ const (
 	namespace = "prometheus"
 )
 
-var (
+// DefaultSDConfig is the default Consul SD configuration.
-	rpcFailuresCount = prometheus.NewCounter(
+var DefaultSDConfig = SDConfig{
-		prometheus.CounterOpts{
+	TagSeparator:     ",",
-			Namespace: namespace,
+	Scheme:           "http",
-			Name:      "sd_consul_rpc_failures_total",
+	Server:           "localhost:8500",
-			Help:      "The number of Consul RPC call failures.",
+	AllowStale:       true,
-		})
+	RefreshInterval:  model.Duration(30 * time.Second),
-	rpcDuration = prometheus.NewSummaryVec(
+	HTTPClientConfig: config.DefaultHTTPClientConfig,
-		prometheus.SummaryOpts{
+}
-			Namespace:  namespace,
-			Name:       "sd_consul_rpc_duration_seconds",
-			Help:       "The duration of a Consul RPC call in seconds.",
-			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
-		},
-		[]string{"endpoint", "call"},
-	)
-
-	// Initialize metric vectors.
-	servicesRPCDuration = rpcDuration.WithLabelValues("catalog", "services")
-	serviceRPCDuration  = rpcDuration.WithLabelValues("catalog", "service")
-
-	// DefaultSDConfig is the default Consul SD configuration.
-	DefaultSDConfig = SDConfig{
-		TagSeparator:     ",",
-		Scheme:           "http",
-		Server:           "localhost:8500",
-		AllowStale:       true,
-		RefreshInterval:  model.Duration(30 * time.Second),
-		HTTPClientConfig: config.DefaultHTTPClientConfig,
-	}
-)
 
 func init() {
 	discovery.RegisterConfig(&SDConfig{})
-	prometheus.MustRegister(rpcFailuresCount, rpcDuration)
 }
 
 // SDConfig is the configuration for Consul service discovery.
@@ -142,12 +119,17 @@ type SDConfig struct {
 	HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return newDiscovererMetrics(reg, rmi)
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "consul" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -196,10 +178,16 @@ type Discovery struct {
 	refreshInterval  time.Duration
 	finalizer        func()
 	logger           log.Logger
+	metrics          *consulMetrics
 }
 
 // NewDiscovery returns a new Discovery for the given config.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*consulMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -237,7 +225,9 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 		clientPartition:  conf.Partition,
 		finalizer:        wrapper.CloseIdleConnections,
 		logger:           logger,
+		metrics:          m,
 	}
+
 	return cd, nil
 }
 
@@ -293,7 +283,7 @@ func (d *Discovery) getDatacenter() error {
 	info, err := d.client.Agent().Self()
 	if err != nil {
 		level.Error(d.logger).Log("msg", "Error retrieving datacenter name", "err", err)
-		rpcFailuresCount.Inc()
+		d.metrics.rpcFailuresCount.Inc()
 		return err
 	}
 
@@ -382,7 +372,7 @@ func (d *Discovery) watchServices(ctx context.Context, ch chan<- []*targetgroup.
 	t0 := time.Now()
 	srvs, meta, err := catalog.Services(opts.WithContext(ctx))
 	elapsed := time.Since(t0)
-	servicesRPCDuration.Observe(elapsed.Seconds())
+	d.metrics.servicesRPCDuration.Observe(elapsed.Seconds())
 
 	// Check the context before in order to exit early.
 	select {
@@ -393,7 +383,7 @@ func (d *Discovery) watchServices(ctx context.Context, ch chan<- []*targetgroup.
 
 	if err != nil {
 		level.Error(d.logger).Log("msg", "Error refreshing service list", "err", err)
-		rpcFailuresCount.Inc()
+		d.metrics.rpcFailuresCount.Inc()
 		time.Sleep(retryInterval)
 		return
 	}
@@ -449,13 +439,15 @@ func (d *Discovery) watchServices(ctx context.Context, ch chan<- []*targetgroup.
 
 // consulService contains data belonging to the same service.
 type consulService struct {
-	name         string
+	name               string
-	tags         []string
+	tags               []string
-	labels       model.LabelSet
+	labels             model.LabelSet
-	discovery    *Discovery
+	discovery          *Discovery
-	client       *consul.Client
+	client             *consul.Client
-	tagSeparator string
+	tagSeparator       string
-	logger       log.Logger
+	logger             log.Logger
+	rpcFailuresCount   prometheus.Counter
+	serviceRPCDuration prometheus.Observer
 }
 
 // Start watching a service.
@@ -469,8 +461,10 @@ func (d *Discovery) watchService(ctx context.Context, ch chan<- []*targetgroup.G
 			serviceLabel:    model.LabelValue(name),
 			datacenterLabel: model.LabelValue(d.clientDatacenter),
 		},
-		tagSeparator: d.tagSeparator,
+		tagSeparator:       d.tagSeparator,
-		logger:       d.logger,
+		logger:             d.logger,
+		rpcFailuresCount:   d.metrics.rpcFailuresCount,
+		serviceRPCDuration: d.metrics.serviceRPCDuration,
 	}
 
 	go func() {
@@ -508,7 +502,7 @@ func (srv *consulService) watch(ctx context.Context, ch chan<- []*targetgroup.Gr
 	t0 := time.Now()
 	serviceNodes, meta, err := health.ServiceMultipleTags(srv.name, srv.tags, false, opts.WithContext(ctx))
 	elapsed := time.Since(t0)
-	serviceRPCDuration.Observe(elapsed.Seconds())
+	srv.serviceRPCDuration.Observe(elapsed.Seconds())
 
 	// Check the context before in order to exit early.
 	select {
@@ -520,7 +514,7 @@ func (srv *consulService) watch(ctx context.Context, ch chan<- []*targetgroup.Gr
 
 	if err != nil {
 		level.Error(srv.logger).Log("msg", "Error refreshing service", "service", srv.name, "tags", strings.Join(srv.tags, ","), "err", err)
-		rpcFailuresCount.Inc()
+		srv.rpcFailuresCount.Inc()
 		time.Sleep(retryInterval)
 		return
 	}

discovery/consul/consul_test.go

@@ -22,12 +22,14 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
 	"gopkg.in/yaml.v2"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -35,11 +37,25 @@ func TestMain(m *testing.M) {
 	goleak.VerifyTestMain(m)
 }
 
+// TODO: Add ability to unregister metrics?
+func NewTestMetrics(t *testing.T, conf discovery.Config, reg prometheus.Registerer) discovery.DiscovererMetrics {
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	require.NoError(t, refreshMetrics.Register())
+
+	metrics := conf.NewDiscovererMetrics(prometheus.NewRegistry(), refreshMetrics)
+	require.NoError(t, metrics.Register())
+
+	return metrics
+}
+
 func TestConfiguredService(t *testing.T) {
 	conf := &SDConfig{
 		Services: []string{"configuredServiceName"},
 	}
-	consulDiscovery, err := NewDiscovery(conf, nil)
+
+	metrics := NewTestMetrics(t, conf, prometheus.NewRegistry())
+
+	consulDiscovery, err := NewDiscovery(conf, nil, metrics)
 	if err != nil {
 		t.Errorf("Unexpected error when initializing discovery %v", err)
 	}
@@ -56,7 +72,10 @@ func TestConfiguredServiceWithTag(t *testing.T) {
 		Services:    []string{"configuredServiceName"},
 		ServiceTags: []string{"http"},
 	}
-	consulDiscovery, err := NewDiscovery(conf, nil)
+
+	metrics := NewTestMetrics(t, conf, prometheus.NewRegistry())
+
+	consulDiscovery, err := NewDiscovery(conf, nil, metrics)
 	if err != nil {
 		t.Errorf("Unexpected error when initializing discovery %v", err)
 	}
@@ -151,7 +170,9 @@ func TestConfiguredServiceWithTags(t *testing.T) {
 	}
 
 	for _, tc := range cases {
-		consulDiscovery, err := NewDiscovery(tc.conf, nil)
+		metrics := NewTestMetrics(t, tc.conf, prometheus.NewRegistry())
+
+		consulDiscovery, err := NewDiscovery(tc.conf, nil, metrics)
 		if err != nil {
 			t.Errorf("Unexpected error when initializing discovery %v", err)
 		}
@@ -159,13 +180,15 @@ func TestConfiguredServiceWithTags(t *testing.T) {
 		if ret != tc.shouldWatch {
 			t.Errorf("Expected should watch? %t, got %t. Watched service and tags: %s %+v, input was %s %+v", tc.shouldWatch, ret, tc.conf.Services, tc.conf.ServiceTags, tc.serviceName, tc.serviceTags)
 		}
-
 	}
 }
 
 func TestNonConfiguredService(t *testing.T) {
 	conf := &SDConfig{}
-	consulDiscovery, err := NewDiscovery(conf, nil)
+
+	metrics := NewTestMetrics(t, conf, prometheus.NewRegistry())
+
+	consulDiscovery, err := NewDiscovery(conf, nil, metrics)
 	if err != nil {
 		t.Errorf("Unexpected error when initializing discovery %v", err)
 	}
@@ -262,7 +285,10 @@ func newServer(t *testing.T) (*httptest.Server, *SDConfig) {
 
 func newDiscovery(t *testing.T, config *SDConfig) *Discovery {
 	logger := log.NewNopLogger()
-	d, err := NewDiscovery(config, logger)
+
+	metrics := NewTestMetrics(t, config, prometheus.NewRegistry())
+
+	d, err := NewDiscovery(config, logger, metrics)
 	require.NoError(t, err)
 	return d
 }

discovery/consul/metrics.go

@@ -0,0 +1,73 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package consul
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*consulMetrics)(nil)
+
+type consulMetrics struct {
+	rpcFailuresCount prometheus.Counter
+	rpcDuration      *prometheus.SummaryVec
+
+	servicesRPCDuration prometheus.Observer
+	serviceRPCDuration  prometheus.Observer
+
+	metricRegisterer discovery.MetricRegisterer
+}
+
+func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	m := &consulMetrics{
+		rpcFailuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      "sd_consul_rpc_failures_total",
+				Help:      "The number of Consul RPC call failures.",
+			}),
+		rpcDuration: prometheus.NewSummaryVec(
+			prometheus.SummaryOpts{
+				Namespace:  namespace,
+				Name:       "sd_consul_rpc_duration_seconds",
+				Help:       "The duration of a Consul RPC call in seconds.",
+				Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+			},
+			[]string{"endpoint", "call"},
+		),
+	}
+
+	m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
+		m.rpcFailuresCount,
+		m.rpcDuration,
+	})
+
+	// Initialize metric vectors.
+	m.servicesRPCDuration = m.rpcDuration.WithLabelValues("catalog", "services")
+	m.serviceRPCDuration = m.rpcDuration.WithLabelValues("catalog", "service")
+
+	return m
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *consulMetrics) Register() error {
+	return m.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *consulMetrics) Unregister() {
+	m.metricRegisterer.UnregisterMetrics()
+}

discovery/digitalocean/digitalocean.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/digitalocean/godo"
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/version"
@@ -62,6 +63,13 @@ func init() {
 	discovery.RegisterConfig(&SDConfig{})
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &digitaloceanMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // SDConfig is the configuration for DigitalOcean based service discovery.
 type SDConfig struct {
 	HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
@@ -75,7 +83,7 @@ func (*SDConfig) Name() string { return "digitalocean" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -103,7 +111,12 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*digitaloceanMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	d := &Discovery{
 		port: conf.Port,
 	}
@@ -125,10 +138,13 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 	}
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
+		refresh.Options{
-		"digitalocean",
+			Logger:              logger,
-		time.Duration(conf.RefreshInterval),
+			Mech:                "digitalocean",
-		d.refresh,
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
+		},
 	)
 	return d, nil
 }

discovery/digitalocean/digitalocean_test.go

@@ -20,8 +20,11 @@ import (
 	"testing"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/prometheus/discovery"
 )
 
 type DigitalOceanSDTestSuite struct {
@@ -46,7 +49,15 @@ func TestDigitalOceanSDRefresh(t *testing.T) {
 
 	cfg := DefaultSDConfig
 	cfg.HTTPClientConfig.BearerToken = tokenID
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 	endpoint, err := url.Parse(sdmock.Mock.Endpoint())
 	require.NoError(t, err)

discovery/digitalocean/metrics.go

@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package digitalocean
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*digitaloceanMetrics)(nil)
+
+type digitaloceanMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *digitaloceanMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *digitaloceanMetrics) Unregister() {}

discovery/discoverer_metrics_noop.go

@@ -0,0 +1,28 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package discovery
+
+// Create a dummy metrics struct, because this SD doesn't have any metrics.
+type NoopDiscovererMetrics struct{}
+
+var _ DiscovererMetrics = (*NoopDiscovererMetrics)(nil)
+
+// Register implements discovery.DiscovererMetrics.
+func (*NoopDiscovererMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (*NoopDiscovererMetrics) Unregister() {
+}

discovery/discovery.go

@@ -18,6 +18,7 @@ import (
 	"reflect"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 
 	"github.com/prometheus/prometheus/discovery/targetgroup"
@@ -38,15 +39,47 @@ type Discoverer interface {
 	Run(ctx context.Context, up chan<- []*targetgroup.Group)
 }
 
+// Internal metrics of service discovery mechanisms.
+type DiscovererMetrics interface {
+	Register() error
+	Unregister()
+}
+
 // DiscovererOptions provides options for a Discoverer.
 type DiscovererOptions struct {
 	Logger log.Logger
 
+	Metrics DiscovererMetrics
+
 	// Extra HTTP client options to expose to Discoverers. This field may be
 	// ignored; Discoverer implementations must opt-in to reading it.
 	HTTPClientOptions []config.HTTPClientOption
 }
 
+// Metrics used by the "refresh" package.
+// We define them here in the "discovery" package in order to avoid a cyclic dependency between
+// "discovery" and "refresh".
+type RefreshMetrics struct {
+	Failures prometheus.Counter
+	Duration prometheus.Observer
+}
+
+// Instantiate the metrics used by the "refresh" package.
+type RefreshMetricsInstantiator interface {
+	Instantiate(mech string) *RefreshMetrics
+}
+
+// An interface for registering, unregistering, and instantiating metrics for the "refresh" package.
+// Refresh metrics are registered and unregistered outside of the service discovery mechanism.
+// This is so that the same metrics can be reused across different service discovery mechanisms.
+// To manage refresh metrics inside the SD mechanism, we'd need to use const labels which are
+// specific to that SD. However, doing so would also expose too many unused metrics on
+// the Prometheus /metrics endpoint.
+type RefreshMetricsManager interface {
+	DiscovererMetrics
+	RefreshMetricsInstantiator
+}
+
 // A Config provides the configuration and constructor for a Discoverer.
 type Config interface {
 	// Name returns the name of the discovery mechanism.
@@ -55,6 +88,9 @@ type Config interface {
 	// NewDiscoverer returns a Discoverer for the Config
 	// with the given DiscovererOptions.
 	NewDiscoverer(DiscovererOptions) (Discoverer, error)
+
+	// NewDiscovererMetrics returns the metrics used by the service discovery.
+	NewDiscovererMetrics(prometheus.Registerer, RefreshMetricsInstantiator) DiscovererMetrics
 }
 
 // Configs is a slice of Config values that uses custom YAML marshaling and unmarshaling
@@ -109,6 +145,11 @@ func (c StaticConfig) NewDiscoverer(DiscovererOptions) (Discoverer, error) {
 	return staticDiscoverer(c), nil
 }
 
+// No metrics are needed for this service discovery mechanism.
+func (c StaticConfig) NewDiscovererMetrics(prometheus.Registerer, RefreshMetricsInstantiator) DiscovererMetrics {
+	return &NoopDiscovererMetrics{}
+}
+
 type staticDiscoverer []*targetgroup.Group
 
 func (c staticDiscoverer) Run(ctx context.Context, up chan<- []*targetgroup.Group) {

discovery/dns/dns.go

@@ -49,30 +49,14 @@ const (
 	namespace = "prometheus"
 )
 
-var (
+// DefaultSDConfig is the default DNS SD configuration.
-	dnsSDLookupsCount = prometheus.NewCounter(
+var DefaultSDConfig = SDConfig{
-		prometheus.CounterOpts{
+	RefreshInterval: model.Duration(30 * time.Second),
-			Namespace: namespace,
+	Type:            "SRV",
-			Name:      "sd_dns_lookups_total",
+}
-			Help:      "The number of DNS-SD lookups.",
-		})
-	dnsSDLookupFailuresCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Namespace: namespace,
-			Name:      "sd_dns_lookup_failures_total",
-			Help:      "The number of DNS-SD lookup failures.",
-		})
-
-	// DefaultSDConfig is the default DNS SD configuration.
-	DefaultSDConfig = SDConfig{
-		RefreshInterval: model.Duration(30 * time.Second),
-		Type:            "SRV",
-	}
-)
 
 func init() {
 	discovery.RegisterConfig(&SDConfig{})
-	prometheus.MustRegister(dnsSDLookupFailuresCount, dnsSDLookupsCount)
 }
 
 // SDConfig is the configuration for DNS based service discovery.
@@ -83,12 +67,17 @@ type SDConfig struct {
 	Port            int            `yaml:"port"` // Ignored for SRV records
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return newDiscovererMetrics(reg, rmi)
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "dns" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(*c, opts.Logger), nil
+	return NewDiscovery(*c, opts.Logger, opts.Metrics)
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
@@ -118,16 +107,22 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 // the Discoverer interface.
 type Discovery struct {
 	*refresh.Discovery
-	names  []string
+	names   []string
-	port   int
+	port    int
-	qtype  uint16
+	qtype   uint16
-	logger log.Logger
+	logger  log.Logger
+	metrics *dnsMetrics
 
 	lookupFn func(name string, qtype uint16, logger log.Logger) (*dns.Msg, error)
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf SDConfig, logger log.Logger) *Discovery {
+func NewDiscovery(conf SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*dnsMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -151,14 +146,20 @@ func NewDiscovery(conf SDConfig, logger log.Logger) *Discovery {
 		port:     conf.Port,
 		logger:   logger,
 		lookupFn: lookupWithSearchPath,
+		metrics:  m,
 	}
+
 	d.Discovery = refresh.NewDiscovery(
-		logger,
+		refresh.Options{
-		"dns",
+			Logger:              logger,
-		time.Duration(conf.RefreshInterval),
+			Mech:                "dns",
-		d.refresh,
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
+		},
 	)
-	return d
+
+	return d, nil
 }
 
 func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
@@ -191,9 +192,9 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 
 func (d *Discovery) refreshOne(ctx context.Context, name string, ch chan<- *targetgroup.Group) error {
 	response, err := d.lookupFn(name, d.qtype, d.logger)
-	dnsSDLookupsCount.Inc()
+	d.metrics.dnsSDLookupsCount.Inc()
 	if err != nil {
-		dnsSDLookupFailuresCount.Inc()
+		d.metrics.dnsSDLookupFailuresCount.Inc()
 		return err
 	}
 

discovery/dns/dns_test.go

@@ -22,11 +22,13 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/miekg/dns"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
 	"gopkg.in/yaml.v2"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -252,12 +254,21 @@ func TestDNS(t *testing.T) {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
-			sd := NewDiscovery(tc.config, nil)
+
+			reg := prometheus.NewRegistry()
+			refreshMetrics := discovery.NewRefreshMetrics(reg)
+			metrics := tc.config.NewDiscovererMetrics(reg, refreshMetrics)
+			require.NoError(t, metrics.Register())
+
+			sd, err := NewDiscovery(tc.config, nil, metrics)
+			require.NoError(t, err)
 			sd.lookupFn = tc.lookup
 
 			tgs, err := sd.refresh(context.Background())
 			require.NoError(t, err)
 			require.Equal(t, tc.expected, tgs)
+
+			metrics.Unregister()
 		})
 	}
 }

discovery/dns/metrics.go

@@ -0,0 +1,66 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dns
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*dnsMetrics)(nil)
+
+type dnsMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+
+	dnsSDLookupsCount        prometheus.Counter
+	dnsSDLookupFailuresCount prometheus.Counter
+
+	metricRegisterer discovery.MetricRegisterer
+}
+
+func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	m := &dnsMetrics{
+		refreshMetrics: rmi,
+		dnsSDLookupsCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      "sd_dns_lookups_total",
+				Help:      "The number of DNS-SD lookups.",
+			}),
+		dnsSDLookupFailuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      "sd_dns_lookup_failures_total",
+				Help:      "The number of DNS-SD lookup failures.",
+			}),
+	}
+
+	m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
+		m.dnsSDLookupsCount,
+		m.dnsSDLookupFailuresCount,
+	})
+
+	return m
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *dnsMetrics) Register() error {
+	return m.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *dnsMetrics) Unregister() {
+	m.metricRegisterer.UnregisterMetrics()
+}

discovery/eureka/eureka.go

@@ -16,6 +16,7 @@ package eureka
 import (
 	"context"
 	"errors"
+	"fmt"
 	"net"
 	"net/http"
 	"net/url"
@@ -23,6 +24,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -75,12 +77,19 @@ type SDConfig struct {
 	RefreshInterval  model.Duration          `yaml:"refresh_interval,omitempty"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &eurekaMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "eureka" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -117,7 +126,12 @@ type Discovery struct {
 }
 
 // NewDiscovery creates a new Eureka discovery for the given role.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*eurekaMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "eureka_sd")
 	if err != nil {
 		return nil, err
@@ -128,10 +142,13 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 		server: conf.Server,
 	}
 	d.Discovery = refresh.NewDiscovery(
-		logger,
+		refresh.Options{
-		"eureka",
+			Logger:              logger,
-		time.Duration(conf.RefreshInterval),
+			Mech:                "eureka",
-		d.refresh,
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
+		},
 	)
 	return d, nil
 }

discovery/eureka/eureka_test.go

@@ -20,9 +20,11 @@ import (
 	"net/http/httptest"
 	"testing"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -35,7 +37,17 @@ func testUpdateServices(respHandler http.HandlerFunc) ([]*targetgroup.Group, err
 		Server: ts.URL,
 	}
 
-	md, err := NewDiscovery(&conf, nil)
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := conf.NewDiscovererMetrics(reg, refreshMetrics)
+	err := metrics.Register()
+	if err != nil {
+		return nil, err
+	}
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	md, err := NewDiscovery(&conf, nil, metrics)
 	if err != nil {
 		return nil, err
 	}

discovery/eureka/metrics.go

@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package eureka
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*eurekaMetrics)(nil)
+
+type eurekaMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *eurekaMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *eurekaMetrics) Unregister() {}

discovery/file/file.go

@@ -39,24 +39,6 @@ import (
 )
 
 var (
-	fileSDReadErrorsCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_file_read_errors_total",
-			Help: "The number of File-SD read errors.",
-		})
-	fileSDScanDuration = prometheus.NewSummary(
-		prometheus.SummaryOpts{
-			Name:       "prometheus_sd_file_scan_duration_seconds",
-			Help:       "The duration of the File-SD scan in seconds.",
-			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
-		})
-	fileSDTimeStamp        = NewTimestampCollector()
-	fileWatcherErrorsCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_file_watcher_errors_total",
-			Help: "The number of File-SD errors caused by filesystem watch failures.",
-		})
-
 	patFileSDName = regexp.MustCompile(`^[^*]*(\*[^/]*)?\.(json|yml|yaml|JSON|YML|YAML)$`)
 
 	// DefaultSDConfig is the default file SD configuration.
@@ -67,7 +49,6 @@ var (
 
 func init() {
 	discovery.RegisterConfig(&SDConfig{})
-	prometheus.MustRegister(fileSDReadErrorsCount, fileSDScanDuration, fileSDTimeStamp, fileWatcherErrorsCount)
 }
 
 // SDConfig is the configuration for file based discovery.
@@ -76,12 +57,17 @@ type SDConfig struct {
 	RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return newDiscovererMetrics(reg, rmi)
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "file" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger), nil
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -113,6 +99,9 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 const fileSDFilepathLabel = model.MetaLabelPrefix + "filepath"
 
 // TimestampCollector is a Custom Collector for Timestamps of the files.
+// TODO(ptodev): Now that each file SD has its own TimestampCollector
+// inside discovery/file/metrics.go, we can refactor this collector
+// (or get rid of it) as each TimestampCollector instance will only use one discoverer.
 type TimestampCollector struct {
 	Description *prometheus.Desc
 	discoverers map[*Discovery]struct{}
@@ -187,10 +176,17 @@ type Discovery struct {
 	// This is used to detect deleted target groups.
 	lastRefresh map[string]int
 	logger      log.Logger
+
+	metrics *fileMetrics
 }
 
 // NewDiscovery returns a new file discovery for the given paths.
-func NewDiscovery(conf *SDConfig, logger log.Logger) *Discovery {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	fm, ok := metrics.(*fileMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -200,9 +196,12 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) *Discovery {
 		interval:   time.Duration(conf.RefreshInterval),
 		timestamps: make(map[string]float64),
 		logger:     logger,
+		metrics:    fm,
 	}
-	fileSDTimeStamp.addDiscoverer(disc)
+
-	return disc
+	fm.init(disc)
+
+	return disc, nil
 }
 
 // listFiles returns a list of all files that match the configured patterns.
@@ -242,7 +241,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 	watcher, err := fsnotify.NewWatcher()
 	if err != nil {
 		level.Error(d.logger).Log("msg", "Error adding file watcher", "err", err)
-		fileWatcherErrorsCount.Inc()
+		d.metrics.fileWatcherErrorsCount.Inc()
 		return
 	}
 	d.watcher = watcher
@@ -306,7 +305,7 @@ func (d *Discovery) stop() {
 	done := make(chan struct{})
 	defer close(done)
 
-	fileSDTimeStamp.removeDiscoverer(d)
+	d.metrics.fileSDTimeStamp.removeDiscoverer(d)
 
 	// Closing the watcher will deadlock unless all events and errors are drained.
 	go func() {
@@ -332,13 +331,13 @@ func (d *Discovery) stop() {
 func (d *Discovery) refresh(ctx context.Context, ch chan<- []*targetgroup.Group) {
 	t0 := time.Now()
 	defer func() {
-		fileSDScanDuration.Observe(time.Since(t0).Seconds())
+		d.metrics.fileSDScanDuration.Observe(time.Since(t0).Seconds())
 	}()
 	ref := map[string]int{}
 	for _, p := range d.listFiles() {
 		tgroups, err := d.readFile(p)
 		if err != nil {
-			fileSDReadErrorsCount.Inc()
+			d.metrics.fileSDReadErrorsCount.Inc()
 
 			level.Error(d.logger).Log("msg", "Error reading file", "path", p, "err", err)
 			// Prevent deletion down below.

discovery/file/file_test.go

@@ -24,10 +24,12 @@ import (
 	"testing"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -143,15 +145,28 @@ func (t *testRunner) run(files ...string) {
 	ctx, cancel := context.WithCancel(context.Background())
 	t.cancelSD = cancel
 	go func() {
-		NewDiscovery(
+		conf := &SDConfig{
-			&SDConfig{
+			Files: files,
-				Files: files,
+			// Setting a high refresh interval to make sure that the tests only
-				// Setting a high refresh interval to make sure that the tests only
+			// rely on file watches.
-				// rely on file watches.
+			RefreshInterval: model.Duration(1 * time.Hour),
-				RefreshInterval: model.Duration(1 * time.Hour),
+		}
-			},
+
+		reg := prometheus.NewRegistry()
+		refreshMetrics := discovery.NewRefreshMetrics(reg)
+		metrics := conf.NewDiscovererMetrics(reg, refreshMetrics)
+		require.NoError(t, metrics.Register())
+
+		d, err := NewDiscovery(
+			conf,
 			nil,
-		).Run(ctx, t.ch)
+			metrics,
+		)
+		require.NoError(t, err)
+
+		d.Run(ctx, t.ch)
+
+		metrics.Unregister()
 	}()
 }
 
@@ -188,9 +203,10 @@ func (t *testRunner) targets() []*targetgroup.Group {
 func (t *testRunner) requireUpdate(ref time.Time, expected []*targetgroup.Group) {
 	t.Helper()
 
+	timeout := time.After(defaultWait)
 	for {
 		select {
-		case <-time.After(defaultWait):
+		case <-timeout:
 			t.Fatalf("Expected update but got none")
 			return
 		case <-time.After(defaultWait / 10):

discovery/file/metrics.go

@@ -0,0 +1,76 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package file
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*fileMetrics)(nil)
+
+type fileMetrics struct {
+	fileSDReadErrorsCount  prometheus.Counter
+	fileSDScanDuration     prometheus.Summary
+	fileWatcherErrorsCount prometheus.Counter
+	fileSDTimeStamp        *TimestampCollector
+
+	metricRegisterer discovery.MetricRegisterer
+}
+
+func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	fm := &fileMetrics{
+		fileSDReadErrorsCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_file_read_errors_total",
+				Help: "The number of File-SD read errors.",
+			}),
+		fileSDScanDuration: prometheus.NewSummary(
+			prometheus.SummaryOpts{
+				Name:       "prometheus_sd_file_scan_duration_seconds",
+				Help:       "The duration of the File-SD scan in seconds.",
+				Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+			}),
+		fileWatcherErrorsCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_file_watcher_errors_total",
+				Help: "The number of File-SD errors caused by filesystem watch failures.",
+			}),
+		fileSDTimeStamp: NewTimestampCollector(),
+	}
+
+	fm.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
+		fm.fileSDReadErrorsCount,
+		fm.fileSDScanDuration,
+		fm.fileWatcherErrorsCount,
+		fm.fileSDTimeStamp,
+	})
+
+	return fm
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (fm *fileMetrics) Register() error {
+	return fm.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (fm *fileMetrics) Unregister() {
+	fm.metricRegisterer.UnregisterMetrics()
+}
+
+func (fm *fileMetrics) init(disc *Discovery) {
+	fm.fileSDTimeStamp.addDiscoverer(disc)
+}

discovery/gce/gce.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"golang.org/x/oauth2/google"
 	"google.golang.org/api/compute/v1"
@@ -81,12 +82,19 @@ type SDConfig struct {
 	TagSeparator    string         `yaml:"tag_separator,omitempty"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &gceMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "gce" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(*c, opts.Logger)
+	return NewDiscovery(*c, opts.Logger, opts.Metrics)
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
@@ -121,7 +129,12 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf SDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*gceMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	d := &Discovery{
 		project:      conf.Project,
 		zone:         conf.Zone,
@@ -141,10 +154,13 @@ func NewDiscovery(conf SDConfig, logger log.Logger) (*Discovery, error) {
 	d.isvc = compute.NewInstancesService(d.svc)
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
+		refresh.Options{
-		"gce",
+			Logger:              logger,
-		time.Duration(conf.RefreshInterval),
+			Mech:                "gce",
-		d.refresh,
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
+		},
 	)
 	return d, nil
 }

discovery/gce/metrics.go

@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gce
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*gceMetrics)(nil)
+
+type gceMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *gceMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *gceMetrics) Unregister() {}

discovery/hetzner/hetzner.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/hetznercloud/hcloud-go/v2/hcloud"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -62,12 +63,19 @@ type SDConfig struct {
 	robotEndpoint   string         // For tests only.
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &hetznerMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "hetzner" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 type refresher interface {
@@ -127,17 +135,25 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*refresh.Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*refresh.Discovery, error) {
+	m, ok := metrics.(*hetznerMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	r, err := newRefresher(conf, logger)
 	if err != nil {
 		return nil, err
 	}
 
 	return refresh.NewDiscovery(
-		logger,
+		refresh.Options{
-		"hetzner",
+			Logger:              logger,
-		time.Duration(conf.RefreshInterval),
+			Mech:                "hetzner",
-		r.refresh,
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            r.refresh,
+			MetricsInstantiator: m.refreshMetrics,
+		},
 	), nil
 }
 

discovery/hetzner/metrics.go

@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hetzner
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*hetznerMetrics)(nil)
+
+type hetznerMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *hetznerMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *hetznerMetrics) Unregister() {}

discovery/http/http.go

@@ -45,17 +45,10 @@ var (
 	}
 	userAgent        = fmt.Sprintf("Prometheus/%s", version.Version)
 	matchContentType = regexp.MustCompile(`^(?i:application\/json(;\s*charset=("utf-8"|utf-8))?)$`)
-
-	failuresCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_http_failures_total",
-			Help: "Number of HTTP service discovery refresh failures.",
-		})
 )
 
 func init() {
 	discovery.RegisterConfig(&SDConfig{})
-	prometheus.MustRegister(failuresCount)
 }
 
 // SDConfig is the configuration for HTTP based discovery.
@@ -65,12 +58,17 @@ type SDConfig struct {
 	URL              string                  `yaml:"url"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return newDiscovererMetrics(reg, rmi)
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "http" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.HTTPClientOptions)
+	return NewDiscovery(c, opts.Logger, opts.HTTPClientOptions, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -112,10 +110,16 @@ type Discovery struct {
 	client          *http.Client
 	refreshInterval time.Duration
 	tgLastLength    int
+	metrics         *httpMetrics
 }
 
 // NewDiscovery returns a new HTTP discovery for the given config.
-func NewDiscovery(conf *SDConfig, logger log.Logger, clientOpts []config.HTTPClientOption) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, clientOpts []config.HTTPClientOption, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*httpMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -130,13 +134,17 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, clientOpts []config.HTTPCli
 		url:             conf.URL,
 		client:          client,
 		refreshInterval: time.Duration(conf.RefreshInterval), // Stored to be sent as headers.
+		metrics:         m,
 	}
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
+		refresh.Options{
-		"http",
+			Logger:              logger,
-		time.Duration(conf.RefreshInterval),
+			Mech:                "http",
-		d.Refresh,
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            d.Refresh,
+			MetricsInstantiator: m.refreshMetrics,
+		},
 	)
 	return d, nil
 }
@@ -152,7 +160,7 @@ func (d *Discovery) Refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 
 	resp, err := d.client.Do(req.WithContext(ctx))
 	if err != nil {
-		failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, err
 	}
 	defer func() {
@@ -161,31 +169,31 @@ func (d *Discovery) Refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 	}()
 
 	if resp.StatusCode != http.StatusOK {
-		failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, fmt.Errorf("server returned HTTP status %s", resp.Status)
 	}
 
 	if !matchContentType.MatchString(strings.TrimSpace(resp.Header.Get("Content-Type"))) {
-		failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, fmt.Errorf("unsupported content type %q", resp.Header.Get("Content-Type"))
 	}
 
 	b, err := io.ReadAll(resp.Body)
 	if err != nil {
-		failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, err
 	}
 
 	var targetGroups []*targetgroup.Group
 
 	if err := json.Unmarshal(b, &targetGroups); err != nil {
-		failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, err
 	}
 
 	for i, tg := range targetGroups {
 		if tg == nil {
-			failuresCount.Inc()
+			d.metrics.failuresCount.Inc()
 			err = errors.New("nil target group item found")
 			return nil, err
 		}

discovery/http/http_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -41,7 +42,14 @@ func TestHTTPValidRefresh(t *testing.T) {
 		RefreshInterval:  model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil)
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	defer refreshMetrics.Unregister()
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()
@@ -63,7 +71,7 @@ func TestHTTPValidRefresh(t *testing.T) {
 		},
 	}
 	require.Equal(t, expectedTargets, tgs)
-	require.Equal(t, 0.0, getFailureCount())
+	require.Equal(t, 0.0, getFailureCount(d.metrics.failuresCount))
 }
 
 func TestHTTPInvalidCode(t *testing.T) {
@@ -79,13 +87,20 @@ func TestHTTPInvalidCode(t *testing.T) {
 		RefreshInterval:  model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil)
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	defer refreshMetrics.Unregister()
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()
 	_, err = d.Refresh(ctx)
 	require.EqualError(t, err, "server returned HTTP status 400 Bad Request")
-	require.Equal(t, 1.0, getFailureCount())
+	require.Equal(t, 1.0, getFailureCount(d.metrics.failuresCount))
 }
 
 func TestHTTPInvalidFormat(t *testing.T) {
@@ -101,18 +116,23 @@ func TestHTTPInvalidFormat(t *testing.T) {
 		RefreshInterval:  model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil)
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	defer refreshMetrics.Unregister()
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()
 	_, err = d.Refresh(ctx)
 	require.EqualError(t, err, `unsupported content type "text/plain; charset=utf-8"`)
-	require.Equal(t, 1.0, getFailureCount())
+	require.Equal(t, 1.0, getFailureCount(d.metrics.failuresCount))
 }
 
-var lastFailureCount float64
+func getFailureCount(failuresCount prometheus.Counter) float64 {
-
-func getFailureCount() float64 {
 	failureChan := make(chan prometheus.Metric)
 
 	go func() {
@@ -129,10 +149,7 @@ func getFailureCount() float64 {
 		metric.Write(&counter)
 	}
 
-	// account for failures in prior tests
+	return *counter.Counter.Value
-	count := *counter.Counter.Value - lastFailureCount
-	lastFailureCount = *counter.Counter.Value
-	return count
 }
 
 func TestContentTypeRegex(t *testing.T) {
@@ -417,7 +434,15 @@ func TestSourceDisappeared(t *testing.T) {
 		URL:              ts.URL,
 		RefreshInterval:  model.Duration(1 * time.Second),
 	}
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil)
+
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	defer refreshMetrics.Unregister()
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, metrics)
 	require.NoError(t, err)
 	for _, test := range cases {
 		ctx := context.Background()

discovery/http/metrics.go

@@ -0,0 +1,57 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package http
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*httpMetrics)(nil)
+
+type httpMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+
+	failuresCount prometheus.Counter
+
+	metricRegisterer discovery.MetricRegisterer
+}
+
+func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	m := &httpMetrics{
+		refreshMetrics: rmi,
+		failuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_http_failures_total",
+				Help: "Number of HTTP service discovery refresh failures.",
+			}),
+	}
+
+	m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
+		m.failuresCount,
+	})
+
+	return m
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *httpMetrics) Register() error {
+	return m.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *httpMetrics) Unregister() {
+	m.metricRegisterer.UnregisterMetrics()
+}

discovery/ionos/ionos.go

@@ -15,9 +15,11 @@ package ionos
 
 import (
 	"errors"
+	"fmt"
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -41,7 +43,12 @@ func init() {
 type Discovery struct{}
 
 // NewDiscovery returns a new refresh.Discovery for IONOS Cloud.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*refresh.Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*refresh.Discovery, error) {
+	m, ok := metrics.(*ionosMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if conf.ionosEndpoint == "" {
 		conf.ionosEndpoint = "https://api.ionos.com"
 	}
@@ -52,10 +59,13 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*refresh.Discovery, error)
 	}
 
 	return refresh.NewDiscovery(
-		logger,
+		refresh.Options{
-		"ionos",
+			Logger:              logger,
-		time.Duration(conf.RefreshInterval),
+			Mech:                "ionos",
-		d.refresh,
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
+		},
 	), nil
 }
 
@@ -79,6 +89,13 @@ type SDConfig struct {
 	ionosEndpoint string // For tests only.
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &ionosMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the IONOS Cloud service discovery.
 func (c SDConfig) Name() string {
 	return "ionos"
@@ -86,7 +103,7 @@ func (c SDConfig) Name() string {
 
 // NewDiscoverer returns a new discovery.Discoverer for IONOS Cloud.
 func (c SDConfig) NewDiscoverer(options discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(&c, options.Logger)
+	return NewDiscovery(&c, options.Logger, options.Metrics)
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.

discovery/ionos/metrics.go

@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ionos
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*ionosMetrics)(nil)
+
+type ionosMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *ionosMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *ionosMetrics) Unregister() {}

discovery/kubernetes/endpoints.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	apiv1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/tools/cache"
@@ -30,12 +31,6 @@ import (
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
-var (
-	epAddCount    = eventCount.WithLabelValues("endpoints", "add")
-	epUpdateCount = eventCount.WithLabelValues("endpoints", "update")
-	epDeleteCount = eventCount.WithLabelValues("endpoints", "delete")
-)
-
 // Endpoints discovers new endpoint targets.
 type Endpoints struct {
 	logger log.Logger
@@ -54,10 +49,19 @@ type Endpoints struct {
 }
 
 // NewEndpoints returns a new endpoints discovery.
-func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node cache.SharedInformer) *Endpoints {
+func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node cache.SharedInformer, eventCount *prometheus.CounterVec) *Endpoints {
 	if l == nil {
 		l = log.NewNopLogger()
 	}
+
+	epAddCount := eventCount.WithLabelValues(RoleEndpoint.String(), MetricLabelRoleAdd)
+	epUpdateCount := eventCount.WithLabelValues(RoleEndpoint.String(), MetricLabelRoleUpdate)
+	epDeleteCount := eventCount.WithLabelValues(RoleEndpoint.String(), MetricLabelRoleDelete)
+
+	svcAddCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleAdd)
+	svcUpdateCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleUpdate)
+	svcDeleteCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleDelete)
+
 	e := &Endpoints{
 		logger:           l,
 		endpointsInf:     eps,
@@ -68,7 +72,7 @@ func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node ca
 		podStore:         pod.GetStore(),
 		nodeInf:          node,
 		withNodeMetadata: node != nil,
-		queue:            workqueue.NewNamed("endpoints"),
+		queue:            workqueue.NewNamed(RoleEndpoint.String()),
 	}
 
 	_, err := e.endpointsInf.AddEventHandler(cache.ResourceEventHandlerFuncs{
@@ -365,6 +369,11 @@ func (e *Endpoints) buildEndpoints(eps *apiv1.Endpoints) *targetgroup.Group {
 	// For all seen pods, check all container ports. If they were not covered
 	// by one of the service endpoints, generate targets for them.
 	for _, pe := range seenPods {
+		// PodIP can be empty when a pod is starting or has been evicted.
+		if len(pe.pod.Status.PodIP) == 0 {
+			continue
+		}
+
 		for _, c := range pe.pod.Spec.Containers {
 			for _, cport := range c.Ports {
 				hasSeenPort := func() bool {
@@ -379,21 +388,18 @@ func (e *Endpoints) buildEndpoints(eps *apiv1.Endpoints) *targetgroup.Group {
 					continue
 				}
 
-				// PodIP can be empty when a pod is starting or has been evicted.
+				a := net.JoinHostPort(pe.pod.Status.PodIP, strconv.FormatUint(uint64(cport.ContainerPort), 10))
-				if len(pe.pod.Status.PodIP) != 0 {
+				ports := strconv.FormatUint(uint64(cport.ContainerPort), 10)
-					a := net.JoinHostPort(pe.pod.Status.PodIP, strconv.FormatUint(uint64(cport.ContainerPort), 10))
-					ports := strconv.FormatUint(uint64(cport.ContainerPort), 10)
 
-					target := model.LabelSet{
+				target := model.LabelSet{
-						model.AddressLabel:            lv(a),
+					model.AddressLabel:            lv(a),
-						podContainerNameLabel:         lv(c.Name),
+					podContainerNameLabel:         lv(c.Name),
-						podContainerImageLabel:        lv(c.Image),
+					podContainerImageLabel:        lv(c.Image),
-						podContainerPortNameLabel:     lv(cport.Name),
+					podContainerPortNameLabel:     lv(cport.Name),
-						podContainerPortNumberLabel:   lv(ports),
+					podContainerPortNumberLabel:   lv(ports),
-						podContainerPortProtocolLabel: lv(string(cport.Protocol)),
+					podContainerPortProtocolLabel: lv(string(cport.Protocol)),
-					}
-					tg.Targets = append(tg.Targets, target.Merge(podLabels(pe.pod)))
 				}
+				tg.Targets = append(tg.Targets, target.Merge(podLabels(pe.pod)))
 			}
 		}
 	}

discovery/kubernetes/endpointslice.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	apiv1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/discovery/v1"
@@ -33,12 +34,6 @@ import (
 	"github.com/prometheus/prometheus/util/strutil"
 )
 
-var (
-	epslAddCount    = eventCount.WithLabelValues("endpointslice", "add")
-	epslUpdateCount = eventCount.WithLabelValues("endpointslice", "update")
-	epslDeleteCount = eventCount.WithLabelValues("endpointslice", "delete")
-)
-
 // EndpointSlice discovers new endpoint targets.
 type EndpointSlice struct {
 	logger log.Logger
@@ -57,10 +52,19 @@ type EndpointSlice struct {
 }
 
 // NewEndpointSlice returns a new endpointslice discovery.
-func NewEndpointSlice(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node cache.SharedInformer) *EndpointSlice {
+func NewEndpointSlice(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node cache.SharedInformer, eventCount *prometheus.CounterVec) *EndpointSlice {
 	if l == nil {
 		l = log.NewNopLogger()
 	}
+
+	epslAddCount := eventCount.WithLabelValues(RoleEndpointSlice.String(), MetricLabelRoleAdd)
+	epslUpdateCount := eventCount.WithLabelValues(RoleEndpointSlice.String(), MetricLabelRoleUpdate)
+	epslDeleteCount := eventCount.WithLabelValues(RoleEndpointSlice.String(), MetricLabelRoleDelete)
+
+	svcAddCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleAdd)
+	svcUpdateCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleUpdate)
+	svcDeleteCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleDelete)
+
 	e := &EndpointSlice{
 		logger:             l,
 		endpointSliceInf:   eps,
@@ -71,7 +75,7 @@ func NewEndpointSlice(l log.Logger, eps cache.SharedIndexInformer, svc, pod, nod
 		podStore:           pod.GetStore(),
 		nodeInf:            node,
 		withNodeMetadata:   node != nil,
-		queue:              workqueue.NewNamed("endpointSlice"),
+		queue:              workqueue.NewNamed(RoleEndpointSlice.String()),
 	}
 
 	_, err := e.endpointSliceInf.AddEventHandler(cache.ResourceEventHandlerFuncs{
@@ -401,6 +405,11 @@ func (e *EndpointSlice) buildEndpointSlice(eps endpointSliceAdaptor) *targetgrou
 	// For all seen pods, check all container ports. If they were not covered
 	// by one of the service endpoints, generate targets for them.
 	for _, pe := range seenPods {
+		// PodIP can be empty when a pod is starting or has been evicted.
+		if len(pe.pod.Status.PodIP) == 0 {
+			continue
+		}
+
 		for _, c := range pe.pod.Spec.Containers {
 			for _, cport := range c.Ports {
 				hasSeenPort := func() bool {
@@ -418,21 +427,18 @@ func (e *EndpointSlice) buildEndpointSlice(eps endpointSliceAdaptor) *targetgrou
 					continue
 				}
 
-				// PodIP can be empty when a pod is starting or has been evicted.
+				a := net.JoinHostPort(pe.pod.Status.PodIP, strconv.FormatUint(uint64(cport.ContainerPort), 10))
-				if len(pe.pod.Status.PodIP) != 0 {
+				ports := strconv.FormatUint(uint64(cport.ContainerPort), 10)
-					a := net.JoinHostPort(pe.pod.Status.PodIP, strconv.FormatUint(uint64(cport.ContainerPort), 10))
-					ports := strconv.FormatUint(uint64(cport.ContainerPort), 10)
 
-					target := model.LabelSet{
+				target := model.LabelSet{
-						model.AddressLabel:            lv(a),
+					model.AddressLabel:            lv(a),
-						podContainerNameLabel:         lv(c.Name),
+					podContainerNameLabel:         lv(c.Name),
-						podContainerImageLabel:        lv(c.Image),
+					podContainerImageLabel:        lv(c.Image),
-						podContainerPortNameLabel:     lv(cport.Name),
+					podContainerPortNameLabel:     lv(cport.Name),
-						podContainerPortNumberLabel:   lv(ports),
+					podContainerPortNumberLabel:   lv(ports),
-						podContainerPortProtocolLabel: lv(string(cport.Protocol)),
+					podContainerPortProtocolLabel: lv(string(cport.Protocol)),
-					}
-					tg.Targets = append(tg.Targets, target.Merge(podLabels(pe.pod)))
 				}
+				tg.Targets = append(tg.Targets, target.Merge(podLabels(pe.pod)))
 			}
 		}
 	}

discovery/kubernetes/ingress.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	v1 "k8s.io/api/networking/v1"
 	"k8s.io/api/networking/v1beta1"
@@ -30,12 +31,6 @@ import (
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
-var (
-	ingressAddCount    = eventCount.WithLabelValues("ingress", "add")
-	ingressUpdateCount = eventCount.WithLabelValues("ingress", "update")
-	ingressDeleteCount = eventCount.WithLabelValues("ingress", "delete")
-)
-
 // Ingress implements discovery of Kubernetes ingress.
 type Ingress struct {
 	logger   log.Logger
@@ -45,8 +40,18 @@ type Ingress struct {
 }
 
 // NewIngress returns a new ingress discovery.
-func NewIngress(l log.Logger, inf cache.SharedInformer) *Ingress {
+func NewIngress(l log.Logger, inf cache.SharedInformer, eventCount *prometheus.CounterVec) *Ingress {
-	s := &Ingress{logger: l, informer: inf, store: inf.GetStore(), queue: workqueue.NewNamed("ingress")}
+	ingressAddCount := eventCount.WithLabelValues(RoleIngress.String(), MetricLabelRoleAdd)
+	ingressUpdateCount := eventCount.WithLabelValues(RoleIngress.String(), MetricLabelRoleUpdate)
+	ingressDeleteCount := eventCount.WithLabelValues(RoleIngress.String(), MetricLabelRoleDelete)
+
+	s := &Ingress{
+		logger:   l,
+		informer: inf,
+		store:    inf.GetStore(),
+		queue:    workqueue.NewNamed(RoleIngress.String()),
+	}
+
 	_, err := s.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(o interface{}) {
 			ingressAddCount.Inc()

discovery/kubernetes/kubernetes.go

@@ -58,24 +58,14 @@ import (
 const (
 	// metaLabelPrefix is the meta prefix used for all meta labels.
 	// in this discovery.
-	metaLabelPrefix  = model.MetaLabelPrefix + "kubernetes_"
+	metaLabelPrefix = model.MetaLabelPrefix + "kubernetes_"
-	namespaceLabel   = metaLabelPrefix + "namespace"
+	namespaceLabel  = metaLabelPrefix + "namespace"
-	metricsNamespace = "prometheus_sd_kubernetes"
+	presentValue    = model.LabelValue("true")
-	presentValue     = model.LabelValue("true")
 )
 
 var (
 	// Http header.
 	userAgent = fmt.Sprintf("Prometheus/%s", version.Version)
-	// Custom events metric.
-	eventCount = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Name:      "events_total",
-			Help:      "The number of Kubernetes events handled.",
-		},
-		[]string{"role", "event"},
-	)
 	// DefaultSDConfig is the default Kubernetes SD configuration.
 	DefaultSDConfig = SDConfig{
 		HTTPClientConfig: config.DefaultHTTPClientConfig,
@@ -84,15 +74,6 @@ var (
 
 func init() {
 	discovery.RegisterConfig(&SDConfig{})
-	prometheus.MustRegister(eventCount)
-	// Initialize metric vectors.
-	for _, role := range []string{"endpointslice", "endpoints", "node", "pod", "service", "ingress"} {
-		for _, evt := range []string{"add", "delete", "update"} {
-			eventCount.WithLabelValues(role, evt)
-		}
-	}
-	(&clientGoRequestMetricAdapter{}).Register(prometheus.DefaultRegisterer)
-	(&clientGoWorkqueueMetricsProvider{}).Register(prometheus.DefaultRegisterer)
 }
 
 // Role is role of the service in Kubernetes.
@@ -121,6 +102,16 @@ func (c *Role) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	}
 }
 
+func (c Role) String() string {
+	return string(c)
+}
+
+const (
+	MetricLabelRoleAdd    = "add"
+	MetricLabelRoleDelete = "delete"
+	MetricLabelRoleUpdate = "update"
+)
+
 // SDConfig is the configuration for Kubernetes service discovery.
 type SDConfig struct {
 	APIServer          config.URL              `yaml:"api_server,omitempty"`
@@ -132,12 +123,17 @@ type SDConfig struct {
 	AttachMetadata     AttachMetadataConfig    `yaml:"attach_metadata,omitempty"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return newDiscovererMetrics(reg, rmi)
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "kubernetes" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return New(opts.Logger, c)
+	return New(opts.Logger, opts.Metrics, c)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -274,6 +270,7 @@ type Discovery struct {
 	selectors          roleSelector
 	ownNamespace       string
 	attachMetadata     AttachMetadataConfig
+	metrics            *kubernetesMetrics
 }
 
 func (d *Discovery) getNamespaces() []string {
@@ -292,7 +289,12 @@ func (d *Discovery) getNamespaces() []string {
 }
 
 // New creates a new Kubernetes discovery for the given role.
-func New(l log.Logger, conf *SDConfig) (*Discovery, error) {
+func New(l log.Logger, metrics discovery.DiscovererMetrics, conf *SDConfig) (*Discovery, error) {
+	m, ok := metrics.(*kubernetesMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if l == nil {
 		l = log.NewNopLogger()
 	}
@@ -346,7 +348,7 @@ func New(l log.Logger, conf *SDConfig) (*Discovery, error) {
 		return nil, err
 	}
 
-	return &Discovery{
+	d := &Discovery{
 		client:             c,
 		logger:             l,
 		role:               conf.Role,
@@ -355,7 +357,10 @@ func New(l log.Logger, conf *SDConfig) (*Discovery, error) {
 		selectors:          mapSelector(conf.Selectors),
 		ownNamespace:       ownNamespace,
 		attachMetadata:     conf.AttachMetadata,
-	}, nil
+		metrics:            m,
+	}
+
+	return d, nil
 }
 
 func mapSelector(rawSelector []SelectorConfig) roleSelector {
@@ -391,6 +396,7 @@ const resyncDisabled = 0
 // Run implements the discoverer interface.
 func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 	d.Lock()
+
 	namespaces := d.getNamespaces()
 
 	switch d.role {
@@ -482,6 +488,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 				cache.NewSharedInformer(slw, &apiv1.Service{}, resyncDisabled),
 				cache.NewSharedInformer(plw, &apiv1.Pod{}, resyncDisabled),
 				nodeInf,
+				d.metrics.eventCount,
 			)
 			d.discoverers = append(d.discoverers, eps)
 			go eps.endpointSliceInf.Run(ctx.Done())
@@ -541,6 +548,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 				cache.NewSharedInformer(slw, &apiv1.Service{}, resyncDisabled),
 				cache.NewSharedInformer(plw, &apiv1.Pod{}, resyncDisabled),
 				nodeInf,
+				d.metrics.eventCount,
 			)
 			d.discoverers = append(d.discoverers, eps)
 			go eps.endpointsInf.Run(ctx.Done())
@@ -572,6 +580,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 				log.With(d.logger, "role", "pod"),
 				d.newPodsByNodeInformer(plw),
 				nodeInformer,
+				d.metrics.eventCount,
 			)
 			d.discoverers = append(d.discoverers, pod)
 			go pod.podInf.Run(ctx.Done())
@@ -594,6 +603,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 			svc := NewService(
 				log.With(d.logger, "role", "service"),
 				cache.NewSharedInformer(slw, &apiv1.Service{}, resyncDisabled),
+				d.metrics.eventCount,
 			)
 			d.discoverers = append(d.discoverers, svc)
 			go svc.informer.Run(ctx.Done())
@@ -651,13 +661,14 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 			ingress := NewIngress(
 				log.With(d.logger, "role", "ingress"),
 				informer,
+				d.metrics.eventCount,
 			)
 			d.discoverers = append(d.discoverers, ingress)
 			go ingress.informer.Run(ctx.Done())
 		}
 	case RoleNode:
 		nodeInformer := d.newNodeInformer(ctx)
-		node := NewNode(log.With(d.logger, "role", "node"), nodeInformer)
+		node := NewNode(log.With(d.logger, "role", "node"), nodeInformer, d.metrics.eventCount)
 		d.discoverers = append(d.discoverers, node)
 		go node.informer.Run(ctx.Done())
 	default:

discovery/kubernetes/kubernetes_test.go

@@ -29,6 +29,8 @@ import (
 	"k8s.io/client-go/kubernetes/fake"
 	"k8s.io/client-go/tools/cache"
 
+	"github.com/prometheus/client_golang/prometheus"
+
 	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 	"github.com/prometheus/prometheus/util/testutil"
@@ -49,13 +51,30 @@ func makeDiscoveryWithVersion(role Role, nsDiscovery NamespaceDiscovery, k8sVer
 	fakeDiscovery, _ := clientset.Discovery().(*fakediscovery.FakeDiscovery)
 	fakeDiscovery.FakedServerVersion = &version.Info{GitVersion: k8sVer}
 
-	return &Discovery{
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := newDiscovererMetrics(reg, refreshMetrics)
+	err := metrics.Register()
+	if err != nil {
+		panic(err)
+	}
+	// TODO(ptodev): Unregister the metrics at the end of the test.
+
+	kubeMetrics, ok := metrics.(*kubernetesMetrics)
+	if !ok {
+		panic("invalid discovery metrics type")
+	}
+
+	d := &Discovery{
 		client:             clientset,
 		logger:             log.NewNopLogger(),
 		role:               role,
 		namespaceDiscovery: &nsDiscovery,
 		ownNamespace:       "own-ns",
-	}, clientset
+		metrics:            kubeMetrics,
+	}
+
+	return d, clientset
 }
 
 // makeDiscoveryWithMetadata creates a kubernetes.Discovery instance with the specified metadata config.

discovery/kubernetes/metrics.go

@@ -0,0 +1,75 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kubernetes
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*kubernetesMetrics)(nil)
+
+type kubernetesMetrics struct {
+	eventCount *prometheus.CounterVec
+
+	metricRegisterer discovery.MetricRegisterer
+}
+
+func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	m := &kubernetesMetrics{
+		eventCount: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Namespace: discovery.KubernetesMetricsNamespace,
+				Name:      "events_total",
+				Help:      "The number of Kubernetes events handled.",
+			},
+			[]string{"role", "event"},
+		),
+	}
+
+	m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
+		m.eventCount,
+	})
+
+	// Initialize metric vectors.
+	for _, role := range []string{
+		RoleEndpointSlice.String(),
+		RoleEndpoint.String(),
+		RoleNode.String(),
+		RolePod.String(),
+		RoleService.String(),
+		RoleIngress.String(),
+	} {
+		for _, evt := range []string{
+			MetricLabelRoleAdd,
+			MetricLabelRoleDelete,
+			MetricLabelRoleUpdate,
+		} {
+			m.eventCount.WithLabelValues(role, evt)
+		}
+	}
+
+	return m
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *kubernetesMetrics) Register() error {
+	return m.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *kubernetesMetrics) Unregister() {
+	m.metricRegisterer.UnregisterMetrics()
+}

discovery/kubernetes/node.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	apiv1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/tools/cache"
@@ -35,12 +36,6 @@ const (
 	NodeLegacyHostIP = "LegacyHostIP"
 )
 
-var (
-	nodeAddCount    = eventCount.WithLabelValues("node", "add")
-	nodeUpdateCount = eventCount.WithLabelValues("node", "update")
-	nodeDeleteCount = eventCount.WithLabelValues("node", "delete")
-)
-
 // Node discovers Kubernetes nodes.
 type Node struct {
 	logger   log.Logger
@@ -50,11 +45,22 @@ type Node struct {
 }
 
 // NewNode returns a new node discovery.
-func NewNode(l log.Logger, inf cache.SharedInformer) *Node {
+func NewNode(l log.Logger, inf cache.SharedInformer, eventCount *prometheus.CounterVec) *Node {
 	if l == nil {
 		l = log.NewNopLogger()
 	}
-	n := &Node{logger: l, informer: inf, store: inf.GetStore(), queue: workqueue.NewNamed("node")}
+
+	nodeAddCount := eventCount.WithLabelValues(RoleNode.String(), MetricLabelRoleAdd)
+	nodeUpdateCount := eventCount.WithLabelValues(RoleNode.String(), MetricLabelRoleUpdate)
+	nodeDeleteCount := eventCount.WithLabelValues(RoleNode.String(), MetricLabelRoleDelete)
+
+	n := &Node{
+		logger:   l,
+		informer: inf,
+		store:    inf.GetStore(),
+		queue:    workqueue.NewNamed(RoleNode.String()),
+	}
+
 	_, err := n.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(o interface{}) {
 			nodeAddCount.Inc()

discovery/kubernetes/pod.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	apiv1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -34,12 +35,6 @@ import (
 
 const nodeIndex = "node"
 
-var (
-	podAddCount    = eventCount.WithLabelValues("pod", "add")
-	podUpdateCount = eventCount.WithLabelValues("pod", "update")
-	podDeleteCount = eventCount.WithLabelValues("pod", "delete")
-)
-
 // Pod discovers new pod targets.
 type Pod struct {
 	podInf           cache.SharedIndexInformer
@@ -51,18 +46,22 @@ type Pod struct {
 }
 
 // NewPod creates a new pod discovery.
-func NewPod(l log.Logger, pods cache.SharedIndexInformer, nodes cache.SharedInformer) *Pod {
+func NewPod(l log.Logger, pods cache.SharedIndexInformer, nodes cache.SharedInformer, eventCount *prometheus.CounterVec) *Pod {
 	if l == nil {
 		l = log.NewNopLogger()
 	}
 
+	podAddCount := eventCount.WithLabelValues(RolePod.String(), MetricLabelRoleAdd)
+	podDeleteCount := eventCount.WithLabelValues(RolePod.String(), MetricLabelRoleDelete)
+	podUpdateCount := eventCount.WithLabelValues(RolePod.String(), MetricLabelRoleUpdate)
+
 	p := &Pod{
 		podInf:           pods,
 		nodeInf:          nodes,
 		withNodeMetadata: nodes != nil,
 		store:            pods.GetStore(),
 		logger:           l,
-		queue:            workqueue.NewNamed("pod"),
+		queue:            workqueue.NewNamed(RolePod.String()),
 	}
 	_, err := p.podInf.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(o interface{}) {

discovery/kubernetes/service.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	apiv1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/tools/cache"
@@ -30,12 +31,6 @@ import (
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
-var (
-	svcAddCount    = eventCount.WithLabelValues("service", "add")
-	svcUpdateCount = eventCount.WithLabelValues("service", "update")
-	svcDeleteCount = eventCount.WithLabelValues("service", "delete")
-)
-
 // Service implements discovery of Kubernetes services.
 type Service struct {
 	logger   log.Logger
@@ -45,11 +40,22 @@ type Service struct {
 }
 
 // NewService returns a new service discovery.
-func NewService(l log.Logger, inf cache.SharedInformer) *Service {
+func NewService(l log.Logger, inf cache.SharedInformer, eventCount *prometheus.CounterVec) *Service {
 	if l == nil {
 		l = log.NewNopLogger()
 	}
-	s := &Service{logger: l, informer: inf, store: inf.GetStore(), queue: workqueue.NewNamed("service")}
+
+	svcAddCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleAdd)
+	svcUpdateCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleUpdate)
+	svcDeleteCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleDelete)
+
+	s := &Service{
+		logger:   l,
+		informer: inf,
+		store:    inf.GetStore(),
+		queue:    workqueue.NewNamed(RoleService.String()),
+	}
+
 	_, err := s.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(o interface{}) {
 			svcAddCount.Inc()

discovery/legacymanager/manager.go

@@ -28,48 +28,6 @@ import (
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
-var (
-	failedConfigs = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "prometheus_sd_failed_configs",
-			Help: "Current number of service discovery configurations that failed to load.",
-		},
-		[]string{"name"},
-	)
-	discoveredTargets = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "prometheus_sd_discovered_targets",
-			Help: "Current number of discovered targets.",
-		},
-		[]string{"name", "config"},
-	)
-	receivedUpdates = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_received_updates_total",
-			Help: "Total number of update events received from the SD providers.",
-		},
-		[]string{"name"},
-	)
-	delayedUpdates = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_updates_delayed_total",
-			Help: "Total number of update events that couldn't be sent immediately.",
-		},
-		[]string{"name"},
-	)
-	sentUpdates = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_updates_total",
-			Help: "Total number of update events sent to the SD consumers.",
-		},
-		[]string{"name"},
-	)
-)
-
-func RegisterMetrics() {
-	prometheus.MustRegister(failedConfigs, discoveredTargets, receivedUpdates, delayedUpdates, sentUpdates)
-}
-
 type poolKey struct {
 	setName  string
 	provider string
@@ -84,7 +42,7 @@ type provider struct {
 }
 
 // NewManager is the Discovery Manager constructor.
-func NewManager(ctx context.Context, logger log.Logger, options ...func(*Manager)) *Manager {
+func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Registerer, sdMetrics map[string]discovery.DiscovererMetrics, options ...func(*Manager)) *Manager {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -96,10 +54,22 @@ func NewManager(ctx context.Context, logger log.Logger, options ...func(*Manager
 		ctx:            ctx,
 		updatert:       5 * time.Second,
 		triggerSend:    make(chan struct{}, 1),
+		registerer:     registerer,
+		sdMetrics:      sdMetrics,
 	}
 	for _, option := range options {
 		option(mgr)
 	}
+
+	// Register the metrics.
+	// We have to do this after setting all options, so that the name of the Manager is set.
+	if metrics, err := discovery.NewManagerMetrics(registerer, mgr.name); err == nil {
+		mgr.metrics = metrics
+	} else {
+		level.Error(logger).Log("msg", "Failed to create discovery manager metrics", "manager", mgr.name, "err", err)
+		return nil
+	}
+
 	return mgr
 }
 
@@ -135,6 +105,12 @@ type Manager struct {
 
 	// The triggerSend channel signals to the manager that new updates have been received from providers.
 	triggerSend chan struct{}
+
+	// A registerer for all service discovery metrics.
+	registerer prometheus.Registerer
+
+	metrics   *discovery.Metrics
+	sdMetrics map[string]discovery.DiscovererMetrics
 }
 
 // Run starts the background processing.
@@ -157,7 +133,7 @@ func (m *Manager) ApplyConfig(cfg map[string]discovery.Configs) error {
 
 	for pk := range m.targets {
 		if _, ok := cfg[pk.setName]; !ok {
-			discoveredTargets.DeleteLabelValues(m.name, pk.setName)
+			m.metrics.DiscoveredTargets.DeleteLabelValues(m.name, pk.setName)
 		}
 	}
 	m.cancelDiscoverers()
@@ -168,9 +144,9 @@ func (m *Manager) ApplyConfig(cfg map[string]discovery.Configs) error {
 	failedCount := 0
 	for name, scfg := range cfg {
 		failedCount += m.registerProviders(scfg, name)
-		discoveredTargets.WithLabelValues(m.name, name).Set(0)
+		m.metrics.DiscoveredTargets.WithLabelValues(name).Set(0)
 	}
-	failedConfigs.WithLabelValues(m.name).Set(float64(failedCount))
+	m.metrics.FailedConfigs.Set(float64(failedCount))
 
 	for _, prov := range m.providers {
 		m.startProvider(m.ctx, prov)
@@ -207,7 +183,7 @@ func (m *Manager) updater(ctx context.Context, p *provider, updates chan []*targ
 		case <-ctx.Done():
 			return
 		case tgs, ok := <-updates:
-			receivedUpdates.WithLabelValues(m.name).Inc()
+			m.metrics.ReceivedUpdates.Inc()
 			if !ok {
 				level.Debug(m.logger).Log("msg", "Discoverer channel closed", "provider", p.name)
 				return
@@ -236,11 +212,11 @@ func (m *Manager) sender() {
 		case <-ticker.C: // Some discoverers send updates too often so we throttle these with the ticker.
 			select {
 			case <-m.triggerSend:
-				sentUpdates.WithLabelValues(m.name).Inc()
+				m.metrics.SentUpdates.Inc()
 				select {
 				case m.syncCh <- m.allGroups():
 				default:
-					delayedUpdates.WithLabelValues(m.name).Inc()
+					m.metrics.DelayedUpdates.Inc()
 					level.Debug(m.logger).Log("msg", "Discovery receiver's channel was full so will retry the next cycle")
 					select {
 					case m.triggerSend <- struct{}{}:
@@ -288,7 +264,7 @@ func (m *Manager) allGroups() map[string][]*targetgroup.Group {
 		}
 	}
 	for setName, v := range n {
-		discoveredTargets.WithLabelValues(m.name, setName).Set(float64(v))
+		m.metrics.DiscoveredTargets.WithLabelValues(setName).Set(float64(v))
 	}
 	return tSets
 }
@@ -309,7 +285,8 @@ func (m *Manager) registerProviders(cfgs discovery.Configs, setName string) int
 		}
 		typ := cfg.Name()
 		d, err := cfg.NewDiscoverer(discovery.DiscovererOptions{
-			Logger: log.With(m.logger, "discovery", typ, "config", setName),
+			Logger:  log.With(m.logger, "discovery", typ, "config", setName),
+			Metrics: m.sdMetrics[typ],
 		})
 		if err != nil {
 			level.Error(m.logger).Log("msg", "Cannot create service discovery", "err", err, "type", typ, "config", setName)

discovery/legacymanager/manager_test.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	client_testutil "github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
@@ -35,6 +36,13 @@ func TestMain(m *testing.M) {
 	testutil.TolerantVerifyLeak(m)
 }
 
+func newTestMetrics(t *testing.T, reg prometheus.Registerer) (*discovery.RefreshMetricsManager, map[string]discovery.DiscovererMetrics) {
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	sdMetrics, err := discovery.RegisterSDMetrics(reg, refreshMetrics)
+	require.NoError(t, err)
+	return &refreshMetrics, sdMetrics
+}
+
 // TestTargetUpdatesOrder checks that the target updates are received in the expected order.
 func TestTargetUpdatesOrder(t *testing.T) {
 	// The order by which the updates are send is determined by the interval passed to the mock discovery adapter
@@ -664,7 +672,11 @@ func TestTargetUpdatesOrder(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel()
 
-			discoveryManager := NewManager(ctx, log.NewNopLogger())
+			reg := prometheus.NewRegistry()
+			_, sdMetrics := newTestMetrics(t, reg)
+
+			discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
+			require.NotNil(t, discoveryManager)
 			discoveryManager.updatert = 100 * time.Millisecond
 
 			var totalUpdatesCount int
@@ -746,7 +758,12 @@ func verifyPresence(t *testing.T, tSets map[poolKey]map[string]*targetgroup.Grou
 func TestTargetSetRecreatesTargetGroupsEveryRun(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := newTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -774,7 +791,12 @@ func TestTargetSetRecreatesTargetGroupsEveryRun(t *testing.T) {
 func TestDiscovererConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := newTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -798,7 +820,12 @@ func TestDiscovererConfigs(t *testing.T) {
 func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := newTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -837,7 +864,12 @@ func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 func TestIdenticalConfigurationsAreCoalesced(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, nil)
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := newTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, nil, reg, sdMetrics)
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -868,7 +900,12 @@ func TestApplyConfigDoesNotModifyStaticTargets(t *testing.T) {
 	}
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := newTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -890,10 +927,20 @@ func (e errorConfig) NewDiscoverer(discovery.DiscovererOptions) (discovery.Disco
 	return nil, e.err
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (errorConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &discovery.NoopDiscovererMetrics{}
+}
+
 func TestGaugeFailedConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := newTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -907,7 +954,7 @@ func TestGaugeFailedConfigs(t *testing.T) {
 	discoveryManager.ApplyConfig(c)
 	<-discoveryManager.SyncCh()
 
-	failedCount := client_testutil.ToFloat64(failedConfigs)
+	failedCount := client_testutil.ToFloat64(discoveryManager.metrics.FailedConfigs)
 	if failedCount != 3 {
 		t.Fatalf("Expected to have 3 failed configs, got: %v", failedCount)
 	}
@@ -918,7 +965,7 @@ func TestGaugeFailedConfigs(t *testing.T) {
 	discoveryManager.ApplyConfig(c)
 	<-discoveryManager.SyncCh()
 
-	failedCount = client_testutil.ToFloat64(failedConfigs)
+	failedCount = client_testutil.ToFloat64(discoveryManager.metrics.FailedConfigs)
 	if failedCount != 0 {
 		t.Fatalf("Expected to get no failed config, got: %v", failedCount)
 	}
@@ -1049,7 +1096,11 @@ func TestCoordinationWithReceiver(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel()
 
-			mgr := NewManager(ctx, nil)
+			reg := prometheus.NewRegistry()
+			_, sdMetrics := newTestMetrics(t, reg)
+
+			mgr := NewManager(ctx, nil, reg, sdMetrics)
+			require.NotNil(t, mgr)
 			mgr.updatert = updateDelay
 			go mgr.Run()
 

discovery/linode/linode.go

@@ -67,24 +67,15 @@ const (
 )
 
 // DefaultSDConfig is the default Linode SD configuration.
-var (
+var DefaultSDConfig = SDConfig{
-	DefaultSDConfig = SDConfig{
+	TagSeparator:     ",",
-		TagSeparator:     ",",
+	Port:             80,
-		Port:             80,
+	RefreshInterval:  model.Duration(60 * time.Second),
-		RefreshInterval:  model.Duration(60 * time.Second),
+	HTTPClientConfig: config.DefaultHTTPClientConfig,
-		HTTPClientConfig: config.DefaultHTTPClientConfig,
+}
-	}
-
-	failuresCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_linode_failures_total",
-			Help: "Number of Linode service discovery refresh failures.",
-		})
-)
 
 func init() {
 	discovery.RegisterConfig(&SDConfig{})
-	prometheus.MustRegister(failuresCount)
 }
 
 // SDConfig is the configuration for Linode based service discovery.
@@ -96,12 +87,17 @@ type SDConfig struct {
 	TagSeparator    string         `yaml:"tag_separator,omitempty"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return newDiscovererMetrics(reg, rmi)
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "linode" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -131,16 +127,23 @@ type Discovery struct {
 	pollCount            int
 	lastResults          []*targetgroup.Group
 	eventPollingEnabled  bool
+	metrics              *linodeMetrics
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*linodeMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	d := &Discovery{
 		port:                 conf.Port,
 		tagSeparator:         conf.TagSeparator,
 		pollCount:            0,
 		lastRefreshTimestamp: time.Now().UTC(),
 		eventPollingEnabled:  true,
+		metrics:              m,
 	}
 
 	rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "linode_sd")
@@ -158,10 +161,13 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 	d.client = &client
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
+		refresh.Options{
-		"linode",
+			Logger:              logger,
-		time.Duration(conf.RefreshInterval),
+			Mech:                "linode",
-		d.refresh,
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
+		},
 	)
 	return d, nil
 }
@@ -222,14 +228,14 @@ func (d *Discovery) refreshData(ctx context.Context) ([]*targetgroup.Group, erro
 	// Gather all linode instances.
 	instances, err := d.client.ListInstances(ctx, &linodego.ListOptions{PageSize: 500})
 	if err != nil {
-		failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, err
 	}
 
 	// Gather detailed IP address info for all IPs on all linode instances.
 	detailedIPs, err := d.client.ListIPAddresses(ctx, &linodego.ListOptions{PageSize: 500})
 	if err != nil {
-		failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, err
 	}
 

discovery/linode/linode_test.go

@@ -20,9 +20,12 @@ import (
 	"testing"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/prometheus/discovery"
 )
 
 type LinodeSDTestSuite struct {
@@ -52,7 +55,15 @@ func TestLinodeSDRefresh(t *testing.T) {
 		Credentials: tokenID,
 		Type:        "Bearer",
 	}
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 	endpoint, err := url.Parse(sdmock.Mock.Endpoint())
 	require.NoError(t, err)

discovery/linode/metrics.go

@@ -0,0 +1,57 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package linode
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*linodeMetrics)(nil)
+
+type linodeMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+
+	failuresCount prometheus.Counter
+
+	metricRegisterer discovery.MetricRegisterer
+}
+
+func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	m := &linodeMetrics{
+		refreshMetrics: rmi,
+		failuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_linode_failures_total",
+				Help: "Number of Linode service discovery refresh failures.",
+			}),
+	}
+
+	m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
+		m.failuresCount,
+	})
+
+	return m
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *linodeMetrics) Register() error {
+	return m.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *linodeMetrics) Unregister() {
+	m.metricRegisterer.UnregisterMetrics()
+}

discovery/manager.go

@@ -28,48 +28,6 @@ import (
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
-var (
-	failedConfigs = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "prometheus_sd_failed_configs",
-			Help: "Current number of service discovery configurations that failed to load.",
-		},
-		[]string{"name"},
-	)
-	discoveredTargets = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "prometheus_sd_discovered_targets",
-			Help: "Current number of discovered targets.",
-		},
-		[]string{"name", "config"},
-	)
-	receivedUpdates = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_received_updates_total",
-			Help: "Total number of update events received from the SD providers.",
-		},
-		[]string{"name"},
-	)
-	delayedUpdates = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_updates_delayed_total",
-			Help: "Total number of update events that couldn't be sent immediately.",
-		},
-		[]string{"name"},
-	)
-	sentUpdates = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_updates_total",
-			Help: "Total number of update events sent to the SD consumers.",
-		},
-		[]string{"name"},
-	)
-)
-
-func RegisterMetrics() {
-	prometheus.MustRegister(failedConfigs, discoveredTargets, receivedUpdates, delayedUpdates, sentUpdates)
-}
-
 type poolKey struct {
 	setName  string
 	provider string
@@ -106,8 +64,24 @@ func (p *Provider) Config() interface{} {
 	return p.config
 }
 
+// Registers the metrics needed for SD mechanisms.
+// Does not register the metrics for the Discovery Manager.
+// TODO(ptodev): Add ability to unregister the metrics?
+func CreateAndRegisterSDMetrics(reg prometheus.Registerer) (map[string]DiscovererMetrics, error) {
+	// Some SD mechanisms use the "refresh" package, which has its own metrics.
+	refreshSdMetrics := NewRefreshMetrics(reg)
+
+	// Register the metrics specific for each SD mechanism, and the ones for the refresh package.
+	sdMetrics, err := RegisterSDMetrics(reg, refreshSdMetrics)
+	if err != nil {
+		return nil, fmt.Errorf("failed to register service discovery metrics: %w", err)
+	}
+
+	return sdMetrics, nil
+}
+
 // NewManager is the Discovery Manager constructor.
-func NewManager(ctx context.Context, logger log.Logger, options ...func(*Manager)) *Manager {
+func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Registerer, sdMetrics map[string]DiscovererMetrics, options ...func(*Manager)) *Manager {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -118,10 +92,22 @@ func NewManager(ctx context.Context, logger log.Logger, options ...func(*Manager
 		ctx:         ctx,
 		updatert:    5 * time.Second,
 		triggerSend: make(chan struct{}, 1),
+		registerer:  registerer,
+		sdMetrics:   sdMetrics,
 	}
 	for _, option := range options {
 		option(mgr)
 	}
+
+	// Register the metrics.
+	// We have to do this after setting all options, so that the name of the Manager is set.
+	if metrics, err := NewManagerMetrics(registerer, mgr.name); err == nil {
+		mgr.metrics = metrics
+	} else {
+		level.Error(logger).Log("msg", "Failed to create discovery manager metrics", "manager", mgr.name, "err", err)
+		return nil
+	}
+
 	return mgr
 }
 
@@ -170,6 +156,12 @@ type Manager struct {
 
 	// lastProvider counts providers registered during Manager's lifetime.
 	lastProvider uint
+
+	// A registerer for all service discovery metrics.
+	registerer prometheus.Registerer
+
+	metrics   *Metrics
+	sdMetrics map[string]DiscovererMetrics
 }
 
 // Providers returns the currently configured SD providers.
@@ -200,7 +192,7 @@ func (m *Manager) ApplyConfig(cfg map[string]Configs) error {
 	for name, scfg := range cfg {
 		failedCount += m.registerProviders(scfg, name)
 	}
-	failedConfigs.WithLabelValues(m.name).Set(float64(failedCount))
+	m.metrics.FailedConfigs.Set(float64(failedCount))
 
 	var (
 		wg sync.WaitGroup
@@ -230,13 +222,13 @@ func (m *Manager) ApplyConfig(cfg map[string]Configs) error {
 			// Remove obsolete subs' targets.
 			if _, ok := prov.newSubs[s]; !ok {
 				delete(m.targets, poolKey{s, prov.name})
-				discoveredTargets.DeleteLabelValues(m.name, s)
+				m.metrics.DiscoveredTargets.DeleteLabelValues(m.name, s)
 			}
 		}
 		// Set metrics and targets for new subs.
 		for s := range prov.newSubs {
 			if _, ok := prov.subs[s]; !ok {
-				discoveredTargets.WithLabelValues(m.name, s).Set(0)
+				m.metrics.DiscoveredTargets.WithLabelValues(s).Set(0)
 			}
 			if l := len(refTargets); l > 0 {
 				m.targets[poolKey{s, prov.name}] = make(map[string]*targetgroup.Group, l)
@@ -316,7 +308,7 @@ func (m *Manager) updater(ctx context.Context, p *Provider, updates chan []*targ
 		case <-ctx.Done():
 			return
 		case tgs, ok := <-updates:
-			receivedUpdates.WithLabelValues(m.name).Inc()
+			m.metrics.ReceivedUpdates.Inc()
 			if !ok {
 				level.Debug(m.logger).Log("msg", "Discoverer channel closed", "provider", p.name)
 				// Wait for provider cancellation to ensure targets are cleaned up when expected.
@@ -349,11 +341,11 @@ func (m *Manager) sender() {
 		case <-ticker.C: // Some discoverers send updates too often, so we throttle these with the ticker.
 			select {
 			case <-m.triggerSend:
-				sentUpdates.WithLabelValues(m.name).Inc()
+				m.metrics.SentUpdates.Inc()
 				select {
 				case m.syncCh <- m.allGroups():
 				default:
-					delayedUpdates.WithLabelValues(m.name).Inc()
+					m.metrics.DelayedUpdates.Inc()
 					level.Debug(m.logger).Log("msg", "Discovery receiver's channel was full so will retry the next cycle")
 					select {
 					case m.triggerSend <- struct{}{}:
@@ -405,7 +397,7 @@ func (m *Manager) allGroups() map[string][]*targetgroup.Group {
 		}
 	}
 	for setName, v := range n {
-		discoveredTargets.WithLabelValues(m.name, setName).Set(float64(v))
+		m.metrics.DiscoveredTargets.WithLabelValues(setName).Set(float64(v))
 	}
 	return tSets
 }
@@ -428,6 +420,7 @@ func (m *Manager) registerProviders(cfgs Configs, setName string) int {
 		d, err := cfg.NewDiscoverer(DiscovererOptions{
 			Logger:            log.With(m.logger, "discovery", typ, "config", setName),
 			HTTPClientOptions: m.httpOpts,
+			Metrics:           m.sdMetrics[typ],
 		})
 		if err != nil {
 			level.Error(m.logger).Log("msg", "Cannot create service discovery", "err", err, "type", typ, "config", setName)

discovery/manager_test.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	client_testutil "github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
@@ -35,6 +36,13 @@ func TestMain(m *testing.M) {
 	testutil.TolerantVerifyLeak(m)
 }
 
+func NewTestMetrics(t *testing.T, reg prometheus.Registerer) (*RefreshMetricsManager, map[string]DiscovererMetrics) {
+	refreshMetrics := NewRefreshMetrics(reg)
+	sdMetrics, err := RegisterSDMetrics(reg, refreshMetrics)
+	require.NoError(t, err)
+	return &refreshMetrics, sdMetrics
+}
+
 // TestTargetUpdatesOrder checks that the target updates are received in the expected order.
 func TestTargetUpdatesOrder(t *testing.T) {
 	// The order by which the updates are send is determined by the interval passed to the mock discovery adapter
@@ -664,7 +672,11 @@ func TestTargetUpdatesOrder(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel()
 
-			discoveryManager := NewManager(ctx, log.NewNopLogger())
+			reg := prometheus.NewRegistry()
+			_, sdMetrics := NewTestMetrics(t, reg)
+
+			discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
+			require.NotNil(t, discoveryManager)
 			discoveryManager.updatert = 100 * time.Millisecond
 
 			var totalUpdatesCount int
@@ -778,7 +790,12 @@ func pk(provider, setName string, n int) poolKey {
 func TestTargetSetTargetGroupsPresentOnConfigReload(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -810,7 +827,12 @@ func TestTargetSetTargetGroupsPresentOnConfigReload(t *testing.T) {
 func TestTargetSetTargetGroupsPresentOnConfigRename(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -845,7 +867,12 @@ func TestTargetSetTargetGroupsPresentOnConfigRename(t *testing.T) {
 func TestTargetSetTargetGroupsPresentOnConfigDuplicateAndDeleteOriginal(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -883,7 +910,12 @@ func TestTargetSetTargetGroupsPresentOnConfigDuplicateAndDeleteOriginal(t *testi
 func TestTargetSetTargetGroupsPresentOnConfigChange(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -944,7 +976,12 @@ func TestTargetSetTargetGroupsPresentOnConfigChange(t *testing.T) {
 func TestTargetSetRecreatesTargetGroupsOnConfigChange(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -983,7 +1020,12 @@ func TestTargetSetRecreatesTargetGroupsOnConfigChange(t *testing.T) {
 func TestDiscovererConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -1015,7 +1057,12 @@ func TestDiscovererConfigs(t *testing.T) {
 func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -1062,7 +1109,12 @@ func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 func TestIdenticalConfigurationsAreCoalesced(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, nil)
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, nil, reg, sdMetrics)
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -1098,7 +1150,12 @@ func TestApplyConfigDoesNotModifyStaticTargets(t *testing.T) {
 	}
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -1118,11 +1175,21 @@ type errorConfig struct{ err error }
 func (e errorConfig) Name() string                                        { return "error" }
 func (e errorConfig) NewDiscoverer(DiscovererOptions) (Discoverer, error) { return nil, e.err }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (errorConfig) NewDiscovererMetrics(prometheus.Registerer, RefreshMetricsInstantiator) DiscovererMetrics {
+	return &NoopDiscovererMetrics{}
+}
+
 type lockStaticConfig struct {
 	mu     *sync.Mutex
 	config StaticConfig
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (lockStaticConfig) NewDiscovererMetrics(prometheus.Registerer, RefreshMetricsInstantiator) DiscovererMetrics {
+	return &NoopDiscovererMetrics{}
+}
+
 func (s lockStaticConfig) Name() string { return "lockstatic" }
 func (s lockStaticConfig) NewDiscoverer(DiscovererOptions) (Discoverer, error) {
 	return (lockStaticDiscoverer)(s), nil
@@ -1144,7 +1211,12 @@ func (s lockStaticDiscoverer) Run(ctx context.Context, up chan<- []*targetgroup.
 func TestGaugeFailedConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -1158,7 +1230,7 @@ func TestGaugeFailedConfigs(t *testing.T) {
 	discoveryManager.ApplyConfig(c)
 	<-discoveryManager.SyncCh()
 
-	failedCount := client_testutil.ToFloat64(failedConfigs)
+	failedCount := client_testutil.ToFloat64(discoveryManager.metrics.FailedConfigs)
 	if failedCount != 3 {
 		t.Fatalf("Expected to have 3 failed configs, got: %v", failedCount)
 	}
@@ -1169,7 +1241,7 @@ func TestGaugeFailedConfigs(t *testing.T) {
 	discoveryManager.ApplyConfig(c)
 	<-discoveryManager.SyncCh()
 
-	failedCount = client_testutil.ToFloat64(failedConfigs)
+	failedCount = client_testutil.ToFloat64(discoveryManager.metrics.FailedConfigs)
 	if failedCount != 0 {
 		t.Fatalf("Expected to get no failed config, got: %v", failedCount)
 	}
@@ -1300,7 +1372,11 @@ func TestCoordinationWithReceiver(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel()
 
-			mgr := NewManager(ctx, nil)
+			reg := prometheus.NewRegistry()
+			_, sdMetrics := NewTestMetrics(t, reg)
+
+			mgr := NewManager(ctx, nil, reg, sdMetrics)
+			require.NotNil(t, mgr)
 			mgr.updatert = updateDelay
 			go mgr.Run()
 
@@ -1392,10 +1468,15 @@ func (o onceProvider) Run(_ context.Context, ch chan<- []*targetgroup.Group) {
 
 // TestTargetSetTargetGroupsUpdateDuringApplyConfig is used to detect races when
 // ApplyConfig happens at the same time as targets update.
-func TestTargetSetTargetGroupsUpdateDuringApplyConfig(*testing.T) {
+func TestTargetSetTargetGroupsUpdateDuringApplyConfig(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -1456,6 +1537,11 @@ func newTestDiscoverer() *testDiscoverer {
 	}
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*testDiscoverer) NewDiscovererMetrics(prometheus.Registerer, RefreshMetricsInstantiator) DiscovererMetrics {
+	return &NoopDiscovererMetrics{}
+}
+
 // Name implements Config.
 func (t *testDiscoverer) Name() string {
 	return "test"

discovery/marathon/marathon.go

@@ -28,6 +28,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -78,12 +79,19 @@ type SDConfig struct {
 	HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &marathonMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "marathon" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(*c, opts.Logger)
+	return NewDiscovery(*c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -132,7 +140,12 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Marathon Discovery.
-func NewDiscovery(conf SDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*marathonMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "marathon_sd")
 	if err != nil {
 		return nil, err
@@ -154,10 +167,13 @@ func NewDiscovery(conf SDConfig, logger log.Logger) (*Discovery, error) {
 		appsClient: fetchApps,
 	}
 	d.Discovery = refresh.NewDiscovery(
-		logger,
+		refresh.Options{
-		"marathon",
+			Logger:              logger,
-		time.Duration(conf.RefreshInterval),
+			Mech:                "marathon",
-		d.refresh,
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
+		},
 	)
 	return d, nil
 }

discovery/marathon/marathon_test.go

@@ -21,8 +21,11 @@ import (
 	"net/http/httptest"
 	"testing"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/require"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -36,7 +39,19 @@ func testConfig() SDConfig {
 }
 
 func testUpdateServices(client appListClient) ([]*targetgroup.Group, error) {
-	md, err := NewDiscovery(testConfig(), nil)
+	cfg := testConfig()
+
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	err := metrics.Register()
+	if err != nil {
+		return nil, err
+	}
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	md, err := NewDiscovery(cfg, nil, metrics)
 	if err != nil {
 		return nil, err
 	}
@@ -129,7 +144,15 @@ func TestMarathonSDSendGroup(t *testing.T) {
 }
 
 func TestMarathonSDRemoveApp(t *testing.T) {
-	md, err := NewDiscovery(testConfig(), nil)
+	cfg := testConfig()
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	md, err := NewDiscovery(cfg, nil, metrics)
 	if err != nil {
 		t.Fatalf("%s", err)
 	}

discovery/marathon/metrics.go

@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package marathon
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*marathonMetrics)(nil)
+
+type marathonMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *marathonMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *marathonMetrics) Unregister() {}

discovery/metrics.go

@@ -0,0 +1,101 @@
+// Copyright 2016 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package discovery
+
+import (
+	"fmt"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	clientGoRequestMetrics  = &clientGoRequestMetricAdapter{}
+	clientGoWorkloadMetrics = &clientGoWorkqueueMetricsProvider{}
+)
+
+func init() {
+	clientGoRequestMetrics.RegisterWithK8sGoClient()
+	clientGoWorkloadMetrics.RegisterWithK8sGoClient()
+}
+
+// Metrics to be used with a discovery manager.
+type Metrics struct {
+	FailedConfigs     prometheus.Gauge
+	DiscoveredTargets *prometheus.GaugeVec
+	ReceivedUpdates   prometheus.Counter
+	DelayedUpdates    prometheus.Counter
+	SentUpdates       prometheus.Counter
+}
+
+func NewManagerMetrics(registerer prometheus.Registerer, sdManagerName string) (*Metrics, error) {
+	m := &Metrics{}
+
+	m.FailedConfigs = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name:        "prometheus_sd_failed_configs",
+			Help:        "Current number of service discovery configurations that failed to load.",
+			ConstLabels: prometheus.Labels{"name": sdManagerName},
+		},
+	)
+
+	m.DiscoveredTargets = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name:        "prometheus_sd_discovered_targets",
+			Help:        "Current number of discovered targets.",
+			ConstLabels: prometheus.Labels{"name": sdManagerName},
+		},
+		[]string{"config"},
+	)
+
+	m.ReceivedUpdates = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name:        "prometheus_sd_received_updates_total",
+			Help:        "Total number of update events received from the SD providers.",
+			ConstLabels: prometheus.Labels{"name": sdManagerName},
+		},
+	)
+
+	m.DelayedUpdates = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name:        "prometheus_sd_updates_delayed_total",
+			Help:        "Total number of update events that couldn't be sent immediately.",
+			ConstLabels: prometheus.Labels{"name": sdManagerName},
+		},
+	)
+
+	m.SentUpdates = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name:        "prometheus_sd_updates_total",
+			Help:        "Total number of update events sent to the SD consumers.",
+			ConstLabels: prometheus.Labels{"name": sdManagerName},
+		},
+	)
+
+	metrics := []prometheus.Collector{
+		m.FailedConfigs,
+		m.DiscoveredTargets,
+		m.ReceivedUpdates,
+		m.DelayedUpdates,
+		m.SentUpdates,
+	}
+
+	for _, collector := range metrics {
+		err := registerer.Register(collector)
+		if err != nil {
+			return nil, fmt.Errorf("failed to register discovery manager metrics: %w", err)
+		}
+	}
+
+	return m, nil
+}

discovery/metrics_k8s_client.go

@@ -11,10 +11,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package kubernetes
+package discovery
 
 import (
 	"context"
+	"fmt"
 	"net/url"
 	"time"
 
@@ -23,13 +24,22 @@ import (
 	"k8s.io/client-go/util/workqueue"
 )
 
-const workqueueMetricsNamespace = metricsNamespace + "_workqueue"
+// This file registers metrics used by the Kubernetes Go client (k8s.io/client-go).
+// Unfortunately, k8s.io/client-go metrics are global.
+// If we instantiate multiple k8s SD instances, their k8s/client-go metrics will overlap.
+// To prevent us from displaying misleading metrics, we register k8s.io/client-go metrics
+// outside of the Kubernetes SD.
+
+const (
+	KubernetesMetricsNamespace = "prometheus_sd_kubernetes"
+	workqueueMetricsNamespace  = KubernetesMetricsNamespace + "_workqueue"
+)
 
 var (
 	// Metrics for client-go's HTTP requests.
 	clientGoRequestResultMetricVec = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
+			Namespace: KubernetesMetricsNamespace,
 			Name:      "http_request_total",
 			Help:      "Total number of HTTP requests to the Kubernetes API by status code.",
 		},
@@ -37,7 +47,7 @@ var (
 	)
 	clientGoRequestLatencyMetricVec = prometheus.NewSummaryVec(
 		prometheus.SummaryOpts{
-			Namespace:  metricsNamespace,
+			Namespace:  KubernetesMetricsNamespace,
 			Name:       "http_request_duration_seconds",
 			Help:       "Summary of latencies for HTTP requests to the Kubernetes API by endpoint.",
 			Objectives: map[float64]float64{},
@@ -109,17 +119,38 @@ func (noopMetric) Set(float64)     {}
 // Definition of client-go metrics adapters for HTTP requests observation.
 type clientGoRequestMetricAdapter struct{}
 
-func (f *clientGoRequestMetricAdapter) Register(registerer prometheus.Registerer) {
+// Returns all of the Prometheus metrics derived from k8s.io/client-go.
+// This may be used tu register and unregister the metrics.
+func clientGoMetrics() []prometheus.Collector {
+	return []prometheus.Collector{
+		clientGoRequestResultMetricVec,
+		clientGoRequestLatencyMetricVec,
+		clientGoWorkqueueDepthMetricVec,
+		clientGoWorkqueueAddsMetricVec,
+		clientGoWorkqueueLatencyMetricVec,
+		clientGoWorkqueueUnfinishedWorkSecondsMetricVec,
+		clientGoWorkqueueLongestRunningProcessorMetricVec,
+		clientGoWorkqueueWorkDurationMetricVec,
+	}
+}
+
+func RegisterK8sClientMetricsWithPrometheus(registerer prometheus.Registerer) error {
+	for _, collector := range clientGoMetrics() {
+		err := registerer.Register(collector)
+		if err != nil {
+			return fmt.Errorf("failed to register Kubernetes Go Client metrics: %w", err)
+		}
+	}
+	return nil
+}
+
+func (f *clientGoRequestMetricAdapter) RegisterWithK8sGoClient() {
 	metrics.Register(
 		metrics.RegisterOpts{
 			RequestLatency: f,
 			RequestResult:  f,
 		},
 	)
-	registerer.MustRegister(
-		clientGoRequestResultMetricVec,
-		clientGoRequestLatencyMetricVec,
-	)
 }
 
 func (clientGoRequestMetricAdapter) Increment(_ context.Context, code, _, _ string) {
@@ -133,16 +164,8 @@ func (clientGoRequestMetricAdapter) Observe(_ context.Context, _ string, u url.U
 // Definition of client-go workqueue metrics provider definition.
 type clientGoWorkqueueMetricsProvider struct{}
 
-func (f *clientGoWorkqueueMetricsProvider) Register(registerer prometheus.Registerer) {
+func (f *clientGoWorkqueueMetricsProvider) RegisterWithK8sGoClient() {
 	workqueue.SetProvider(f)
-	registerer.MustRegister(
-		clientGoWorkqueueDepthMetricVec,
-		clientGoWorkqueueAddsMetricVec,
-		clientGoWorkqueueLatencyMetricVec,
-		clientGoWorkqueueWorkDurationMetricVec,
-		clientGoWorkqueueUnfinishedWorkSecondsMetricVec,
-		clientGoWorkqueueLongestRunningProcessorMetricVec,
-	)
 }
 
 func (f *clientGoWorkqueueMetricsProvider) NewDepthMetric(name string) workqueue.GaugeMetric {

discovery/metrics_refresh.go

@@ -0,0 +1,75 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package discovery
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Metric vectors for the "refresh" package.
+// We define them here in the "discovery" package in order to avoid a cyclic dependency between
+// "discovery" and "refresh".
+type RefreshMetricsVecs struct {
+	failuresVec *prometheus.CounterVec
+	durationVec *prometheus.SummaryVec
+
+	metricRegisterer MetricRegisterer
+}
+
+var _ RefreshMetricsManager = (*RefreshMetricsVecs)(nil)
+
+func NewRefreshMetrics(reg prometheus.Registerer) RefreshMetricsManager {
+	m := &RefreshMetricsVecs{
+		failuresVec: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_refresh_failures_total",
+				Help: "Number of refresh failures for the given SD mechanism.",
+			},
+			[]string{"mechanism"}),
+		durationVec: prometheus.NewSummaryVec(
+			prometheus.SummaryOpts{
+				Name:       "prometheus_sd_refresh_duration_seconds",
+				Help:       "The duration of a refresh in seconds for the given SD mechanism.",
+				Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+			},
+			[]string{"mechanism"}),
+	}
+
+	// The reason we register metric vectors instead of metrics is so that
+	// the metrics are not visible until they are recorded.
+	m.metricRegisterer = NewMetricRegisterer(reg, []prometheus.Collector{
+		m.failuresVec,
+		m.durationVec,
+	})
+
+	return m
+}
+
+// Instantiate returns metrics out of metric vectors.
+func (m *RefreshMetricsVecs) Instantiate(mech string) *RefreshMetrics {
+	return &RefreshMetrics{
+		Failures: m.failuresVec.WithLabelValues(mech),
+		Duration: m.durationVec.WithLabelValues(mech),
+	}
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *RefreshMetricsVecs) Register() error {
+	return m.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *RefreshMetricsVecs) Unregister() {
+	m.metricRegisterer.UnregisterMetrics()
+}

discovery/moby/docker.go

@@ -22,10 +22,11 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -75,12 +76,19 @@ type DockerSDConfig struct {
 	RefreshInterval model.Duration `yaml:"refresh_interval"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*DockerSDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &dockerMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Config.
 func (*DockerSDConfig) Name() string { return "docker" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *DockerSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDockerDiscovery(c, opts.Logger)
+	return NewDockerDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -114,8 +122,11 @@ type DockerDiscovery struct {
 }
 
 // NewDockerDiscovery returns a new DockerDiscovery which periodically refreshes its targets.
-func NewDockerDiscovery(conf *DockerSDConfig, logger log.Logger) (*DockerDiscovery, error) {
+func NewDockerDiscovery(conf *DockerSDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*DockerDiscovery, error) {
-	var err error
+	m, ok := metrics.(*dockerMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
 
 	d := &DockerDiscovery{
 		port:               conf.Port,
@@ -165,10 +176,13 @@ func NewDockerDiscovery(conf *DockerSDConfig, logger log.Logger) (*DockerDiscove
 	}
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
+		refresh.Options{
-		"docker",
+			Logger:              logger,
-		time.Duration(conf.RefreshInterval),
+			Mech:                "docker",
-		d.refresh,
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
+		},
 	)
 	return d, nil
 }
@@ -178,7 +192,7 @@ func (d *DockerDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, er
 		Source: "Docker",
 	}
 
-	containers, err := d.client.ContainerList(ctx, types.ContainerListOptions{Filters: d.filters})
+	containers, err := d.client.ContainerList(ctx, container.ListOptions{Filters: d.filters})
 	if err != nil {
 		return nil, fmt.Errorf("error while listing containers: %w", err)
 	}

discovery/moby/docker_test.go

@@ -19,9 +19,12 @@ import (
 	"testing"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
+
+	"github.com/prometheus/prometheus/discovery"
 )
 
 func TestDockerSDRefresh(t *testing.T) {
@@ -37,7 +40,14 @@ host: %s
 	var cfg DockerSDConfig
 	require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg))
 
-	d, err := NewDockerDiscovery(&cfg, log.NewNopLogger())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	d, err := NewDockerDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()

discovery/moby/dockerswarm.go

@@ -23,6 +23,7 @@ import (
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/version"
@@ -69,12 +70,19 @@ type Filter struct {
 	Values []string `yaml:"values"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*DockerSwarmSDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &dockerswarmMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Config.
 func (*DockerSwarmSDConfig) Name() string { return "dockerswarm" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *DockerSwarmSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -117,8 +125,11 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *DockerSwarmSDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf *DockerSwarmSDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
-	var err error
+	m, ok := metrics.(*dockerswarmMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
 
 	d := &Discovery{
 		port: conf.Port,
@@ -168,10 +179,13 @@ func NewDiscovery(conf *DockerSwarmSDConfig, logger log.Logger) (*Discovery, err
 	}
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
+		refresh.Options{
-		"dockerswarm",
+			Logger:              logger,
-		time.Duration(conf.RefreshInterval),
+			Mech:                "dockerswarm",
-		d.refresh,
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
+		},
 	)
 	return d, nil
 }

discovery/moby/metrics_docker.go

@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package moby
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*dockerMetrics)(nil)
+
+type dockerMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *dockerMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *dockerMetrics) Unregister() {}

discovery/moby/metrics_dockerswarm.go

@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package moby
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*dockerswarmMetrics)(nil)
+
+type dockerswarmMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *dockerswarmMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *dockerswarmMetrics) Unregister() {}

discovery/moby/nodes_test.go

@@ -19,9 +19,12 @@ import (
 	"testing"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
+
+	"github.com/prometheus/prometheus/discovery"
 )
 
 func TestDockerSwarmNodesSDRefresh(t *testing.T) {
@@ -38,7 +41,14 @@ host: %s
 	var cfg DockerSwarmSDConfig
 	require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg))
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()

discovery/moby/services_test.go

@@ -19,9 +19,12 @@ import (
 	"testing"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
+
+	"github.com/prometheus/prometheus/discovery"
 )
 
 func TestDockerSwarmSDServicesRefresh(t *testing.T) {
@@ -38,7 +41,14 @@ host: %s
 	var cfg DockerSwarmSDConfig
 	require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg))
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()
@@ -332,7 +342,14 @@ filters:
 	var cfg DockerSwarmSDConfig
 	require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg))
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()

discovery/moby/tasks_test.go

@@ -19,9 +19,12 @@ import (
 	"testing"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
+
+	"github.com/prometheus/prometheus/discovery"
 )
 
 func TestDockerSwarmTasksSDRefresh(t *testing.T) {
@@ -38,7 +41,14 @@ host: %s
 	var cfg DockerSwarmSDConfig
 	require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg))
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()

discovery/nomad/metrics.go

@@ -0,0 +1,57 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nomad
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*nomadMetrics)(nil)
+
+type nomadMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+
+	failuresCount prometheus.Counter
+
+	metricRegisterer discovery.MetricRegisterer
+}
+
+func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	m := &nomadMetrics{
+		refreshMetrics: rmi,
+		failuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_nomad_failures_total",
+				Help: "Number of nomad service discovery refresh failures.",
+			}),
+	}
+
+	m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
+		m.failuresCount,
+	})
+
+	return m
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *nomadMetrics) Register() error {
+	return m.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *nomadMetrics) Unregister() {
+	m.metricRegisterer.UnregisterMetrics()
+}

discovery/nomad/nomad.go

@@ -49,27 +49,18 @@ const (
 )
 
 // DefaultSDConfig is the default nomad SD configuration.
-var (
+var DefaultSDConfig = SDConfig{
-	DefaultSDConfig = SDConfig{
+	AllowStale:       true,
-		AllowStale:       true,
+	HTTPClientConfig: config.DefaultHTTPClientConfig,
-		HTTPClientConfig: config.DefaultHTTPClientConfig,
+	Namespace:        "default",
-		Namespace:        "default",
+	RefreshInterval:  model.Duration(60 * time.Second),
-		RefreshInterval:  model.Duration(60 * time.Second),
+	Region:           "global",
-		Region:           "global",
+	Server:           "http://localhost:4646",
-		Server:           "http://localhost:4646",
+	TagSeparator:     ",",
-		TagSeparator:     ",",
+}
-	}
-
-	failuresCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_nomad_failures_total",
-			Help: "Number of nomad service discovery refresh failures.",
-		})
-)
 
 func init() {
 	discovery.RegisterConfig(&SDConfig{})
-	prometheus.MustRegister(failuresCount)
 }
 
 // SDConfig is the configuration for nomad based service discovery.
@@ -83,12 +74,17 @@ type SDConfig struct {
 	TagSeparator     string                  `yaml:"tag_separator,omitempty"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return newDiscovererMetrics(reg, rmi)
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "nomad" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -121,10 +117,16 @@ type Discovery struct {
 	region          string
 	server          string
 	tagSeparator    string
+	metrics         *nomadMetrics
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*nomadMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	d := &Discovery{
 		allowStale:      conf.AllowStale,
 		namespace:       conf.Namespace,
@@ -132,6 +134,7 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 		region:          conf.Region,
 		server:          conf.Server,
 		tagSeparator:    conf.TagSeparator,
+		metrics:         m,
 	}
 
 	HTTPClient, err := config.NewClientFromConfig(conf.HTTPClientConfig, "nomad_sd")
@@ -153,10 +156,13 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 	d.client = client
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
+		refresh.Options{
-		"nomad",
+			Logger:              logger,
-		time.Duration(conf.RefreshInterval),
+			Mech:                "nomad",
-		d.refresh,
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
+		},
 	)
 	return d, nil
 }
@@ -167,7 +173,7 @@ func (d *Discovery) refresh(context.Context) ([]*targetgroup.Group, error) {
 	}
 	stubs, _, err := d.client.Services().List(opts)
 	if err != nil {
-		failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, err
 	}
 
@@ -179,7 +185,7 @@ func (d *Discovery) refresh(context.Context) ([]*targetgroup.Group, error) {
 		for _, service := range stub.Services {
 			instances, _, err := d.client.Services().Get(service.ServiceName, opts)
 			if err != nil {
-				failuresCount.Inc()
+				d.metrics.failuresCount.Inc()
 				return nil, fmt.Errorf("failed to fetch services: %w", err)
 			}
 

discovery/nomad/nomad_test.go

@@ -22,8 +22,11 @@ import (
 	"testing"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/prometheus/discovery"
 )
 
 type NomadSDTestSuite struct {
@@ -127,8 +130,16 @@ func TestConfiguredService(t *testing.T) {
 	conf := &SDConfig{
 		Server: "http://localhost:4646",
 	}
-	_, err := NewDiscovery(conf, nil)
+
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := conf.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+
+	_, err := NewDiscovery(conf, nil, metrics)
 	require.NoError(t, err)
+
+	metrics.Unregister()
 }
 
 func TestNomadSDRefresh(t *testing.T) {
@@ -141,7 +152,15 @@ func TestNomadSDRefresh(t *testing.T) {
 
 	cfg := DefaultSDConfig
 	cfg.Server = endpoint.String()
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 
 	tgs, err := d.refresh(context.Background())

discovery/openstack/metrics.go

@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package openstack
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+type openstackMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+var _ discovery.DiscovererMetrics = (*openstackMetrics)(nil)
+
+// Register implements discovery.DiscovererMetrics.
+func (m *openstackMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *openstackMetrics) Unregister() {}