Merge remote-tracking branch 'origin/main' into owilliams/utf8-02

.github/CODEOWNERS

@@ -1,6 +1,6 @@
 /web/ui @juliusv
 /web/ui/module @juliusv @nexucis
-/storage/remote @csmarchbanks @cstyan @bwplotka @tomwilkie
+/storage/remote @cstyan @bwplotka @tomwilkie
 /storage/remote/otlptranslator @gouthamve @jesusvazquez
 /discovery/kubernetes @brancz
 /tsdb @jesusvazquez

.github/workflows/buf.yml

@@ -23,7 +23,7 @@ jobs:
         with:
           input: 'prompb'
           against: 'https://github.com/prometheus/prometheus.git#branch=main,ref=HEAD~1,subdir=prompb'
-      - uses: bufbuild/buf-push-action@342fc4cdcf29115a01cf12a2c6dd6aac68dc51e1 # v1.1.1
+      - uses: bufbuild/buf-push-action@a654ff18effe4641ebea4a4ce242c49800728459 # v1.1.1
         with:
           input: 'prompb'
           buf_token: ${{ secrets.BUF_TOKEN }}

.github/workflows/fuzzing.yml

@@ -21,7 +21,7 @@ jobs:
           fuzz-seconds: 600
           dry-run: false
       - name: Upload Crash
-        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
+        uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
         if: failure() && steps.build.outcome == 'success'
         with:
           name: artifacts

.github/workflows/scorecards.yml

@@ -37,7 +37,7 @@ jobs:
       # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
       # format to the repository Actions tab.
       - name: "Upload artifact"
-        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # tag=v3.1.3
+        uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # tag=v4.0.0
         with:
           name: SARIF file
           path: results.sarif

CHANGELOG.md

@@ -1,9 +1,8 @@
 # Changelog
 
-## unreleased
+## 2.49.1 / 2024-01-15
 
-* [ENHANCEMENT] TSDB: Make the wlog watcher read segments synchronously when not tailing. #13224
+* [BUGFIX] TSDB: Fixed a wrong `q=` value in scrape accept header #13313
-* [BUGFIX] Agent: Participate in notify calls. #13223
 
 ## 2.49.0 / 2024-01-15
 

MAINTAINERS.md

@@ -8,8 +8,10 @@ Julien Pivotto (<roidelapluie@prometheus.io> / @roidelapluie) and Levi Harrison
   * `k8s`: Frederic Branczyk (<fbranczyk@gmail.com> / @brancz)
 * `documentation`
   * `prometheus-mixin`: Matthias Loibl (<mail@matthiasloibl.com> / @metalmatze)
+* `model/histogram` and other code related to native histograms: Björn Rabenstein (<beorn@grafana.com> / @beorn7), 
+George Krajcsovits (<gyorgy.krajcsovits@grafana.com> / @krajorama)
 * `storage`
-  * `remote`: Chris Marchbanks (<csmarchbanks@gmail.com> / @csmarchbanks), Callum Styan (<callumstyan@gmail.com> / @cstyan), Bartłomiej Płotka (<bwplotka@gmail.com> / @bwplotka), Tom Wilkie (<tom.wilkie@gmail.com> / @tomwilkie)
+  * `remote`: Callum Styan (<callumstyan@gmail.com> / @cstyan), Bartłomiej Płotka (<bwplotka@gmail.com> / @bwplotka), Tom Wilkie (<tom.wilkie@gmail.com> / @tomwilkie)
 * `tsdb`: Ganesh Vernekar (<ganesh@grafana.com> / @codesome), Bartłomiej Płotka (<bwplotka@gmail.com> / @bwplotka), Jesús Vázquez (<jesus.vazquez@grafana.com> / @jesusvazquez)
   * `agent`: Robert Fratto (<robert.fratto@grafana.com> / @rfratto)
 * `web`
@@ -17,6 +19,7 @@ Julien Pivotto (<roidelapluie@prometheus.io> / @roidelapluie) and Levi Harrison
     * `module`: Augustin Husson (<husson.augustin@gmail.com> @nexucis)
 * `Makefile` and related build configuration: Simon Pasquier (<pasquier.simon@gmail.com> / @simonpasquier), Ben Kochie (<superq@gmail.com> / @SuperQ)
 
+
 For the sake of brevity, not all subtrees are explicitly listed. Due to the
 size of this repository, the natural changes in focus of maintainers over time,
 and nuances of where particular features live, this list will always be

Makefile.common

@@ -62,10 +62,10 @@ SKIP_GOLANGCI_LINT :=
 GOLANGCI_LINT :=
 GOLANGCI_LINT_OPTS ?=
 GOLANGCI_LINT_VERSION ?= v1.55.2
-# golangci-lint only supports linux, darwin and windows platforms on i386/amd64.
+# golangci-lint only supports linux, darwin and windows platforms on i386/amd64/arm64.
 # windows isn't included here because of the path separator being different.
 ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin))
-	ifeq ($(GOHOSTARCH),$(filter $(GOHOSTARCH),amd64 i386))
+	ifeq ($(GOHOSTARCH),$(filter $(GOHOSTARCH),amd64 i386 arm64))
 		# If we're in CI and there is an Actions file, that means the linter
 		# is being run in Actions, so we don't need to run it here.
 		ifneq (,$(SKIP_GOLANGCI_LINT))

README.md

@@ -14,7 +14,7 @@ examples and guides.</p>
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/486/badge)](https://bestpractices.coreinfrastructure.org/projects/486)
 [![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/prometheus/prometheus)
 [![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/prometheus.svg)](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:prometheus)
-[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/prometheus/prometheus/badge)](https://api.securityscorecards.dev/projects/github.com/prometheus/prometheus)
+[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/prometheus/prometheus/badge)](https://securityscorecards.dev/viewer/?uri=github.com/prometheus/prometheus)
 
 </div>
 

VERSION

@@ -1 +1 @@
-2.49.0
+2.49.1

cmd/prometheus/main.go

@@ -33,6 +33,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/KimMachineGun/automemlimit/memlimit"
 	"github.com/alecthomas/kingpin/v2"
 	"github.com/alecthomas/units"
 	"github.com/go-kit/log"
@@ -136,6 +137,7 @@ type flagConfig struct {
 	forGracePeriod      model.Duration
 	outageTolerance     model.Duration
 	resendDelay         model.Duration
+	maxConcurrentEvals  int64
 	web                 web.Options
 	scrape              scrape.Options
 	tsdb                tsdbOptions
@@ -148,13 +150,16 @@ type flagConfig struct {
 	RemoteFlushDeadline model.Duration
 	nameEscapingScheme string
 
-	featureList []string
+	featureList   []string
+	memlimitRatio float64
 	// These options are extracted from featureList
 	// for ease of use.
 	enableExpandExternalLabels bool
 	enableNewSDManager         bool
 	enablePerStepStats         bool
 	enableAutoGOMAXPROCS       bool
+	enableAutoGOMEMLIMIT       bool
+	enableConcurrentRuleEval   bool
 
 	prometheusURL   string
 	corsRegexString string
@@ -198,6 +203,12 @@ func (c *flagConfig) setFeatureListOptions(logger log.Logger) error {
 			case "auto-gomaxprocs":
 				c.enableAutoGOMAXPROCS = true
 				level.Info(logger).Log("msg", "Automatically set GOMAXPROCS to match Linux container CPU quota")
+			case "auto-gomemlimit":
+				c.enableAutoGOMEMLIMIT = true
+				level.Info(logger).Log("msg", "Automatically set GOMEMLIMIT to match Linux container or system memory limit")
+			case "concurrent-rule-eval":
+				c.enableConcurrentRuleEval = true
+				level.Info(logger).Log("msg", "Experimental concurrent rule evaluation enabled.")
 			case "no-default-scrape-port":
 				c.scrape.NoDefaultPort = true
 				level.Info(logger).Log("msg", "No default port will be appended to scrape targets' addresses.")
@@ -266,6 +277,9 @@ func main() {
 	a.Flag("web.listen-address", "Address to listen on for UI, API, and telemetry.").
 		Default("0.0.0.0:9090").StringVar(&cfg.web.ListenAddress)
 
+	a.Flag("auto-gomemlimit.ratio", "The ratio of reserved GOMEMLIMIT memory to the detected maximum container or system memory").
+		Default("0.9").FloatVar(&cfg.memlimitRatio)
+
 	webConfig := a.Flag(
 		"web.config.file",
 		"[EXPERIMENTAL] Path to configuration file that can enable TLS or authentication.",
@@ -406,6 +420,9 @@ func main() {
 	serverOnlyFlag(a, "rules.alert.resend-delay", "Minimum amount of time to wait before resending an alert to Alertmanager.").
 		Default("1m").SetValue(&cfg.resendDelay)
 
+	serverOnlyFlag(a, "rules.max-concurrent-evals", "Global concurrency limit for independent rules that can run concurrently.").
+		Default("4").Int64Var(&cfg.maxConcurrentEvals)
+
 	a.Flag("scrape.adjust-timestamps", "Adjust scrape timestamps by up to `scrape.timestamp-tolerance` to align them to the intended schedule. See https://github.com/prometheus/prometheus/issues/7846 for more context. Experimental. This flag will be removed in a future release.").
 		Hidden().Default("true").BoolVar(&scrape.AlignScrapeTimestamps)
 
@@ -435,7 +452,7 @@ func main() {
 
 	a.Flag("scrape.name-escaping-scheme", "method to escape legacy invalid names when sending to an old version of prometheus.  can be one of values (default), underscores, or dots").Default(scrape.DefaultNameEscapingScheme.String()).StringVar(&cfg.nameEscapingScheme)
 
-	a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: agent, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-at-modifier, promql-negative-offset, promql-per-step-stats, promql-experimental-functions, remote-write-receiver (DEPRECATED), extra-scrape-metrics, new-service-discovery-manager, auto-gomaxprocs, no-default-scrape-port, native-histograms, otlp-write-receiver, utf8-names. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details.").
+	a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: agent, auto-gomemlimit, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-at-modifier, promql-negative-offset, promql-per-step-stats, promql-experimental-functions, remote-write-receiver (DEPRECATED), extra-scrape-metrics, new-service-discovery-manager, auto-gomaxprocs, no-default-scrape-port, native-histograms, otlp-write-receiver, utf8-names. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details.").
 		Default("").StringsVar(&cfg.featureList)
 
 	promlogflag.AddFlags(a, &cfg.promlogConfig)
@@ -482,6 +499,11 @@ func main() {
 		os.Exit(3)
 	}
 
+	if cfg.memlimitRatio <= 0.0 || cfg.memlimitRatio > 1.0 {
+		fmt.Fprintf(os.Stderr, "--auto-gomemlimit.ratio must be greater than 0 and less than or equal to 1.")
+		os.Exit(1)
+	}
+
 	localStoragePath := cfg.serverStoragePath
 	if agentMode {
 		localStoragePath = cfg.agentStoragePath
@@ -646,9 +668,16 @@ func main() {
 		level.Error(logger).Log("msg", "failed to register Kubernetes client metrics", "err", err)
 		os.Exit(1)
 	}
+
+	sdMetrics, err := discovery.CreateAndRegisterSDMetrics(prometheus.DefaultRegisterer)
+	if err != nil {
+		level.Error(logger).Log("msg", "failed to register service discovery metrics", "err", err)
+		os.Exit(1)
+	}
+
 	if cfg.enableNewSDManager {
 		{
-			discMgr := discovery.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, discovery.Name("scrape"))
+			discMgr := discovery.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, sdMetrics, discovery.Name("scrape"))
 			if discMgr == nil {
 				level.Error(logger).Log("msg", "failed to create a discovery manager scrape")
 				os.Exit(1)
@@ -657,7 +686,7 @@ func main() {
 		}
 
 		{
-			discMgr := discovery.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, discovery.Name("notify"))
+			discMgr := discovery.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, sdMetrics, discovery.Name("notify"))
 			if discMgr == nil {
 				level.Error(logger).Log("msg", "failed to create a discovery manager notify")
 				os.Exit(1)
@@ -666,7 +695,7 @@ func main() {
 		}
 	} else {
 		{
-			discMgr := legacymanager.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, legacymanager.Name("scrape"))
+			discMgr := legacymanager.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, sdMetrics, legacymanager.Name("scrape"))
 			if discMgr == nil {
 				level.Error(logger).Log("msg", "failed to create a discovery manager scrape")
 				os.Exit(1)
@@ -675,7 +704,7 @@ func main() {
 		}
 
 		{
-			discMgr := legacymanager.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, legacymanager.Name("notify"))
+			discMgr := legacymanager.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, sdMetrics, legacymanager.Name("notify"))
 			if discMgr == nil {
 				level.Error(logger).Log("msg", "failed to create a discovery manager notify")
 				os.Exit(1)
@@ -711,6 +740,20 @@ func main() {
 		}
 	}
 
+	if cfg.enableAutoGOMEMLIMIT {
+		if _, err := memlimit.SetGoMemLimitWithOpts(
+			memlimit.WithRatio(cfg.memlimitRatio),
+			memlimit.WithProvider(
+				memlimit.ApplyFallback(
+					memlimit.FromCgroup,
+					memlimit.FromSystem,
+				),
+			),
+		); err != nil {
+			level.Warn(logger).Log("component", "automemlimit", "msg", "Failed to set GOMEMLIMIT automatically", "err", err)
+		}
+	}
+
 	if !agentMode {
 		opts := promql.EngineOpts{
 			Logger:                   log.With(logger, "component", "query engine"),
@@ -730,17 +773,19 @@ func main() {
 		queryEngine = promql.NewEngine(opts)
 
 		ruleManager = rules.NewManager(&rules.ManagerOptions{
-			Appendable:      fanoutStorage,
+			Appendable:             fanoutStorage,
-			Queryable:       localStorage,
+			Queryable:              localStorage,
-			QueryFunc:       rules.EngineQueryFunc(queryEngine, fanoutStorage),
+			QueryFunc:              rules.EngineQueryFunc(queryEngine, fanoutStorage),
-			NotifyFunc:      rules.SendAlerts(notifierManager, cfg.web.ExternalURL.String()),
+			NotifyFunc:             rules.SendAlerts(notifierManager, cfg.web.ExternalURL.String()),
-			Context:         ctxRule,
+			Context:                ctxRule,
-			ExternalURL:     cfg.web.ExternalURL,
+			ExternalURL:            cfg.web.ExternalURL,
-			Registerer:      prometheus.DefaultRegisterer,
+			Registerer:             prometheus.DefaultRegisterer,
-			Logger:          log.With(logger, "component", "rule manager"),
+			Logger:                 log.With(logger, "component", "rule manager"),
-			OutageTolerance: time.Duration(cfg.outageTolerance),
+			OutageTolerance:        time.Duration(cfg.outageTolerance),
-			ForGracePeriod:  time.Duration(cfg.forGracePeriod),
+			ForGracePeriod:         time.Duration(cfg.forGracePeriod),
-			ResendDelay:     time.Duration(cfg.resendDelay),
+			ResendDelay:            time.Duration(cfg.resendDelay),
+			MaxConcurrentEvals:     cfg.maxConcurrentEvals,
+			ConcurrentEvalsEnabled: cfg.enableConcurrentRuleEval,
 		})
 	}
 

cmd/prometheus/main_test.go

@@ -126,12 +126,9 @@ func TestFailedStartupExitCode(t *testing.T) {
 	require.Error(t, err)
 
 	var exitError *exec.ExitError
-	if errors.As(err, &exitError) {
+	require.ErrorAs(t, err, &exitError)
-		status := exitError.Sys().(syscall.WaitStatus)
+	status := exitError.Sys().(syscall.WaitStatus)
-		require.Equal(t, expectedExitStatus, status.ExitStatus())
+	require.Equal(t, expectedExitStatus, status.ExitStatus())
-	} else {
-		t.Errorf("unable to retrieve the exit status for prometheus: %v", err)
-	}
 }
 
 type senderFunc func(alerts ...*notifier.Alert)
@@ -194,9 +191,7 @@ func TestSendAlerts(t *testing.T) {
 		tc := tc
 		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
 			senderFunc := senderFunc(func(alerts ...*notifier.Alert) {
-				if len(tc.in) == 0 {
+				require.NotEmpty(t, tc.in, "sender called with 0 alert")
-					t.Fatalf("sender called with 0 alert")
-				}
 				require.Equal(t, tc.exp, alerts)
 			})
 			rules.SendAlerts(senderFunc, "http://localhost:9090")(context.TODO(), "up", tc.in...)
@@ -228,7 +223,7 @@ func TestWALSegmentSizeBounds(t *testing.T) {
 			go func() { done <- prom.Wait() }()
 			select {
 			case err := <-done:
-				t.Errorf("prometheus should be still running: %v", err)
+				require.Fail(t, "prometheus should be still running: %v", err)
 			case <-time.After(startupTime):
 				prom.Process.Kill()
 				<-done
@@ -239,12 +234,9 @@ func TestWALSegmentSizeBounds(t *testing.T) {
 		err = prom.Wait()
 		require.Error(t, err)
 		var exitError *exec.ExitError
-		if errors.As(err, &exitError) {
+		require.ErrorAs(t, err, &exitError)
-			status := exitError.Sys().(syscall.WaitStatus)
+		status := exitError.Sys().(syscall.WaitStatus)
-			require.Equal(t, expectedExitStatus, status.ExitStatus())
+		require.Equal(t, expectedExitStatus, status.ExitStatus())
-		} else {
-			t.Errorf("unable to retrieve the exit status for prometheus: %v", err)
-		}
 	}
 }
 
@@ -274,7 +266,7 @@ func TestMaxBlockChunkSegmentSizeBounds(t *testing.T) {
 			go func() { done <- prom.Wait() }()
 			select {
 			case err := <-done:
-				t.Errorf("prometheus should be still running: %v", err)
+				require.Fail(t, "prometheus should be still running: %v", err)
 			case <-time.After(startupTime):
 				prom.Process.Kill()
 				<-done
@@ -285,12 +277,9 @@ func TestMaxBlockChunkSegmentSizeBounds(t *testing.T) {
 		err = prom.Wait()
 		require.Error(t, err)
 		var exitError *exec.ExitError
-		if errors.As(err, &exitError) {
+		require.ErrorAs(t, err, &exitError)
-			status := exitError.Sys().(syscall.WaitStatus)
+		status := exitError.Sys().(syscall.WaitStatus)
-			require.Equal(t, expectedExitStatus, status.ExitStatus())
+		require.Equal(t, expectedExitStatus, status.ExitStatus())
-		} else {
-			t.Errorf("unable to retrieve the exit status for prometheus: %v", err)
-		}
 	}
 }
 
@@ -347,10 +336,8 @@ func getCurrentGaugeValuesFor(t *testing.T, reg prometheus.Gatherer, metricNames
 			}
 
 			require.Len(t, g.GetMetric(), 1)
-			if _, ok := res[m]; ok {
+			_, ok := res[m]
-				t.Error("expected only one metric family for", m)
+			require.False(t, ok, "expected only one metric family for", m)
-				t.FailNow()
-			}
 			res[m] = *g.GetMetric()[0].GetGauge().Value
 		}
 	}

cmd/prometheus/main_unix_test.go

@@ -23,6 +23,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/prometheus/prometheus/util/testutil"
 )
 
@@ -37,9 +39,7 @@ func TestStartupInterrupt(t *testing.T) {
 
 	prom := exec.Command(promPath, "-test.main", "--config.file="+promConfig, "--storage.tsdb.path="+t.TempDir(), "--web.listen-address=0.0.0.0"+port)
 	err := prom.Start()
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("execution error: %v", err)
-	}
 
 	done := make(chan error, 1)
 	go func() {
@@ -68,14 +68,11 @@ Loop:
 		time.Sleep(500 * time.Millisecond)
 	}
 
-	if !startedOk {
+	require.True(t, startedOk, "prometheus didn't start in the specified timeout")
-		t.Fatal("prometheus didn't start in the specified timeout")
+	err = prom.Process.Kill()
-	}
+	require.Error(t, err, "prometheus didn't shutdown gracefully after sending the Interrupt signal")
-	switch err := prom.Process.Kill(); {
+	// TODO - find a better way to detect when the process didn't exit as expected!
-	case err == nil:
+	if stoppedErr != nil {
-		t.Errorf("prometheus didn't shutdown gracefully after sending the Interrupt signal")
+		require.EqualError(t, stoppedErr, "signal: interrupt", "prometheus exit")
-	case stoppedErr != nil && stoppedErr.Error() != "signal: interrupt":
-		// TODO: find a better way to detect when the process didn't exit as expected!
-		t.Errorf("prometheus exited with an unexpected error: %v", stoppedErr)
 	}
 }

cmd/promtool/sd.go

@@ -78,12 +78,25 @@ func CheckSD(sdConfigFiles, sdJobName string, sdTimeout time.Duration, noDefault
 	defer cancel()
 
 	for _, cfg := range scrapeConfig.ServiceDiscoveryConfigs {
-		d, err := cfg.NewDiscoverer(discovery.DiscovererOptions{Logger: logger, Registerer: registerer})
+		reg := prometheus.NewRegistry()
+		refreshMetrics := discovery.NewRefreshMetrics(reg)
+		metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+		err := metrics.Register()
+		if err != nil {
+			fmt.Fprintln(os.Stderr, "Could not register service discovery metrics", err)
+			return failureExitCode
+		}
+
+		d, err := cfg.NewDiscoverer(discovery.DiscovererOptions{Logger: logger, Metrics: metrics})
 		if err != nil {
 			fmt.Fprintln(os.Stderr, "Could not create new discoverer", err)
 			return failureExitCode
 		}
-		go d.Run(ctx, targetGroupChan)
+		go func() {
+			d.Run(ctx, targetGroupChan)
+			metrics.Unregister()
+			refreshMetrics.Unregister()
+		}()
 	}
 
 	var targetGroups []*targetgroup.Group

cmd/promtool/tsdb.go

@@ -667,7 +667,7 @@ func analyzeCompaction(ctx context.Context, block tsdb.BlockReader, indexr tsdb.
 				it := fhchk.Iterator(nil)
 				bucketCount := 0
 				for it.Next() == chunkenc.ValFloatHistogram {
-					_, f := it.AtFloatHistogram()
+					_, f := it.AtFloatHistogram(nil)
 					bucketCount += len(f.PositiveBuckets)
 					bucketCount += len(f.NegativeBuckets)
 				}
@@ -682,7 +682,7 @@ func analyzeCompaction(ctx context.Context, block tsdb.BlockReader, indexr tsdb.
 				it := hchk.Iterator(nil)
 				bucketCount := 0
 				for it.Next() == chunkenc.ValHistogram {
-					_, f := it.AtHistogram()
+					_, f := it.AtHistogram(nil)
 					bucketCount += len(f.PositiveBuckets)
 					bucketCount += len(f.NegativeBuckets)
 				}
@@ -745,11 +745,11 @@ func dumpSamples(ctx context.Context, path string, mint, maxt int64, match []str
 			fmt.Printf("%s %g %d\n", lbs, val, ts)
 		}
 		for it.Next() == chunkenc.ValFloatHistogram {
-			ts, fh := it.AtFloatHistogram()
+			ts, fh := it.AtFloatHistogram(nil)
 			fmt.Printf("%s %s %d\n", lbs, fh.String(), ts)
 		}
 		for it.Next() == chunkenc.ValHistogram {
-			ts, h := it.AtHistogram()
+			ts, h := it.AtHistogram(nil)
 			fmt.Printf("%s %s %d\n", lbs, h.String(), ts)
 		}
 		if it.Err() != nil {

cmd/promtool/unittest_test.go

@@ -16,6 +16,8 @@ package main
 import (
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/prometheus/prometheus/promql"
 )
 
@@ -178,9 +180,8 @@ func TestRulesUnitTestRun(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := RulesUnitTest(tt.queryOpts, tt.args.run, false, tt.args.files...); got != tt.want {
+			got := RulesUnitTest(tt.queryOpts, tt.args.run, false, tt.args.files...)
-				t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want)
+			require.Equal(t, tt.want, got)
-			}
 		})
 	}
 }

consoles/node-cpu.html

@@ -47,7 +47,7 @@
   <script>
   new PromConsole.Graph({
     node: document.querySelector("#cpuGraph"),
-    expr: "sum by (mode)(irate(node_cpu_seconds_total{job='node',instance='{{ .Params.instance }}',mode!='idle'}[5m]))",
+    expr: "sum by (mode)(irate(node_cpu_seconds_total{job='node',instance='{{ .Params.instance }}',mode!='idle',mode!='iowait',mode!='steal'}[5m]))",
     renderer: 'area',
     max: {{ with printf "count(count by (cpu)(node_cpu_seconds_total{job='node',instance='%s'}))" .Params.instance | query }}{{ . | first | value }}{{ else}}undefined{{end}},
     yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,

consoles/node-overview.html

@@ -69,7 +69,7 @@
   <script>
   new PromConsole.Graph({
     node: document.querySelector("#cpuGraph"),
-    expr: "sum by (mode)(irate(node_cpu_seconds_total{job='node',instance='{{ .Params.instance }}',mode!='idle'}[5m]))",
+    expr: "sum by (mode)(irate(node_cpu_seconds_total{job='node',instance='{{ .Params.instance }}',mode!='idle',mode!='iowait',mode!='steal'}[5m]))",
     renderer: 'area',
     max: {{ with printf "count(count by (cpu)(node_cpu_seconds_total{job='node',instance='%s'}))" .Params.instance | query }}{{ . | first | value }}{{ else}}undefined{{end}},
     yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,

consoles/node.html

@@ -21,7 +21,7 @@
 <tr>
   <td><a href="node-overview.html?instance={{ .Labels.instance }}">{{ reReplaceAll "(.*?://)([^:/]+?)(:\\d+)?/.*" "$2" .Labels.instance }}</a></td>
   <td{{ if eq (. | value) 1.0 }}>Yes{{ else }} class="alert-danger">No{{ end }}</td>
-  <td>{{ template "prom_query_drilldown" (args (printf "100 * (1 - avg by(instance)(irate(node_cpu_seconds_total{job='node',mode='idle',instance='%s'}[5m])))" .Labels.instance) "%" "printf.1f") }}</td>
+  <td>{{ template "prom_query_drilldown" (args (printf "100 * (1 - avg by(instance) (sum without(mode) (irate(node_cpu_seconds_total{job='node',mode=~'idle|iowait|steal',instance='%s'}[5m]))))" .Labels.instance) "%" "printf.1f") }}</td>
   <td>{{ template "prom_query_drilldown" (args (printf "node_memory_MemFree_bytes{job='node',instance='%s'} + node_memory_Cached_bytes{job='node',instance='%s'} + node_memory_Buffers_bytes{job='node',instance='%s'}" .Labels.instance .Labels.instance .Labels.instance) "B" "humanize1024") }}</td>
 </tr>
 {{ else }}

discovery/aws/ec2.go

@@ -97,12 +97,19 @@ type EC2SDConfig struct {
 	HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*EC2SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &ec2Metrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the EC2 Config.
 func (*EC2SDConfig) Name() string { return "ec2" }
 
 // NewDiscoverer returns a Discoverer for the EC2 Config.
 func (c *EC2SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewEC2Discovery(c, opts.Logger, opts.Registerer), nil
+	return NewEC2Discovery(c, opts.Logger, opts.Metrics)
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface for the EC2 Config.
@@ -148,7 +155,12 @@ type EC2Discovery struct {
 }
 
 // NewEC2Discovery returns a new EC2Discovery which periodically refreshes its targets.
-func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger, reg prometheus.Registerer) *EC2Discovery {
+func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*EC2Discovery, error) {
+	m, ok := metrics.(*ec2Metrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -158,14 +170,14 @@ func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger, reg prometheus.Regist
 	}
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
+			Logger:              logger,
-			Mech:     "ec2",
+			Mech:                "ec2",
-			Interval: time.Duration(d.cfg.RefreshInterval),
+			Interval:            time.Duration(d.cfg.RefreshInterval),
-			RefreshF: d.refresh,
+			RefreshF:            d.refresh,
-			Registry: reg,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
-	return d
+	return d, nil
 }
 
 func (d *EC2Discovery) ec2Client(context.Context) (*ec2.EC2, error) {

discovery/aws/lightsail.go

@@ -80,12 +80,19 @@ type LightsailSDConfig struct {
 	HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*LightsailSDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &lightsailMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Lightsail Config.
 func (*LightsailSDConfig) Name() string { return "lightsail" }
 
 // NewDiscoverer returns a Discoverer for the Lightsail Config.
 func (c *LightsailSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewLightsailDiscovery(c, opts.Logger, opts.Registerer), nil
+	return NewLightsailDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface for the Lightsail Config.
@@ -122,23 +129,29 @@ type LightsailDiscovery struct {
 }
 
 // NewLightsailDiscovery returns a new LightsailDiscovery which periodically refreshes its targets.
-func NewLightsailDiscovery(conf *LightsailSDConfig, logger log.Logger, reg prometheus.Registerer) *LightsailDiscovery {
+func NewLightsailDiscovery(conf *LightsailSDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*LightsailDiscovery, error) {
+	m, ok := metrics.(*lightsailMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
+
 	d := &LightsailDiscovery{
 		cfg: conf,
 	}
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
+			Logger:              logger,
-			Mech:     "lightsail",
+			Mech:                "lightsail",
-			Interval: time.Duration(d.cfg.RefreshInterval),
+			Interval:            time.Duration(d.cfg.RefreshInterval),
-			RefreshF: d.refresh,
+			RefreshF:            d.refresh,
-			Registry: reg,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
-	return d
+	return d, nil
 }
 
 func (d *LightsailDiscovery) lightsailClient() (*lightsail.Lightsail, error) {

discovery/aws/metrics_ec2.go

@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aws
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+type ec2Metrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+var _ discovery.DiscovererMetrics = (*ec2Metrics)(nil)
+
+// Register implements discovery.DiscovererMetrics.
+func (m *ec2Metrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *ec2Metrics) Unregister() {}

discovery/aws/metrics_lightsail.go

@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aws
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+type lightsailMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+var _ discovery.DiscovererMetrics = (*lightsailMetrics)(nil)
+
+// Register implements discovery.DiscovererMetrics.
+func (m *lightsailMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *lightsailMetrics) Unregister() {}

discovery/azure/azure.go

@@ -120,12 +120,17 @@ type SDConfig struct {
 	HTTPClientConfig config_util.HTTPClientConfig `yaml:",inline"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return newDiscovererMetrics(reg, rmi)
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "azure" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.Registerer)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 func validateAuthParam(param, name string) error {
@@ -168,45 +173,39 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 
 type Discovery struct {
 	*refresh.Discovery
-	logger        log.Logger
+	logger  log.Logger
-	cfg           *SDConfig
+	cfg     *SDConfig
-	port          int
+	port    int
-	cache         *cache.Cache[string, *armnetwork.Interface]
+	cache   *cache.Cache[string, *armnetwork.Interface]
-	failuresCount prometheus.Counter
+	metrics *azureMetrics
-	cacheHitCount prometheus.Counter
 }
 
 // NewDiscovery returns a new AzureDiscovery which periodically refreshes its targets.
-func NewDiscovery(cfg *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(cfg *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*azureMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
 	l := cache.New(cache.AsLRU[string, *armnetwork.Interface](lru.WithCapacity(5000)))
 	d := &Discovery{
-		cfg:    cfg,
+		cfg:     cfg,
-		port:   cfg.Port,
+		port:    cfg.Port,
-		logger: logger,
+		logger:  logger,
-		cache:  l,
+		cache:   l,
-		failuresCount: prometheus.NewCounter(
+		metrics: m,
-			prometheus.CounterOpts{
-				Name: "prometheus_sd_azure_failures_total",
-				Help: "Number of Azure service discovery refresh failures.",
-			}),
-		cacheHitCount: prometheus.NewCounter(
-			prometheus.CounterOpts{
-				Name: "prometheus_sd_azure_cache_hit_total",
-				Help: "Number of cache hit during refresh.",
-			}),
 	}
 
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
+			Logger:              logger,
-			Mech:     "azure",
+			Mech:                "azure",
-			Interval: time.Duration(cfg.RefreshInterval),
+			Interval:            time.Duration(cfg.RefreshInterval),
-			RefreshF: d.refresh,
+			RefreshF:            d.refresh,
-			Registry: reg,
+			MetricsInstantiator: m.refreshMetrics,
-			Metrics:  []prometheus.Collector{d.failuresCount, d.cacheHitCount},
 		},
 	)
 
@@ -333,14 +332,14 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 
 	client, err := createAzureClient(*d.cfg)
 	if err != nil {
-		d.failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, fmt.Errorf("could not create Azure client: %w", err)
 	}
 	client.logger = d.logger
 
 	machines, err := client.getVMs(ctx, d.cfg.ResourceGroup)
 	if err != nil {
-		d.failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, fmt.Errorf("could not get virtual machines: %w", err)
 	}
 
@@ -349,14 +348,14 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 	// Load the vms managed by scale sets.
 	scaleSets, err := client.getScaleSets(ctx, d.cfg.ResourceGroup)
 	if err != nil {
-		d.failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, fmt.Errorf("could not get virtual machine scale sets: %w", err)
 	}
 
 	for _, scaleSet := range scaleSets {
 		scaleSetVms, err := client.getScaleSetVMs(ctx, scaleSet)
 		if err != nil {
-			d.failuresCount.Inc()
+			d.metrics.failuresCount.Inc()
 			return nil, fmt.Errorf("could not get virtual machine scale set vms: %w", err)
 		}
 		machines = append(machines, scaleSetVms...)
@@ -407,7 +406,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 				var networkInterface *armnetwork.Interface
 				if v, ok := d.getFromCache(nicID); ok {
 					networkInterface = v
-					d.cacheHitCount.Add(1)
+					d.metrics.cacheHitCount.Add(1)
 				} else {
 					if vm.ScaleSet == "" {
 						networkInterface, err = client.getVMNetworkInterfaceByID(ctx, nicID)
@@ -480,7 +479,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 	var tg targetgroup.Group
 	for tgt := range ch {
 		if tgt.err != nil {
-			d.failuresCount.Inc()
+			d.metrics.failuresCount.Inc()
 			return nil, fmt.Errorf("unable to complete Azure service discovery: %w", tgt.err)
 		}
 		if tgt.labelSet != nil {

discovery/azure/metrics.go

@@ -0,0 +1,64 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package azure
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*azureMetrics)(nil)
+
+type azureMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+
+	failuresCount prometheus.Counter
+	cacheHitCount prometheus.Counter
+
+	metricRegisterer discovery.MetricRegisterer
+}
+
+func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	m := &azureMetrics{
+		refreshMetrics: rmi,
+		failuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_azure_failures_total",
+				Help: "Number of Azure service discovery refresh failures.",
+			}),
+		cacheHitCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_azure_cache_hit_total",
+				Help: "Number of cache hit during refresh.",
+			}),
+	}
+
+	m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
+		m.failuresCount,
+		m.cacheHitCount,
+	})
+
+	return m
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *azureMetrics) Register() error {
+	return m.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *azureMetrics) Unregister() {
+	m.metricRegisterer.UnregisterMetrics()
+}

discovery/consul/consul.go

@@ -119,12 +119,17 @@ type SDConfig struct {
 	HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return newDiscovererMetrics(reg, rmi)
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "consul" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.Registerer)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -161,27 +166,28 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 // Discovery retrieves target information from a Consul server
 // and updates them via watches.
 type Discovery struct {
-	client              *consul.Client
+	client           *consul.Client
-	clientDatacenter    string
+	clientDatacenter string
-	clientNamespace     string
+	clientNamespace  string
-	clientPartition     string
+	clientPartition  string
-	tagSeparator        string
+	tagSeparator     string
-	watchedServices     []string // Set of services which will be discovered.
+	watchedServices  []string // Set of services which will be discovered.
-	watchedTags         []string // Tags used to filter instances of a service.
+	watchedTags      []string // Tags used to filter instances of a service.
-	watchedNodeMeta     map[string]string
+	watchedNodeMeta  map[string]string
-	allowStale          bool
+	allowStale       bool
-	refreshInterval     time.Duration
+	refreshInterval  time.Duration
-	finalizer           func()
+	finalizer        func()
-	logger              log.Logger
+	logger           log.Logger
-	rpcFailuresCount    prometheus.Counter
+	metrics          *consulMetrics
-	rpcDuration         *prometheus.SummaryVec
-	servicesRPCDuration prometheus.Observer
-	serviceRPCDuration  prometheus.Observer
-	metricRegisterer    discovery.MetricRegisterer
 }
 
 // NewDiscovery returns a new Discovery for the given config.
-func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*consulMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -219,35 +225,9 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer)
 		clientPartition:  conf.Partition,
 		finalizer:        wrapper.CloseIdleConnections,
 		logger:           logger,
-		rpcFailuresCount: prometheus.NewCounter(
+		metrics:          m,
-			prometheus.CounterOpts{
-				Namespace: namespace,
-				Name:      "sd_consul_rpc_failures_total",
-				Help:      "The number of Consul RPC call failures.",
-			}),
-		rpcDuration: prometheus.NewSummaryVec(
-			prometheus.SummaryOpts{
-				Namespace:  namespace,
-				Name:       "sd_consul_rpc_duration_seconds",
-				Help:       "The duration of a Consul RPC call in seconds.",
-				Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
-			},
-			[]string{"endpoint", "call"},
-		),
 	}
 
-	cd.metricRegisterer = discovery.NewMetricRegisterer(
-		reg,
-		[]prometheus.Collector{
-			cd.rpcFailuresCount,
-			cd.rpcDuration,
-		},
-	)
-
-	// Initialize metric vectors.
-	cd.servicesRPCDuration = cd.rpcDuration.WithLabelValues("catalog", "services")
-	cd.serviceRPCDuration = cd.rpcDuration.WithLabelValues("catalog", "service")
-
 	return cd, nil
 }
 
@@ -303,7 +283,7 @@ func (d *Discovery) getDatacenter() error {
 	info, err := d.client.Agent().Self()
 	if err != nil {
 		level.Error(d.logger).Log("msg", "Error retrieving datacenter name", "err", err)
-		d.rpcFailuresCount.Inc()
+		d.metrics.rpcFailuresCount.Inc()
 		return err
 	}
 
@@ -344,13 +324,6 @@ func (d *Discovery) initialize(ctx context.Context) {
 
 // Run implements the Discoverer interface.
 func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
-	err := d.metricRegisterer.RegisterMetrics()
-	if err != nil {
-		level.Error(d.logger).Log("msg", "Unable to register metrics", "err", err.Error())
-		return
-	}
-	defer d.metricRegisterer.UnregisterMetrics()
-
 	if d.finalizer != nil {
 		defer d.finalizer()
 	}
@@ -399,7 +372,7 @@ func (d *Discovery) watchServices(ctx context.Context, ch chan<- []*targetgroup.
 	t0 := time.Now()
 	srvs, meta, err := catalog.Services(opts.WithContext(ctx))
 	elapsed := time.Since(t0)
-	d.servicesRPCDuration.Observe(elapsed.Seconds())
+	d.metrics.servicesRPCDuration.Observe(elapsed.Seconds())
 
 	// Check the context before in order to exit early.
 	select {
@@ -410,7 +383,7 @@ func (d *Discovery) watchServices(ctx context.Context, ch chan<- []*targetgroup.
 
 	if err != nil {
 		level.Error(d.logger).Log("msg", "Error refreshing service list", "err", err)
-		d.rpcFailuresCount.Inc()
+		d.metrics.rpcFailuresCount.Inc()
 		time.Sleep(retryInterval)
 		return
 	}
@@ -490,8 +463,8 @@ func (d *Discovery) watchService(ctx context.Context, ch chan<- []*targetgroup.G
 		},
 		tagSeparator:       d.tagSeparator,
 		logger:             d.logger,
-		rpcFailuresCount:   d.rpcFailuresCount,
+		rpcFailuresCount:   d.metrics.rpcFailuresCount,
-		serviceRPCDuration: d.serviceRPCDuration,
+		serviceRPCDuration: d.metrics.serviceRPCDuration,
 	}
 
 	go func() {

discovery/consul/consul_test.go

@@ -29,6 +29,7 @@ import (
 	"go.uber.org/goleak"
 	"gopkg.in/yaml.v2"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -36,20 +37,30 @@ func TestMain(m *testing.M) {
 	goleak.VerifyTestMain(m)
 }
 
+// TODO: Add ability to unregister metrics?
+func NewTestMetrics(t *testing.T, conf discovery.Config, reg prometheus.Registerer) discovery.DiscovererMetrics {
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	require.NoError(t, refreshMetrics.Register())
+
+	metrics := conf.NewDiscovererMetrics(prometheus.NewRegistry(), refreshMetrics)
+	require.NoError(t, metrics.Register())
+
+	return metrics
+}
+
 func TestConfiguredService(t *testing.T) {
 	conf := &SDConfig{
 		Services: []string{"configuredServiceName"},
 	}
-	consulDiscovery, err := NewDiscovery(conf, nil, prometheus.NewRegistry())
+
-	if err != nil {
+	metrics := NewTestMetrics(t, conf, prometheus.NewRegistry())
-		t.Errorf("Unexpected error when initializing discovery %v", err)
+
-	}
+	consulDiscovery, err := NewDiscovery(conf, nil, metrics)
-	if !consulDiscovery.shouldWatch("configuredServiceName", []string{""}) {
+	require.NoError(t, err, "when initializing discovery")
-		t.Errorf("Expected service %s to be watched", "configuredServiceName")
+	require.True(t, consulDiscovery.shouldWatch("configuredServiceName", []string{""}),
-	}
+		"Expected service %s to be watched", "configuredServiceName")
-	if consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{""}) {
+	require.False(t, consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{""}),
-		t.Errorf("Expected service %s to not be watched", "nonConfiguredServiceName")
+		"Expected service %s to not be watched", "nonConfiguredServiceName")
-	}
 }
 
 func TestConfiguredServiceWithTag(t *testing.T) {
@@ -57,22 +68,22 @@ func TestConfiguredServiceWithTag(t *testing.T) {
 		Services:    []string{"configuredServiceName"},
 		ServiceTags: []string{"http"},
 	}
-	consulDiscovery, err := NewDiscovery(conf, nil, prometheus.NewRegistry())
+
-	if err != nil {
+	metrics := NewTestMetrics(t, conf, prometheus.NewRegistry())
-		t.Errorf("Unexpected error when initializing discovery %v", err)
+
-	}
+	consulDiscovery, err := NewDiscovery(conf, nil, metrics)
-	if consulDiscovery.shouldWatch("configuredServiceName", []string{""}) {
+	require.NoError(t, err, "when initializing discovery")
-		t.Errorf("Expected service %s to not be watched without tag", "configuredServiceName")
+	require.False(t, consulDiscovery.shouldWatch("configuredServiceName", []string{""}),
-	}
+		"Expected service %s to not be watched without tag", "configuredServiceName")
-	if !consulDiscovery.shouldWatch("configuredServiceName", []string{"http"}) {
+
-		t.Errorf("Expected service %s to be watched with tag %s", "configuredServiceName", "http")
+	require.True(t, consulDiscovery.shouldWatch("configuredServiceName", []string{"http"}),
-	}
+		"Expected service %s to be watched with tag %s", "configuredServiceName", "http")
-	if consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{""}) {
+
-		t.Errorf("Expected service %s to not be watched without tag", "nonConfiguredServiceName")
+	require.False(t, consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{""}),
-	}
+		"Expected service %s to not be watched without tag", "nonConfiguredServiceName")
-	if consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{"http"}) {
+
-		t.Errorf("Expected service %s to not be watched with tag %s", "nonConfiguredServiceName", "http")
+	require.False(t, consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{"http"}),
-	}
+		"Expected service %s to not be watched with tag %s", "nonConfiguredServiceName", "http")
 }
 
 func TestConfiguredServiceWithTags(t *testing.T) {
@@ -152,27 +163,24 @@ func TestConfiguredServiceWithTags(t *testing.T) {
 	}
 
 	for _, tc := range cases {
-		consulDiscovery, err := NewDiscovery(tc.conf, nil, prometheus.NewRegistry())
+		metrics := NewTestMetrics(t, tc.conf, prometheus.NewRegistry())
-		if err != nil {
-			t.Errorf("Unexpected error when initializing discovery %v", err)
-		}
-		ret := consulDiscovery.shouldWatch(tc.serviceName, tc.serviceTags)
-		if ret != tc.shouldWatch {
-			t.Errorf("Expected should watch? %t, got %t. Watched service and tags: %s %+v, input was %s %+v", tc.shouldWatch, ret, tc.conf.Services, tc.conf.ServiceTags, tc.serviceName, tc.serviceTags)
-		}
 
+		consulDiscovery, err := NewDiscovery(tc.conf, nil, metrics)
+		require.NoError(t, err, "when initializing discovery")
+		ret := consulDiscovery.shouldWatch(tc.serviceName, tc.serviceTags)
+		require.Equal(t, tc.shouldWatch, ret, "Watched service and tags: %s %+v, input was %s %+v",
+			tc.conf.Services, tc.conf.ServiceTags, tc.serviceName, tc.serviceTags)
 	}
 }
 
 func TestNonConfiguredService(t *testing.T) {
 	conf := &SDConfig{}
-	consulDiscovery, err := NewDiscovery(conf, nil, prometheus.NewRegistry())
+
-	if err != nil {
+	metrics := NewTestMetrics(t, conf, prometheus.NewRegistry())
-		t.Errorf("Unexpected error when initializing discovery %v", err)
+
-	}
+	consulDiscovery, err := NewDiscovery(conf, nil, metrics)
-	if !consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{""}) {
+	require.NoError(t, err, "when initializing discovery")
-		t.Errorf("Expected service %s to be watched", "nonConfiguredServiceName")
+	require.True(t, consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{""}), "Expected service %s to be watched", "nonConfiguredServiceName")
-	}
 }
 
 const (
@@ -263,7 +271,10 @@ func newServer(t *testing.T) (*httptest.Server, *SDConfig) {
 
 func newDiscovery(t *testing.T, config *SDConfig) *Discovery {
 	logger := log.NewNopLogger()
-	d, err := NewDiscovery(config, logger, prometheus.NewRegistry())
+
+	metrics := NewTestMetrics(t, config, prometheus.NewRegistry())
+
+	d, err := NewDiscovery(config, logger, metrics)
 	require.NoError(t, err)
 	return d
 }
@@ -477,13 +488,10 @@ oauth2:
 			var config SDConfig
 			err := config.UnmarshalYAML(unmarshal([]byte(test.config)))
 			if err != nil {
-				require.Equalf(t, err.Error(), test.errMessage, "Expected error '%s', got '%v'", test.errMessage, err)
+				require.EqualError(t, err, test.errMessage)
-				return
-			}
-			if test.errMessage != "" {
-				t.Errorf("Expected error %s, got none", test.errMessage)
 				return
 			}
+			require.Empty(t, test.errMessage, "Expected error.")
 
 			require.Equal(t, test.expected, config)
 		})

discovery/consul/metrics.go

@@ -0,0 +1,73 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package consul
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*consulMetrics)(nil)
+
+type consulMetrics struct {
+	rpcFailuresCount prometheus.Counter
+	rpcDuration      *prometheus.SummaryVec
+
+	servicesRPCDuration prometheus.Observer
+	serviceRPCDuration  prometheus.Observer
+
+	metricRegisterer discovery.MetricRegisterer
+}
+
+func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	m := &consulMetrics{
+		rpcFailuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      "sd_consul_rpc_failures_total",
+				Help:      "The number of Consul RPC call failures.",
+			}),
+		rpcDuration: prometheus.NewSummaryVec(
+			prometheus.SummaryOpts{
+				Namespace:  namespace,
+				Name:       "sd_consul_rpc_duration_seconds",
+				Help:       "The duration of a Consul RPC call in seconds.",
+				Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+			},
+			[]string{"endpoint", "call"},
+		),
+	}
+
+	m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
+		m.rpcFailuresCount,
+		m.rpcDuration,
+	})
+
+	// Initialize metric vectors.
+	m.servicesRPCDuration = m.rpcDuration.WithLabelValues("catalog", "services")
+	m.serviceRPCDuration = m.rpcDuration.WithLabelValues("catalog", "service")
+
+	return m
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *consulMetrics) Register() error {
+	return m.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *consulMetrics) Unregister() {
+	m.metricRegisterer.UnregisterMetrics()
+}

discovery/digitalocean/digitalocean.go

@@ -63,6 +63,13 @@ func init() {
 	discovery.RegisterConfig(&SDConfig{})
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &digitaloceanMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // SDConfig is the configuration for DigitalOcean based service discovery.
 type SDConfig struct {
 	HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
@@ -76,7 +83,7 @@ func (*SDConfig) Name() string { return "digitalocean" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.Registerer)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -104,7 +111,12 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*digitaloceanMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	d := &Discovery{
 		port: conf.Port,
 	}
@@ -127,11 +139,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer)
 
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
+			Logger:              logger,
-			Mech:     "digitalocean",
+			Mech:                "digitalocean",
-			Interval: time.Duration(conf.RefreshInterval),
+			Interval:            time.Duration(conf.RefreshInterval),
-			RefreshF: d.refresh,
+			RefreshF:            d.refresh,
-			Registry: reg,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
 	return d, nil

discovery/digitalocean/digitalocean_test.go

@@ -23,6 +23,8 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/prometheus/discovery"
 )
 
 type DigitalOceanSDTestSuite struct {
@@ -47,7 +49,15 @@ func TestDigitalOceanSDRefresh(t *testing.T) {
 
 	cfg := DefaultSDConfig
 	cfg.HTTPClientConfig.BearerToken = tokenID
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 	endpoint, err := url.Parse(sdmock.Mock.Endpoint())
 	require.NoError(t, err)

discovery/digitalocean/metrics.go

@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package digitalocean
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*digitaloceanMetrics)(nil)
+
+type digitaloceanMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *digitaloceanMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *digitaloceanMetrics) Unregister() {}

discovery/discoverer_metrics_noop.go

@@ -0,0 +1,28 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package discovery
+
+// Create a dummy metrics struct, because this SD doesn't have any metrics.
+type NoopDiscovererMetrics struct{}
+
+var _ DiscovererMetrics = (*NoopDiscovererMetrics)(nil)
+
+// Register implements discovery.DiscovererMetrics.
+func (*NoopDiscovererMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (*NoopDiscovererMetrics) Unregister() {
+}

discovery/discovery.go

@@ -39,24 +39,47 @@ type Discoverer interface {
 	Run(ctx context.Context, up chan<- []*targetgroup.Group)
 }
 
+// Internal metrics of service discovery mechanisms.
+type DiscovererMetrics interface {
+	Register() error
+	Unregister()
+}
+
 // DiscovererOptions provides options for a Discoverer.
 type DiscovererOptions struct {
 	Logger log.Logger
 
-	// A registerer for the Discoverer's metrics.
+	Metrics DiscovererMetrics
-	// Some Discoverers may ignore this registerer and use the global one instead.
-	// For now this will work, because the Prometheus `main` function uses the global registry.
-	// However, in the future the Prometheus `main` function will be updated to not use the global registry.
-	// Hence, if a discoverer wants its metrics to be visible via the Prometheus executable's
-	// `/metrics` endpoint, it should use this explicit registerer.
-	// TODO(ptodev): Update this comment once the Prometheus `main` function does not use the global registry.
-	Registerer prometheus.Registerer
 
 	// Extra HTTP client options to expose to Discoverers. This field may be
 	// ignored; Discoverer implementations must opt-in to reading it.
 	HTTPClientOptions []config.HTTPClientOption
 }
 
+// Metrics used by the "refresh" package.
+// We define them here in the "discovery" package in order to avoid a cyclic dependency between
+// "discovery" and "refresh".
+type RefreshMetrics struct {
+	Failures prometheus.Counter
+	Duration prometheus.Observer
+}
+
+// Instantiate the metrics used by the "refresh" package.
+type RefreshMetricsInstantiator interface {
+	Instantiate(mech string) *RefreshMetrics
+}
+
+// An interface for registering, unregistering, and instantiating metrics for the "refresh" package.
+// Refresh metrics are registered and unregistered outside of the service discovery mechanism.
+// This is so that the same metrics can be reused across different service discovery mechanisms.
+// To manage refresh metrics inside the SD mechanism, we'd need to use const labels which are
+// specific to that SD. However, doing so would also expose too many unused metrics on
+// the Prometheus /metrics endpoint.
+type RefreshMetricsManager interface {
+	DiscovererMetrics
+	RefreshMetricsInstantiator
+}
+
 // A Config provides the configuration and constructor for a Discoverer.
 type Config interface {
 	// Name returns the name of the discovery mechanism.
@@ -65,6 +88,9 @@ type Config interface {
 	// NewDiscoverer returns a Discoverer for the Config
 	// with the given DiscovererOptions.
 	NewDiscoverer(DiscovererOptions) (Discoverer, error)
+
+	// NewDiscovererMetrics returns the metrics used by the service discovery.
+	NewDiscovererMetrics(prometheus.Registerer, RefreshMetricsInstantiator) DiscovererMetrics
 }
 
 // Configs is a slice of Config values that uses custom YAML marshaling and unmarshaling
@@ -119,6 +145,11 @@ func (c StaticConfig) NewDiscoverer(DiscovererOptions) (Discoverer, error) {
 	return staticDiscoverer(c), nil
 }
 
+// No metrics are needed for this service discovery mechanism.
+func (c StaticConfig) NewDiscovererMetrics(prometheus.Registerer, RefreshMetricsInstantiator) DiscovererMetrics {
+	return &NoopDiscovererMetrics{}
+}
+
 type staticDiscoverer []*targetgroup.Group
 
 func (c staticDiscoverer) Run(ctx context.Context, up chan<- []*targetgroup.Group) {

discovery/dns/dns.go

@@ -67,12 +67,17 @@ type SDConfig struct {
 	Port            int            `yaml:"port"` // Ignored for SRV records
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return newDiscovererMetrics(reg, rmi)
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "dns" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(*c, opts.Logger, opts.Registerer)
+	return NewDiscovery(*c, opts.Logger, opts.Metrics)
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
@@ -102,18 +107,22 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 // the Discoverer interface.
 type Discovery struct {
 	*refresh.Discovery
-	names                    []string
+	names   []string
-	port                     int
+	port    int
-	qtype                    uint16
+	qtype   uint16
-	logger                   log.Logger
+	logger  log.Logger
-	dnsSDLookupsCount        prometheus.Counter
+	metrics *dnsMetrics
-	dnsSDLookupFailuresCount prometheus.Counter
 
 	lookupFn func(name string, qtype uint16, logger log.Logger) (*dns.Msg, error)
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(conf SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*dnsMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -137,28 +146,16 @@ func NewDiscovery(conf SDConfig, logger log.Logger, reg prometheus.Registerer) (
 		port:     conf.Port,
 		logger:   logger,
 		lookupFn: lookupWithSearchPath,
-		dnsSDLookupsCount: prometheus.NewCounter(
+		metrics:  m,
-			prometheus.CounterOpts{
-				Namespace: namespace,
-				Name:      "sd_dns_lookups_total",
-				Help:      "The number of DNS-SD lookups.",
-			}),
-		dnsSDLookupFailuresCount: prometheus.NewCounter(
-			prometheus.CounterOpts{
-				Namespace: namespace,
-				Name:      "sd_dns_lookup_failures_total",
-				Help:      "The number of DNS-SD lookup failures.",
-			}),
 	}
 
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
+			Logger:              logger,
-			Mech:     "dns",
+			Mech:                "dns",
-			Interval: time.Duration(conf.RefreshInterval),
+			Interval:            time.Duration(conf.RefreshInterval),
-			RefreshF: d.refresh,
+			RefreshF:            d.refresh,
-			Registry: prometheus.NewRegistry(),
+			MetricsInstantiator: m.refreshMetrics,
-			Metrics:  []prometheus.Collector{d.dnsSDLookupsCount, d.dnsSDLookupFailuresCount},
 		},
 	)
 
@@ -195,9 +192,9 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 
 func (d *Discovery) refreshOne(ctx context.Context, name string, ch chan<- *targetgroup.Group) error {
 	response, err := d.lookupFn(name, d.qtype, d.logger)
-	d.dnsSDLookupsCount.Inc()
+	d.metrics.dnsSDLookupsCount.Inc()
 	if err != nil {
-		d.dnsSDLookupFailuresCount.Inc()
+		d.metrics.dnsSDLookupFailuresCount.Inc()
 		return err
 	}
 

discovery/dns/dns_test.go

@@ -28,6 +28,7 @@ import (
 	"go.uber.org/goleak"
 	"gopkg.in/yaml.v2"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -253,13 +254,21 @@ func TestDNS(t *testing.T) {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
-			sd, err := NewDiscovery(tc.config, nil, prometheus.NewRegistry())
+
+			reg := prometheus.NewRegistry()
+			refreshMetrics := discovery.NewRefreshMetrics(reg)
+			metrics := tc.config.NewDiscovererMetrics(reg, refreshMetrics)
+			require.NoError(t, metrics.Register())
+
+			sd, err := NewDiscovery(tc.config, nil, metrics)
 			require.NoError(t, err)
 			sd.lookupFn = tc.lookup
 
 			tgs, err := sd.refresh(context.Background())
 			require.NoError(t, err)
 			require.Equal(t, tc.expected, tgs)
+
+			metrics.Unregister()
 		})
 	}
 }

discovery/dns/metrics.go

@@ -0,0 +1,66 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dns
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*dnsMetrics)(nil)
+
+type dnsMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+
+	dnsSDLookupsCount        prometheus.Counter
+	dnsSDLookupFailuresCount prometheus.Counter
+
+	metricRegisterer discovery.MetricRegisterer
+}
+
+func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	m := &dnsMetrics{
+		refreshMetrics: rmi,
+		dnsSDLookupsCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      "sd_dns_lookups_total",
+				Help:      "The number of DNS-SD lookups.",
+			}),
+		dnsSDLookupFailuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      "sd_dns_lookup_failures_total",
+				Help:      "The number of DNS-SD lookup failures.",
+			}),
+	}
+
+	m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
+		m.dnsSDLookupsCount,
+		m.dnsSDLookupFailuresCount,
+	})
+
+	return m
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *dnsMetrics) Register() error {
+	return m.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *dnsMetrics) Unregister() {
+	m.metricRegisterer.UnregisterMetrics()
+}

discovery/eureka/eureka.go

@@ -16,6 +16,7 @@ package eureka
 import (
 	"context"
 	"errors"
+	"fmt"
 	"net"
 	"net/http"
 	"net/url"
@@ -76,12 +77,19 @@ type SDConfig struct {
 	RefreshInterval  model.Duration          `yaml:"refresh_interval,omitempty"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &eurekaMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "eureka" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.Registerer)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -118,7 +126,12 @@ type Discovery struct {
 }
 
 // NewDiscovery creates a new Eureka discovery for the given role.
-func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*eurekaMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "eureka_sd")
 	if err != nil {
 		return nil, err
@@ -130,11 +143,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer)
 	}
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
+			Logger:              logger,
-			Mech:     "eureka",
+			Mech:                "eureka",
-			Interval: time.Duration(conf.RefreshInterval),
+			Interval:            time.Duration(conf.RefreshInterval),
-			RefreshF: d.refresh,
+			RefreshF:            d.refresh,
-			Registry: reg,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
 	return d, nil

discovery/eureka/eureka_test.go

@@ -24,6 +24,7 @@ import (
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -36,7 +37,17 @@ func testUpdateServices(respHandler http.HandlerFunc) ([]*targetgroup.Group, err
 		Server: ts.URL,
 	}
 
-	md, err := NewDiscovery(&conf, nil, prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := conf.NewDiscovererMetrics(reg, refreshMetrics)
+	err := metrics.Register()
+	if err != nil {
+		return nil, err
+	}
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	md, err := NewDiscovery(&conf, nil, metrics)
 	if err != nil {
 		return nil, err
 	}

discovery/eureka/metrics.go

@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package eureka
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*eurekaMetrics)(nil)
+
+type eurekaMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *eurekaMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *eurekaMetrics) Unregister() {}

discovery/file/file.go

@@ -57,12 +57,17 @@ type SDConfig struct {
 	RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return newDiscovererMetrics(reg, rmi)
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "file" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.Registerer)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -94,6 +99,9 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 const fileSDFilepathLabel = model.MetaLabelPrefix + "filepath"
 
 // TimestampCollector is a Custom Collector for Timestamps of the files.
+// TODO(ptodev): Now that each file SD has its own TimestampCollector
+// inside discovery/file/metrics.go, we can refactor this collector
+// (or get rid of it) as each TimestampCollector instance will only use one discoverer.
 type TimestampCollector struct {
 	Description *prometheus.Desc
 	discoverers map[*Discovery]struct{}
@@ -169,16 +177,16 @@ type Discovery struct {
 	lastRefresh map[string]int
 	logger      log.Logger
 
-	fileSDReadErrorsCount  prometheus.Counter
+	metrics *fileMetrics
-	fileSDScanDuration     prometheus.Summary
-	fileWatcherErrorsCount prometheus.Counter
-	fileSDTimeStamp        *TimestampCollector
-
-	metricRegisterer discovery.MetricRegisterer
 }
 
 // NewDiscovery returns a new file discovery for the given paths.
-func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	fm, ok := metrics.(*fileMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -188,33 +196,10 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer)
 		interval:   time.Duration(conf.RefreshInterval),
 		timestamps: make(map[string]float64),
 		logger:     logger,
-		fileSDReadErrorsCount: prometheus.NewCounter(
+		metrics:    fm,
-			prometheus.CounterOpts{
-				Name: "prometheus_sd_file_read_errors_total",
-				Help: "The number of File-SD read errors.",
-			}),
-		fileSDScanDuration: prometheus.NewSummary(
-			prometheus.SummaryOpts{
-				Name:       "prometheus_sd_file_scan_duration_seconds",
-				Help:       "The duration of the File-SD scan in seconds.",
-				Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
-			}),
-		fileWatcherErrorsCount: prometheus.NewCounter(
-			prometheus.CounterOpts{
-				Name: "prometheus_sd_file_watcher_errors_total",
-				Help: "The number of File-SD errors caused by filesystem watch failures.",
-			}),
-		fileSDTimeStamp: NewTimestampCollector(),
 	}
 
-	disc.fileSDTimeStamp.addDiscoverer(disc)
+	fm.init(disc)
-
-	disc.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
-		disc.fileSDReadErrorsCount,
-		disc.fileSDScanDuration,
-		disc.fileWatcherErrorsCount,
-		disc.fileSDTimeStamp,
-	})
 
 	return disc, nil
 }
@@ -253,17 +238,10 @@ func (d *Discovery) watchFiles() {
 
 // Run implements the Discoverer interface.
 func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
-	err := d.metricRegisterer.RegisterMetrics()
-	if err != nil {
-		level.Error(d.logger).Log("msg", "Unable to register metrics", "err", err.Error())
-		return
-	}
-	defer d.metricRegisterer.UnregisterMetrics()
-
 	watcher, err := fsnotify.NewWatcher()
 	if err != nil {
 		level.Error(d.logger).Log("msg", "Error adding file watcher", "err", err)
-		d.fileWatcherErrorsCount.Inc()
+		d.metrics.fileWatcherErrorsCount.Inc()
 		return
 	}
 	d.watcher = watcher
@@ -327,7 +305,7 @@ func (d *Discovery) stop() {
 	done := make(chan struct{})
 	defer close(done)
 
-	d.fileSDTimeStamp.removeDiscoverer(d)
+	d.metrics.fileSDTimeStamp.removeDiscoverer(d)
 
 	// Closing the watcher will deadlock unless all events and errors are drained.
 	go func() {
@@ -353,13 +331,13 @@ func (d *Discovery) stop() {
 func (d *Discovery) refresh(ctx context.Context, ch chan<- []*targetgroup.Group) {
 	t0 := time.Now()
 	defer func() {
-		d.fileSDScanDuration.Observe(time.Since(t0).Seconds())
+		d.metrics.fileSDScanDuration.Observe(time.Since(t0).Seconds())
 	}()
 	ref := map[string]int{}
 	for _, p := range d.listFiles() {
 		tgroups, err := d.readFile(p)
 		if err != nil {
-			d.fileSDReadErrorsCount.Inc()
+			d.metrics.fileSDReadErrorsCount.Inc()
 
 			level.Error(d.logger).Log("msg", "Error reading file", "path", p, "err", err)
 			// Prevent deletion down below.

discovery/file/file_test.go

@@ -29,6 +29,7 @@ import (
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -144,19 +145,28 @@ func (t *testRunner) run(files ...string) {
 	ctx, cancel := context.WithCancel(context.Background())
 	t.cancelSD = cancel
 	go func() {
+		conf := &SDConfig{
+			Files: files,
+			// Setting a high refresh interval to make sure that the tests only
+			// rely on file watches.
+			RefreshInterval: model.Duration(1 * time.Hour),
+		}
+
+		reg := prometheus.NewRegistry()
+		refreshMetrics := discovery.NewRefreshMetrics(reg)
+		metrics := conf.NewDiscovererMetrics(reg, refreshMetrics)
+		require.NoError(t, metrics.Register())
+
 		d, err := NewDiscovery(
-			&SDConfig{
+			conf,
-				Files: files,
-				// Setting a high refresh interval to make sure that the tests only
-				// rely on file watches.
-				RefreshInterval: model.Duration(1 * time.Hour),
-			},
 			nil,
-			prometheus.NewRegistry(),
+			metrics,
 		)
 		require.NoError(t, err)
 
 		d.Run(ctx, t.ch)
+
+		metrics.Unregister()
 	}()
 }
 
@@ -348,9 +358,7 @@ func TestInvalidFile(t *testing.T) {
 
 			// Verify that we've received nothing.
 			time.Sleep(defaultWait)
-			if runner.lastReceive().After(now) {
+			require.False(t, runner.lastReceive().After(now), "unexpected targets received: %v", runner.targets())
-				t.Fatalf("unexpected targets received: %v", runner.targets())
-			}
 		})
 	}
 }

discovery/file/metrics.go

@@ -0,0 +1,76 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package file
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*fileMetrics)(nil)
+
+type fileMetrics struct {
+	fileSDReadErrorsCount  prometheus.Counter
+	fileSDScanDuration     prometheus.Summary
+	fileWatcherErrorsCount prometheus.Counter
+	fileSDTimeStamp        *TimestampCollector
+
+	metricRegisterer discovery.MetricRegisterer
+}
+
+func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	fm := &fileMetrics{
+		fileSDReadErrorsCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_file_read_errors_total",
+				Help: "The number of File-SD read errors.",
+			}),
+		fileSDScanDuration: prometheus.NewSummary(
+			prometheus.SummaryOpts{
+				Name:       "prometheus_sd_file_scan_duration_seconds",
+				Help:       "The duration of the File-SD scan in seconds.",
+				Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+			}),
+		fileWatcherErrorsCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_file_watcher_errors_total",
+				Help: "The number of File-SD errors caused by filesystem watch failures.",
+			}),
+		fileSDTimeStamp: NewTimestampCollector(),
+	}
+
+	fm.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
+		fm.fileSDReadErrorsCount,
+		fm.fileSDScanDuration,
+		fm.fileWatcherErrorsCount,
+		fm.fileSDTimeStamp,
+	})
+
+	return fm
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (fm *fileMetrics) Register() error {
+	return fm.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (fm *fileMetrics) Unregister() {
+	fm.metricRegisterer.UnregisterMetrics()
+}
+
+func (fm *fileMetrics) init(disc *Discovery) {
+	fm.fileSDTimeStamp.addDiscoverer(disc)
+}

discovery/gce/gce.go

@@ -82,12 +82,19 @@ type SDConfig struct {
 	TagSeparator    string         `yaml:"tag_separator,omitempty"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &gceMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "gce" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(*c, opts.Logger, opts.Registerer)
+	return NewDiscovery(*c, opts.Logger, opts.Metrics)
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
@@ -122,7 +129,12 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(conf SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*gceMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	d := &Discovery{
 		project:      conf.Project,
 		zone:         conf.Zone,
@@ -143,11 +155,11 @@ func NewDiscovery(conf SDConfig, logger log.Logger, reg prometheus.Registerer) (
 
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
+			Logger:              logger,
-			Mech:     "gce",
+			Mech:                "gce",
-			Interval: time.Duration(conf.RefreshInterval),
+			Interval:            time.Duration(conf.RefreshInterval),
-			RefreshF: d.refresh,
+			RefreshF:            d.refresh,
-			Registry: reg,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
 	return d, nil

discovery/gce/metrics.go

@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gce
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*gceMetrics)(nil)
+
+type gceMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *gceMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *gceMetrics) Unregister() {}

discovery/hetzner/hetzner.go

@@ -63,12 +63,19 @@ type SDConfig struct {
 	robotEndpoint   string         // For tests only.
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &hetznerMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "hetzner" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.Registerer)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 type refresher interface {
@@ -128,7 +135,12 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*refresh.Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*refresh.Discovery, error) {
+	m, ok := metrics.(*hetznerMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	r, err := newRefresher(conf, logger)
 	if err != nil {
 		return nil, err
@@ -136,11 +148,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer)
 
 	return refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
+			Logger:              logger,
-			Mech:     "hetzner",
+			Mech:                "hetzner",
-			Interval: time.Duration(conf.RefreshInterval),
+			Interval:            time.Duration(conf.RefreshInterval),
-			RefreshF: r.refresh,
+			RefreshF:            r.refresh,
-			Registry: reg,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	), nil
 }

discovery/hetzner/metrics.go

@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hetzner
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*hetznerMetrics)(nil)
+
+type hetznerMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *hetznerMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *hetznerMetrics) Unregister() {}

discovery/http/http.go

@@ -58,12 +58,17 @@ type SDConfig struct {
 	URL              string                  `yaml:"url"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return newDiscovererMetrics(reg, rmi)
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "http" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.HTTPClientOptions, opts.Registerer)
+	return NewDiscovery(c, opts.Logger, opts.HTTPClientOptions, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -105,11 +110,16 @@ type Discovery struct {
 	client          *http.Client
 	refreshInterval time.Duration
 	tgLastLength    int
-	failuresCount   prometheus.Counter
+	metrics         *httpMetrics
 }
 
 // NewDiscovery returns a new HTTP discovery for the given config.
-func NewDiscovery(conf *SDConfig, logger log.Logger, clientOpts []config.HTTPClientOption, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, clientOpts []config.HTTPClientOption, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*httpMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -124,21 +134,16 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, clientOpts []config.HTTPCli
 		url:             conf.URL,
 		client:          client,
 		refreshInterval: time.Duration(conf.RefreshInterval), // Stored to be sent as headers.
-		failuresCount: prometheus.NewCounter(
+		metrics:         m,
-			prometheus.CounterOpts{
-				Name: "prometheus_sd_http_failures_total",
-				Help: "Number of HTTP service discovery refresh failures.",
-			}),
 	}
 
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
+			Logger:              logger,
-			Mech:     "http",
+			Mech:                "http",
-			Interval: time.Duration(conf.RefreshInterval),
+			Interval:            time.Duration(conf.RefreshInterval),
-			RefreshF: d.Refresh,
+			RefreshF:            d.Refresh,
-			Registry: reg,
+			MetricsInstantiator: m.refreshMetrics,
-			Metrics:  []prometheus.Collector{d.failuresCount},
 		},
 	)
 	return d, nil
@@ -155,7 +160,7 @@ func (d *Discovery) Refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 
 	resp, err := d.client.Do(req.WithContext(ctx))
 	if err != nil {
-		d.failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, err
 	}
 	defer func() {
@@ -164,31 +169,31 @@ func (d *Discovery) Refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 	}()
 
 	if resp.StatusCode != http.StatusOK {
-		d.failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, fmt.Errorf("server returned HTTP status %s", resp.Status)
 	}
 
 	if !matchContentType.MatchString(strings.TrimSpace(resp.Header.Get("Content-Type"))) {
-		d.failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, fmt.Errorf("unsupported content type %q", resp.Header.Get("Content-Type"))
 	}
 
 	b, err := io.ReadAll(resp.Body)
 	if err != nil {
-		d.failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, err
 	}
 
 	var targetGroups []*targetgroup.Group
 
 	if err := json.Unmarshal(b, &targetGroups); err != nil {
-		d.failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, err
 	}
 
 	for i, tg := range targetGroups {
 		if tg == nil {
-			d.failuresCount.Inc()
+			d.metrics.failuresCount.Inc()
 			err = errors.New("nil target group item found")
 			return nil, err
 		}

discovery/http/http_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -41,7 +42,14 @@ func TestHTTPValidRefresh(t *testing.T) {
 		RefreshInterval:  model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	defer refreshMetrics.Unregister()
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()
@@ -63,7 +71,7 @@ func TestHTTPValidRefresh(t *testing.T) {
 		},
 	}
 	require.Equal(t, expectedTargets, tgs)
-	require.Equal(t, 0.0, getFailureCount(d.failuresCount))
+	require.Equal(t, 0.0, getFailureCount(d.metrics.failuresCount))
 }
 
 func TestHTTPInvalidCode(t *testing.T) {
@@ -79,13 +87,20 @@ func TestHTTPInvalidCode(t *testing.T) {
 		RefreshInterval:  model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	defer refreshMetrics.Unregister()
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()
 	_, err = d.Refresh(ctx)
 	require.EqualError(t, err, "server returned HTTP status 400 Bad Request")
-	require.Equal(t, 1.0, getFailureCount(d.failuresCount))
+	require.Equal(t, 1.0, getFailureCount(d.metrics.failuresCount))
 }
 
 func TestHTTPInvalidFormat(t *testing.T) {
@@ -101,13 +116,20 @@ func TestHTTPInvalidFormat(t *testing.T) {
 		RefreshInterval:  model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	defer refreshMetrics.Unregister()
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()
 	_, err = d.Refresh(ctx)
 	require.EqualError(t, err, `unsupported content type "text/plain; charset=utf-8"`)
-	require.Equal(t, 1.0, getFailureCount(d.failuresCount))
+	require.Equal(t, 1.0, getFailureCount(d.metrics.failuresCount))
 }
 
 func getFailureCount(failuresCount prometheus.Counter) float64 {
@@ -412,7 +434,15 @@ func TestSourceDisappeared(t *testing.T) {
 		URL:              ts.URL,
 		RefreshInterval:  model.Duration(1 * time.Second),
 	}
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	defer refreshMetrics.Unregister()
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, metrics)
 	require.NoError(t, err)
 	for _, test := range cases {
 		ctx := context.Background()

discovery/http/metrics.go

@@ -0,0 +1,57 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package http
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*httpMetrics)(nil)
+
+type httpMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+
+	failuresCount prometheus.Counter
+
+	metricRegisterer discovery.MetricRegisterer
+}
+
+func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	m := &httpMetrics{
+		refreshMetrics: rmi,
+		failuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_http_failures_total",
+				Help: "Number of HTTP service discovery refresh failures.",
+			}),
+	}
+
+	m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
+		m.failuresCount,
+	})
+
+	return m
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *httpMetrics) Register() error {
+	return m.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *httpMetrics) Unregister() {
+	m.metricRegisterer.UnregisterMetrics()
+}

discovery/ionos/ionos.go

@@ -15,16 +15,16 @@ package ionos
 
 import (
 	"errors"
+	"fmt"
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
 	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/refresh"
-
-	"github.com/prometheus/client_golang/prometheus"
 )
 
 const (
@@ -43,7 +43,12 @@ func init() {
 type Discovery struct{}
 
 // NewDiscovery returns a new refresh.Discovery for IONOS Cloud.
-func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*refresh.Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*refresh.Discovery, error) {
+	m, ok := metrics.(*ionosMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if conf.ionosEndpoint == "" {
 		conf.ionosEndpoint = "https://api.ionos.com"
 	}
@@ -55,11 +60,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer)
 
 	return refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
+			Logger:              logger,
-			Mech:     "ionos",
+			Mech:                "ionos",
-			Interval: time.Duration(conf.RefreshInterval),
+			Interval:            time.Duration(conf.RefreshInterval),
-			RefreshF: d.refresh,
+			RefreshF:            d.refresh,
-			Registry: reg,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	), nil
 }
@@ -84,6 +89,13 @@ type SDConfig struct {
 	ionosEndpoint string // For tests only.
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &ionosMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the IONOS Cloud service discovery.
 func (c SDConfig) Name() string {
 	return "ionos"
@@ -91,7 +103,7 @@ func (c SDConfig) Name() string {
 
 // NewDiscoverer returns a new discovery.Discoverer for IONOS Cloud.
 func (c SDConfig) NewDiscoverer(options discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(&c, options.Logger, options.Registerer)
+	return NewDiscovery(&c, options.Logger, options.Metrics)
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.

discovery/ionos/metrics.go

@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ionos
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*ionosMetrics)(nil)
+
+type ionosMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *ionosMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *ionosMetrics) Unregister() {}

discovery/kubernetes/endpoints.go

@@ -62,6 +62,8 @@ func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node ca
 	svcUpdateCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleUpdate)
 	svcDeleteCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleDelete)
 
+	podUpdateCount := eventCount.WithLabelValues(RolePod.String(), MetricLabelRoleUpdate)
+
 	e := &Endpoints{
 		logger:           l,
 		endpointsInf:     eps,
@@ -131,6 +133,29 @@ func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node ca
 	if err != nil {
 		level.Error(l).Log("msg", "Error adding services event handler.", "err", err)
 	}
+	_, err = e.podInf.AddEventHandler(cache.ResourceEventHandlerFuncs{
+		UpdateFunc: func(old, cur interface{}) {
+			podUpdateCount.Inc()
+			oldPod, ok := old.(*apiv1.Pod)
+			if !ok {
+				return
+			}
+
+			curPod, ok := cur.(*apiv1.Pod)
+			if !ok {
+				return
+			}
+
+			// the Pod's phase may change without triggering an update on the Endpoints/Service.
+			// https://github.com/prometheus/prometheus/issues/11305.
+			if curPod.Status.Phase != oldPod.Status.Phase {
+				e.enqueuePod(namespacedName(curPod.Namespace, curPod.Name))
+			}
+		},
+	})
+	if err != nil {
+		level.Error(l).Log("msg", "Error adding pods event handler.", "err", err)
+	}
 	if e.withNodeMetadata {
 		_, err = e.nodeInf.AddEventHandler(cache.ResourceEventHandlerFuncs{
 			AddFunc: func(o interface{}) {
@@ -166,6 +191,18 @@ func (e *Endpoints) enqueueNode(nodeName string) {
 	}
 }
 
+func (e *Endpoints) enqueuePod(podNamespacedName string) {
+	endpoints, err := e.endpointsInf.GetIndexer().ByIndex(podIndex, podNamespacedName)
+	if err != nil {
+		level.Error(e.logger).Log("msg", "Error getting endpoints for pod", "pod", podNamespacedName, "err", err)
+		return
+	}
+
+	for _, endpoint := range endpoints {
+		e.enqueue(endpoint)
+	}
+}
+
 func (e *Endpoints) enqueue(obj interface{}) {
 	key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
 	if err != nil {
@@ -312,7 +349,7 @@ func (e *Endpoints) buildEndpoints(eps *apiv1.Endpoints) *targetgroup.Group {
 			tg.Targets = append(tg.Targets, target)
 			return
 		}
-		s := pod.Namespace + "/" + pod.Name
+		s := namespacedName(pod.Namespace, pod.Name)
 
 		sp, ok := seenPods[s]
 		if !ok {

discovery/kubernetes/endpoints_test.go

@@ -969,3 +969,123 @@ func TestEndpointsDiscoveryEmptyPodStatus(t *testing.T) {
 		expectedRes:      map[string]*targetgroup.Group{},
 	}.Run(t)
 }
+
+// TestEndpointsUpdatePod makes sure that Endpoints discovery detects underlying Pods changes.
+// See https://github.com/prometheus/prometheus/issues/11305 for more details.
+func TestEndpointsDiscoveryUpdatePod(t *testing.T) {
+	pod := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "testpod",
+			Namespace: "default",
+			UID:       types.UID("deadbeef"),
+		},
+		Spec: v1.PodSpec{
+			NodeName: "testnode",
+			Containers: []v1.Container{
+				{
+					Name:  "c1",
+					Image: "c1:latest",
+					Ports: []v1.ContainerPort{
+						{
+							Name:          "mainport",
+							ContainerPort: 9000,
+							Protocol:      v1.ProtocolTCP,
+						},
+					},
+				},
+			},
+		},
+		Status: v1.PodStatus{
+			// Pod is in Pending phase when discovered for first time.
+			Phase: "Pending",
+			Conditions: []v1.PodCondition{
+				{
+					Type:   v1.PodReady,
+					Status: v1.ConditionFalse,
+				},
+			},
+			HostIP: "2.3.4.5",
+			PodIP:  "4.3.2.1",
+		},
+	}
+	objs := []runtime.Object{
+		&v1.Endpoints{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "testendpoints",
+				Namespace: "default",
+			},
+			Subsets: []v1.EndpointSubset{
+				{
+					Addresses: []v1.EndpointAddress{
+						{
+							IP: "4.3.2.1",
+							// The Pending Pod may be included because the Endpoints was created manually.
+							// Or because the corresponding service has ".spec.publishNotReadyAddresses: true".
+							TargetRef: &v1.ObjectReference{
+								Kind:      "Pod",
+								Name:      "testpod",
+								Namespace: "default",
+							},
+						},
+					},
+					Ports: []v1.EndpointPort{
+						{
+							Name:     "mainport",
+							Port:     9000,
+							Protocol: v1.ProtocolTCP,
+						},
+					},
+				},
+			},
+		},
+		pod,
+	}
+	n, c := makeDiscovery(RoleEndpoint, NamespaceDiscovery{}, objs...)
+
+	k8sDiscoveryTest{
+		discovery: n,
+		afterStart: func() {
+			// the Pod becomes Ready.
+			pod.Status.Phase = "Running"
+			pod.Status.Conditions = []v1.PodCondition{
+				{
+					Type:   v1.PodReady,
+					Status: v1.ConditionTrue,
+				},
+			}
+			c.CoreV1().Pods(pod.Namespace).Update(context.Background(), pod, metav1.UpdateOptions{})
+		},
+		expectedMaxItems: 2,
+		expectedRes: map[string]*targetgroup.Group{
+			"endpoints/default/testendpoints": {
+				Targets: []model.LabelSet{
+					{
+						"__address__":                                    "4.3.2.1:9000",
+						"__meta_kubernetes_endpoint_port_name":           "mainport",
+						"__meta_kubernetes_endpoint_port_protocol":       "TCP",
+						"__meta_kubernetes_endpoint_ready":               "true",
+						"__meta_kubernetes_endpoint_address_target_kind": "Pod",
+						"__meta_kubernetes_endpoint_address_target_name": "testpod",
+						"__meta_kubernetes_pod_name":                     "testpod",
+						"__meta_kubernetes_pod_ip":                       "4.3.2.1",
+						"__meta_kubernetes_pod_ready":                    "true",
+						"__meta_kubernetes_pod_phase":                    "Running",
+						"__meta_kubernetes_pod_node_name":                "testnode",
+						"__meta_kubernetes_pod_host_ip":                  "2.3.4.5",
+						"__meta_kubernetes_pod_container_name":           "c1",
+						"__meta_kubernetes_pod_container_image":          "c1:latest",
+						"__meta_kubernetes_pod_container_port_name":      "mainport",
+						"__meta_kubernetes_pod_container_port_number":    "9000",
+						"__meta_kubernetes_pod_container_port_protocol":  "TCP",
+						"__meta_kubernetes_pod_uid":                      "deadbeef",
+					},
+				},
+				Labels: model.LabelSet{
+					"__meta_kubernetes_namespace":      "default",
+					"__meta_kubernetes_endpoints_name": "testendpoints",
+				},
+				Source: "endpoints/default/testendpoints",
+			},
+		},
+	}.Run(t)
+}

discovery/kubernetes/endpointslice.go

@@ -358,7 +358,7 @@ func (e *EndpointSlice) buildEndpointSlice(eps endpointSliceAdaptor) *targetgrou
 			tg.Targets = append(tg.Targets, target)
 			return
 		}
-		s := pod.Namespace + "/" + pod.Name
+		s := namespacedName(pod.Namespace, pod.Name)
 
 		sp, ok := seenPods[s]
 		if !ok {

discovery/kubernetes/kubernetes.go

@@ -123,12 +123,17 @@ type SDConfig struct {
 	AttachMetadata     AttachMetadataConfig    `yaml:"attach_metadata,omitempty"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return newDiscovererMetrics(reg, rmi)
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "kubernetes" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return New(opts.Logger, opts.Registerer, c)
+	return New(opts.Logger, opts.Metrics, c)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -265,8 +270,7 @@ type Discovery struct {
 	selectors          roleSelector
 	ownNamespace       string
 	attachMetadata     AttachMetadataConfig
-	eventCount         *prometheus.CounterVec
+	metrics            *kubernetesMetrics
-	metricRegisterer   discovery.MetricRegisterer
 }
 
 func (d *Discovery) getNamespaces() []string {
@@ -285,7 +289,12 @@ func (d *Discovery) getNamespaces() []string {
 }
 
 // New creates a new Kubernetes discovery for the given role.
-func New(l log.Logger, reg prometheus.Registerer, conf *SDConfig) (*Discovery, error) {
+func New(l log.Logger, metrics discovery.DiscovererMetrics, conf *SDConfig) (*Discovery, error) {
+	m, ok := metrics.(*kubernetesMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if l == nil {
 		l = log.NewNopLogger()
 	}
@@ -348,34 +357,7 @@ func New(l log.Logger, reg prometheus.Registerer, conf *SDConfig) (*Discovery, e
 		selectors:          mapSelector(conf.Selectors),
 		ownNamespace:       ownNamespace,
 		attachMetadata:     conf.AttachMetadata,
-		eventCount: prometheus.NewCounterVec(
+		metrics:            m,
-			prometheus.CounterOpts{
-				Namespace: discovery.KubernetesMetricsNamespace,
-				Name:      "events_total",
-				Help:      "The number of Kubernetes events handled.",
-			},
-			[]string{"role", "event"},
-		),
-	}
-
-	d.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{d.eventCount})
-
-	// Initialize metric vectors.
-	for _, role := range []string{
-		RoleEndpointSlice.String(),
-		RoleEndpoint.String(),
-		RoleNode.String(),
-		RolePod.String(),
-		RoleService.String(),
-		RoleIngress.String(),
-	} {
-		for _, evt := range []string{
-			MetricLabelRoleAdd,
-			MetricLabelRoleDelete,
-			MetricLabelRoleUpdate,
-		} {
-			d.eventCount.WithLabelValues(role, evt)
-		}
 	}
 
 	return d, nil
@@ -415,13 +397,6 @@ const resyncDisabled = 0
 func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 	d.Lock()
 
-	err := d.metricRegisterer.RegisterMetrics()
-	if err != nil {
-		level.Error(d.logger).Log("msg", "Unable to register metrics", "err", err.Error())
-		return
-	}
-	defer d.metricRegisterer.UnregisterMetrics()
-
 	namespaces := d.getNamespaces()
 
 	switch d.role {
@@ -513,7 +488,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 				cache.NewSharedInformer(slw, &apiv1.Service{}, resyncDisabled),
 				cache.NewSharedInformer(plw, &apiv1.Pod{}, resyncDisabled),
 				nodeInf,
-				d.eventCount,
+				d.metrics.eventCount,
 			)
 			d.discoverers = append(d.discoverers, eps)
 			go eps.endpointSliceInf.Run(ctx.Done())
@@ -573,7 +548,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 				cache.NewSharedInformer(slw, &apiv1.Service{}, resyncDisabled),
 				cache.NewSharedInformer(plw, &apiv1.Pod{}, resyncDisabled),
 				nodeInf,
-				d.eventCount,
+				d.metrics.eventCount,
 			)
 			d.discoverers = append(d.discoverers, eps)
 			go eps.endpointsInf.Run(ctx.Done())
@@ -605,7 +580,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 				log.With(d.logger, "role", "pod"),
 				d.newPodsByNodeInformer(plw),
 				nodeInformer,
-				d.eventCount,
+				d.metrics.eventCount,
 			)
 			d.discoverers = append(d.discoverers, pod)
 			go pod.podInf.Run(ctx.Done())
@@ -628,7 +603,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 			svc := NewService(
 				log.With(d.logger, "role", "service"),
 				cache.NewSharedInformer(slw, &apiv1.Service{}, resyncDisabled),
-				d.eventCount,
+				d.metrics.eventCount,
 			)
 			d.discoverers = append(d.discoverers, svc)
 			go svc.informer.Run(ctx.Done())
@@ -686,14 +661,14 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 			ingress := NewIngress(
 				log.With(d.logger, "role", "ingress"),
 				informer,
-				d.eventCount,
+				d.metrics.eventCount,
 			)
 			d.discoverers = append(d.discoverers, ingress)
 			go ingress.informer.Run(ctx.Done())
 		}
 	case RoleNode:
 		nodeInformer := d.newNodeInformer(ctx)
-		node := NewNode(log.With(d.logger, "role", "node"), nodeInformer, d.eventCount)
+		node := NewNode(log.With(d.logger, "role", "node"), nodeInformer, d.metrics.eventCount)
 		d.discoverers = append(d.discoverers, node)
 		go node.informer.Run(ctx.Done())
 	default:
@@ -792,6 +767,21 @@ func (d *Discovery) newPodsByNodeInformer(plw *cache.ListWatch) cache.SharedInde
 
 func (d *Discovery) newEndpointsByNodeInformer(plw *cache.ListWatch) cache.SharedIndexInformer {
 	indexers := make(map[string]cache.IndexFunc)
+	indexers[podIndex] = func(obj interface{}) ([]string, error) {
+		e, ok := obj.(*apiv1.Endpoints)
+		if !ok {
+			return nil, fmt.Errorf("object is not endpoints")
+		}
+		var pods []string
+		for _, target := range e.Subsets {
+			for _, addr := range target.Addresses {
+				if addr.TargetRef != nil && addr.TargetRef.Kind == "Pod" {
+					pods = append(pods, namespacedName(addr.TargetRef.Namespace, addr.TargetRef.Name))
+				}
+			}
+		}
+		return pods, nil
+	}
 	if !d.attachMetadata.Node {
 		return cache.NewSharedIndexInformer(plw, &apiv1.Endpoints{}, resyncDisabled, indexers)
 	}
@@ -897,3 +887,7 @@ func addObjectMetaLabels(labelSet model.LabelSet, objectMeta metav1.ObjectMeta,
 		labelSet[model.LabelName(metaLabelPrefix+string(role)+"_annotationpresent_"+ln)] = presentValue
 	}
 }
+
+func namespacedName(namespace, name string) string {
+	return namespace + "/" + name
+}

discovery/kubernetes/kubernetes_test.go

@@ -51,24 +51,29 @@ func makeDiscoveryWithVersion(role Role, nsDiscovery NamespaceDiscovery, k8sVer
 	fakeDiscovery, _ := clientset.Discovery().(*fakediscovery.FakeDiscovery)
 	fakeDiscovery.FakedServerVersion = &version.Info{GitVersion: k8sVer}
 
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := newDiscovererMetrics(reg, refreshMetrics)
+	err := metrics.Register()
+	if err != nil {
+		panic(err)
+	}
+	// TODO(ptodev): Unregister the metrics at the end of the test.
+
+	kubeMetrics, ok := metrics.(*kubernetesMetrics)
+	if !ok {
+		panic("invalid discovery metrics type")
+	}
+
 	d := &Discovery{
 		client:             clientset,
 		logger:             log.NewNopLogger(),
 		role:               role,
 		namespaceDiscovery: &nsDiscovery,
 		ownNamespace:       "own-ns",
-		eventCount: prometheus.NewCounterVec(
+		metrics:            kubeMetrics,
-			prometheus.CounterOpts{
-				Namespace: discovery.KubernetesMetricsNamespace,
-				Name:      "events_total",
-				Help:      "The number of Kubernetes events handled.",
-			},
-			[]string{"role", "event"},
-		),
 	}
 
-	d.metricRegisterer = discovery.NewMetricRegisterer(prometheus.NewRegistry(), []prometheus.Collector{d.eventCount})
-
 	return d, clientset
 }
 
@@ -126,14 +131,8 @@ func (d k8sDiscoveryTest) Run(t *testing.T) {
 	go readResultWithTimeout(t, ch, d.expectedMaxItems, time.Second, resChan)
 
 	dd, ok := d.discovery.(hasSynced)
-	if !ok {
+	require.True(t, ok, "discoverer does not implement hasSynced interface")
-		t.Errorf("discoverer does not implement hasSynced interface")
+	require.True(t, cache.WaitForCacheSync(ctx.Done(), dd.hasSynced), "discoverer failed to sync: %v", dd)
-		return
-	}
-	if !cache.WaitForCacheSync(ctx.Done(), dd.hasSynced) {
-		t.Errorf("discoverer failed to sync: %v", dd)
-		return
-	}
 
 	if d.afterStart != nil {
 		d.afterStart()

discovery/kubernetes/metrics.go

@@ -0,0 +1,75 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kubernetes
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*kubernetesMetrics)(nil)
+
+type kubernetesMetrics struct {
+	eventCount *prometheus.CounterVec
+
+	metricRegisterer discovery.MetricRegisterer
+}
+
+func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	m := &kubernetesMetrics{
+		eventCount: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Namespace: discovery.KubernetesMetricsNamespace,
+				Name:      "events_total",
+				Help:      "The number of Kubernetes events handled.",
+			},
+			[]string{"role", "event"},
+		),
+	}
+
+	m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
+		m.eventCount,
+	})
+
+	// Initialize metric vectors.
+	for _, role := range []string{
+		RoleEndpointSlice.String(),
+		RoleEndpoint.String(),
+		RoleNode.String(),
+		RolePod.String(),
+		RoleService.String(),
+		RoleIngress.String(),
+	} {
+		for _, evt := range []string{
+			MetricLabelRoleAdd,
+			MetricLabelRoleDelete,
+			MetricLabelRoleUpdate,
+		} {
+			m.eventCount.WithLabelValues(role, evt)
+		}
+	}
+
+	return m
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *kubernetesMetrics) Register() error {
+	return m.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *kubernetesMetrics) Unregister() {
+	m.metricRegisterer.UnregisterMetrics()
+}

discovery/kubernetes/pod.go

@@ -33,7 +33,10 @@ import (
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
-const nodeIndex = "node"
+const (
+	nodeIndex = "node"
+	podIndex  = "pod"
+)
 
 // Pod discovers new pod targets.
 type Pod struct {
@@ -326,7 +329,7 @@ func podSource(pod *apiv1.Pod) string {
 }
 
 func podSourceFromNamespaceAndName(namespace, name string) string {
-	return "pod/" + namespace + "/" + name
+	return "pod/" + namespacedName(namespace, name)
 }
 
 func podReady(pod *apiv1.Pod) model.LabelValue {

discovery/legacymanager/manager.go

@@ -42,7 +42,7 @@ type provider struct {
 }
 
 // NewManager is the Discovery Manager constructor.
-func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Registerer, options ...func(*Manager)) *Manager {
+func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Registerer, sdMetrics map[string]discovery.DiscovererMetrics, options ...func(*Manager)) *Manager {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -55,6 +55,7 @@ func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Re
 		updatert:       5 * time.Second,
 		triggerSend:    make(chan struct{}, 1),
 		registerer:     registerer,
+		sdMetrics:      sdMetrics,
 	}
 	for _, option := range options {
 		option(mgr)
@@ -62,7 +63,7 @@ func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Re
 
 	// Register the metrics.
 	// We have to do this after setting all options, so that the name of the Manager is set.
-	if metrics, err := discovery.NewMetrics(registerer, mgr.name); err == nil {
+	if metrics, err := discovery.NewManagerMetrics(registerer, mgr.name); err == nil {
 		mgr.metrics = metrics
 	} else {
 		level.Error(logger).Log("msg", "Failed to create discovery manager metrics", "manager", mgr.name, "err", err)
@@ -108,7 +109,8 @@ type Manager struct {
 	// A registerer for all service discovery metrics.
 	registerer prometheus.Registerer
 
-	metrics *discovery.Metrics
+	metrics   *discovery.Metrics
+	sdMetrics map[string]discovery.DiscovererMetrics
 }
 
 // Run starts the background processing.
@@ -283,8 +285,8 @@ func (m *Manager) registerProviders(cfgs discovery.Configs, setName string) int
 		}
 		typ := cfg.Name()
 		d, err := cfg.NewDiscoverer(discovery.DiscovererOptions{
-			Logger:     log.With(m.logger, "discovery", typ, "config", setName),
+			Logger:  log.With(m.logger, "discovery", typ, "config", setName),
-			Registerer: m.registerer,
+			Metrics: m.sdMetrics[typ],
 		})
 		if err != nil {
 			level.Error(m.logger).Log("msg", "Cannot create service discovery", "err", err, "type", typ, "config", setName)

discovery/legacymanager/manager_test.go

@@ -36,6 +36,13 @@ func TestMain(m *testing.M) {
 	testutil.TolerantVerifyLeak(m)
 }
 
+func newTestMetrics(t *testing.T, reg prometheus.Registerer) (*discovery.RefreshMetricsManager, map[string]discovery.DiscovererMetrics) {
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	sdMetrics, err := discovery.RegisterSDMetrics(reg, refreshMetrics)
+	require.NoError(t, err)
+	return &refreshMetrics, sdMetrics
+}
+
 // TestTargetUpdatesOrder checks that the target updates are received in the expected order.
 func TestTargetUpdatesOrder(t *testing.T) {
 	// The order by which the updates are send is determined by the interval passed to the mock discovery adapter
@@ -665,7 +672,10 @@ func TestTargetUpdatesOrder(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel()
 
-			discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+			reg := prometheus.NewRegistry()
+			_, sdMetrics := newTestMetrics(t, reg)
+
+			discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 			require.NotNil(t, discoveryManager)
 			discoveryManager.updatert = 100 * time.Millisecond
 
@@ -748,7 +758,11 @@ func verifyPresence(t *testing.T, tSets map[poolKey]map[string]*targetgroup.Grou
 func TestTargetSetRecreatesTargetGroupsEveryRun(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := newTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -777,7 +791,11 @@ func TestTargetSetRecreatesTargetGroupsEveryRun(t *testing.T) {
 func TestDiscovererConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := newTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -802,7 +820,11 @@ func TestDiscovererConfigs(t *testing.T) {
 func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := newTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -842,7 +864,11 @@ func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 func TestIdenticalConfigurationsAreCoalesced(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, nil, prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := newTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, nil, reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -874,7 +900,11 @@ func TestApplyConfigDoesNotModifyStaticTargets(t *testing.T) {
 	}
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := newTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -897,10 +927,19 @@ func (e errorConfig) NewDiscoverer(discovery.DiscovererOptions) (discovery.Disco
 	return nil, e.err
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (errorConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &discovery.NoopDiscovererMetrics{}
+}
+
 func TestGaugeFailedConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := newTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -1057,7 +1096,10 @@ func TestCoordinationWithReceiver(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel()
 
-			mgr := NewManager(ctx, nil, prometheus.NewRegistry())
+			reg := prometheus.NewRegistry()
+			_, sdMetrics := newTestMetrics(t, reg)
+
+			mgr := NewManager(ctx, nil, reg, sdMetrics)
 			require.NotNil(t, mgr)
 			mgr.updatert = updateDelay
 			go mgr.Run()

discovery/linode/linode.go

@@ -87,12 +87,17 @@ type SDConfig struct {
 	TagSeparator    string         `yaml:"tag_separator,omitempty"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return newDiscovererMetrics(reg, rmi)
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "linode" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.Registerer)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -122,22 +127,23 @@ type Discovery struct {
 	pollCount            int
 	lastResults          []*targetgroup.Group
 	eventPollingEnabled  bool
-	failuresCount        prometheus.Counter
+	metrics              *linodeMetrics
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*linodeMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	d := &Discovery{
 		port:                 conf.Port,
 		tagSeparator:         conf.TagSeparator,
 		pollCount:            0,
 		lastRefreshTimestamp: time.Now().UTC(),
 		eventPollingEnabled:  true,
-		failuresCount: prometheus.NewCounter(
+		metrics:              m,
-			prometheus.CounterOpts{
-				Name: "prometheus_sd_linode_failures_total",
-				Help: "Number of Linode service discovery refresh failures.",
-			}),
 	}
 
 	rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "linode_sd")
@@ -156,12 +162,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer)
 
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
+			Logger:              logger,
-			Mech:     "linode",
+			Mech:                "linode",
-			Interval: time.Duration(conf.RefreshInterval),
+			Interval:            time.Duration(conf.RefreshInterval),
-			RefreshF: d.refresh,
+			RefreshF:            d.refresh,
-			Registry: reg,
+			MetricsInstantiator: m.refreshMetrics,
-			Metrics:  []prometheus.Collector{d.failuresCount},
 		},
 	)
 	return d, nil
@@ -223,14 +228,14 @@ func (d *Discovery) refreshData(ctx context.Context) ([]*targetgroup.Group, erro
 	// Gather all linode instances.
 	instances, err := d.client.ListInstances(ctx, &linodego.ListOptions{PageSize: 500})
 	if err != nil {
-		d.failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, err
 	}
 
 	// Gather detailed IP address info for all IPs on all linode instances.
 	detailedIPs, err := d.client.ListIPAddresses(ctx, &linodego.ListOptions{PageSize: 500})
 	if err != nil {
-		d.failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, err
 	}
 

discovery/linode/linode_test.go

@@ -24,6 +24,8 @@ import (
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/prometheus/discovery"
 )
 
 type LinodeSDTestSuite struct {
@@ -53,7 +55,15 @@ func TestLinodeSDRefresh(t *testing.T) {
 		Credentials: tokenID,
 		Type:        "Bearer",
 	}
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 	endpoint, err := url.Parse(sdmock.Mock.Endpoint())
 	require.NoError(t, err)

discovery/linode/metrics.go

@@ -0,0 +1,57 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package linode
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*linodeMetrics)(nil)
+
+type linodeMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+
+	failuresCount prometheus.Counter
+
+	metricRegisterer discovery.MetricRegisterer
+}
+
+func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	m := &linodeMetrics{
+		refreshMetrics: rmi,
+		failuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_linode_failures_total",
+				Help: "Number of Linode service discovery refresh failures.",
+			}),
+	}
+
+	m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
+		m.failuresCount,
+	})
+
+	return m
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *linodeMetrics) Register() error {
+	return m.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *linodeMetrics) Unregister() {
+	m.metricRegisterer.UnregisterMetrics()
+}

discovery/manager.go

@@ -64,8 +64,24 @@ func (p *Provider) Config() interface{} {
 	return p.config
 }
 
+// Registers the metrics needed for SD mechanisms.
+// Does not register the metrics for the Discovery Manager.
+// TODO(ptodev): Add ability to unregister the metrics?
+func CreateAndRegisterSDMetrics(reg prometheus.Registerer) (map[string]DiscovererMetrics, error) {
+	// Some SD mechanisms use the "refresh" package, which has its own metrics.
+	refreshSdMetrics := NewRefreshMetrics(reg)
+
+	// Register the metrics specific for each SD mechanism, and the ones for the refresh package.
+	sdMetrics, err := RegisterSDMetrics(reg, refreshSdMetrics)
+	if err != nil {
+		return nil, fmt.Errorf("failed to register service discovery metrics: %w", err)
+	}
+
+	return sdMetrics, nil
+}
+
 // NewManager is the Discovery Manager constructor.
-func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Registerer, options ...func(*Manager)) *Manager {
+func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Registerer, sdMetrics map[string]DiscovererMetrics, options ...func(*Manager)) *Manager {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -77,6 +93,7 @@ func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Re
 		updatert:    5 * time.Second,
 		triggerSend: make(chan struct{}, 1),
 		registerer:  registerer,
+		sdMetrics:   sdMetrics,
 	}
 	for _, option := range options {
 		option(mgr)
@@ -84,7 +101,7 @@ func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Re
 
 	// Register the metrics.
 	// We have to do this after setting all options, so that the name of the Manager is set.
-	if metrics, err := NewMetrics(registerer, mgr.name); err == nil {
+	if metrics, err := NewManagerMetrics(registerer, mgr.name); err == nil {
 		mgr.metrics = metrics
 	} else {
 		level.Error(logger).Log("msg", "Failed to create discovery manager metrics", "manager", mgr.name, "err", err)
@@ -143,7 +160,8 @@ type Manager struct {
 	// A registerer for all service discovery metrics.
 	registerer prometheus.Registerer
 
-	metrics *Metrics
+	metrics   *Metrics
+	sdMetrics map[string]DiscovererMetrics
 }
 
 // Providers returns the currently configured SD providers.
@@ -402,7 +420,7 @@ func (m *Manager) registerProviders(cfgs Configs, setName string) int {
 		d, err := cfg.NewDiscoverer(DiscovererOptions{
 			Logger:            log.With(m.logger, "discovery", typ, "config", setName),
 			HTTPClientOptions: m.httpOpts,
-			Registerer:        m.registerer,
+			Metrics:           m.sdMetrics[typ],
 		})
 		if err != nil {
 			level.Error(m.logger).Log("msg", "Cannot create service discovery", "err", err, "type", typ, "config", setName)

discovery/manager_test.go

@@ -36,6 +36,13 @@ func TestMain(m *testing.M) {
 	testutil.TolerantVerifyLeak(m)
 }
 
+func NewTestMetrics(t *testing.T, reg prometheus.Registerer) (*RefreshMetricsManager, map[string]DiscovererMetrics) {
+	refreshMetrics := NewRefreshMetrics(reg)
+	sdMetrics, err := RegisterSDMetrics(reg, refreshMetrics)
+	require.NoError(t, err)
+	return &refreshMetrics, sdMetrics
+}
+
 // TestTargetUpdatesOrder checks that the target updates are received in the expected order.
 func TestTargetUpdatesOrder(t *testing.T) {
 	// The order by which the updates are send is determined by the interval passed to the mock discovery adapter
@@ -665,7 +672,10 @@ func TestTargetUpdatesOrder(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel()
 
-			discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+			reg := prometheus.NewRegistry()
+			_, sdMetrics := NewTestMetrics(t, reg)
+
+			discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 			require.NotNil(t, discoveryManager)
 			discoveryManager.updatert = 100 * time.Millisecond
 
@@ -684,7 +694,7 @@ func TestTargetUpdatesOrder(t *testing.T) {
 			for x := 0; x < totalUpdatesCount; x++ {
 				select {
 				case <-ctx.Done():
-					t.Fatalf("%d: no update arrived within the timeout limit", x)
+					require.FailNow(t, "%d: no update arrived within the timeout limit", x)
 				case tgs := <-provUpdates:
 					discoveryManager.updateGroup(poolKey{setName: strconv.Itoa(i), provider: tc.title}, tgs)
 					for _, got := range discoveryManager.allGroups() {
@@ -746,10 +756,8 @@ func verifySyncedPresence(t *testing.T, tGroups map[string][]*targetgroup.Group,
 
 func verifyPresence(t *testing.T, tSets map[poolKey]map[string]*targetgroup.Group, poolKey poolKey, label string, present bool) {
 	t.Helper()
-	if _, ok := tSets[poolKey]; !ok {
+	_, ok := tSets[poolKey]
-		t.Fatalf("'%s' should be present in Pool keys: %v", poolKey, tSets)
+	require.True(t, ok, "'%s' should be present in Pool keys: %v", poolKey, tSets)
-		return
-	}
 
 	match := false
 	var mergedTargets string
@@ -766,7 +774,7 @@ func verifyPresence(t *testing.T, tSets map[poolKey]map[string]*targetgroup.Grou
 		if !present {
 			msg = "not"
 		}
-		t.Fatalf("%q should %s be present in Targets labels: %q", label, msg, mergedTargets)
+		require.FailNow(t, "%q should %s be present in Targets labels: %q", label, msg, mergedTargets)
 	}
 }
 
@@ -780,7 +788,11 @@ func pk(provider, setName string, n int) poolKey {
 func TestTargetSetTargetGroupsPresentOnConfigReload(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -813,7 +825,11 @@ func TestTargetSetTargetGroupsPresentOnConfigReload(t *testing.T) {
 func TestTargetSetTargetGroupsPresentOnConfigRename(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -849,7 +865,11 @@ func TestTargetSetTargetGroupsPresentOnConfigRename(t *testing.T) {
 func TestTargetSetTargetGroupsPresentOnConfigDuplicateAndDeleteOriginal(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -888,7 +908,11 @@ func TestTargetSetTargetGroupsPresentOnConfigDuplicateAndDeleteOriginal(t *testi
 func TestTargetSetTargetGroupsPresentOnConfigChange(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -950,7 +974,11 @@ func TestTargetSetTargetGroupsPresentOnConfigChange(t *testing.T) {
 func TestTargetSetRecreatesTargetGroupsOnConfigChange(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -990,7 +1018,11 @@ func TestTargetSetRecreatesTargetGroupsOnConfigChange(t *testing.T) {
 func TestDiscovererConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -1023,7 +1055,11 @@ func TestDiscovererConfigs(t *testing.T) {
 func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -1050,28 +1086,24 @@ func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 	syncedTargets = <-discoveryManager.SyncCh()
 	p = pk("static", "prometheus", 1)
 	targetGroups, ok := discoveryManager.targets[p]
-	if !ok {
+	require.True(t, ok, "'%v' should be present in target groups", p)
-		t.Fatalf("'%v' should be present in target groups", p)
-	}
 	group, ok := targetGroups[""]
-	if !ok {
+	require.True(t, ok, "missing '' key in target groups %v", targetGroups)
-		t.Fatalf("missing '' key in target groups %v", targetGroups)
-	}
 
-	if len(group.Targets) != 0 {
+	require.Empty(t, group.Targets, "Invalid number of targets.")
-		t.Fatalf("Invalid number of targets: expected 0, got %d", len(group.Targets))
-	}
 	require.Len(t, syncedTargets, 1)
 	require.Len(t, syncedTargets["prometheus"], 1)
-	if lbls := syncedTargets["prometheus"][0].Labels; lbls != nil {
+	require.Nil(t, syncedTargets["prometheus"][0].Labels)
-		t.Fatalf("Unexpected Group: expected nil Labels, got %v", lbls)
-	}
 }
 
 func TestIdenticalConfigurationsAreCoalesced(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, nil, prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, nil, reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -1089,9 +1121,7 @@ func TestIdenticalConfigurationsAreCoalesced(t *testing.T) {
 	syncedTargets := <-discoveryManager.SyncCh()
 	verifyPresence(t, discoveryManager.targets, pk("static", "prometheus", 0), "{__address__=\"foo:9090\"}", true)
 	verifyPresence(t, discoveryManager.targets, pk("static", "prometheus2", 0), "{__address__=\"foo:9090\"}", true)
-	if len(discoveryManager.providers) != 1 {
+	require.Len(t, discoveryManager.providers, 1, "Invalid number of providers.")
-		t.Fatalf("Invalid number of providers: expected 1, got %d", len(discoveryManager.providers))
-	}
 	require.Len(t, syncedTargets, 2)
 	verifySyncedPresence(t, syncedTargets, "prometheus", "{__address__=\"foo:9090\"}", true)
 	require.Len(t, syncedTargets["prometheus"], 1)
@@ -1108,7 +1138,11 @@ func TestApplyConfigDoesNotModifyStaticTargets(t *testing.T) {
 	}
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -1129,11 +1163,21 @@ type errorConfig struct{ err error }
 func (e errorConfig) Name() string                                        { return "error" }
 func (e errorConfig) NewDiscoverer(DiscovererOptions) (Discoverer, error) { return nil, e.err }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (errorConfig) NewDiscovererMetrics(prometheus.Registerer, RefreshMetricsInstantiator) DiscovererMetrics {
+	return &NoopDiscovererMetrics{}
+}
+
 type lockStaticConfig struct {
 	mu     *sync.Mutex
 	config StaticConfig
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (lockStaticConfig) NewDiscovererMetrics(prometheus.Registerer, RefreshMetricsInstantiator) DiscovererMetrics {
+	return &NoopDiscovererMetrics{}
+}
+
 func (s lockStaticConfig) Name() string { return "lockstatic" }
 func (s lockStaticConfig) NewDiscoverer(DiscovererOptions) (Discoverer, error) {
 	return (lockStaticDiscoverer)(s), nil
@@ -1155,7 +1199,11 @@ func (s lockStaticDiscoverer) Run(ctx context.Context, up chan<- []*targetgroup.
 func TestGaugeFailedConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -1171,9 +1219,7 @@ func TestGaugeFailedConfigs(t *testing.T) {
 	<-discoveryManager.SyncCh()
 
 	failedCount := client_testutil.ToFloat64(discoveryManager.metrics.FailedConfigs)
-	if failedCount != 3 {
+	require.Equal(t, 3.0, failedCount, "Expected to have 3 failed configs.")
-		t.Fatalf("Expected to have 3 failed configs, got: %v", failedCount)
-	}
 
 	c["prometheus"] = Configs{
 		staticConfig("foo:9090"),
@@ -1182,9 +1228,7 @@ func TestGaugeFailedConfigs(t *testing.T) {
 	<-discoveryManager.SyncCh()
 
 	failedCount = client_testutil.ToFloat64(discoveryManager.metrics.FailedConfigs)
-	if failedCount != 0 {
+	require.Equal(t, 0.0, failedCount, "Expected to get no failed config.")
-		t.Fatalf("Expected to get no failed config, got: %v", failedCount)
-	}
 }
 
 func TestCoordinationWithReceiver(t *testing.T) {
@@ -1312,7 +1356,10 @@ func TestCoordinationWithReceiver(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel()
 
-			mgr := NewManager(ctx, nil, prometheus.NewRegistry())
+			reg := prometheus.NewRegistry()
+			_, sdMetrics := NewTestMetrics(t, reg)
+
+			mgr := NewManager(ctx, nil, reg, sdMetrics)
 			require.NotNil(t, mgr)
 			mgr.updatert = updateDelay
 			go mgr.Run()
@@ -1325,19 +1372,14 @@ func TestCoordinationWithReceiver(t *testing.T) {
 				time.Sleep(expected.delay)
 				select {
 				case <-ctx.Done():
-					t.Fatalf("step %d: no update received in the expected timeframe", i)
+					require.FailNow(t, "step %d: no update received in the expected timeframe", i)
 				case tgs, ok := <-mgr.SyncCh():
-					if !ok {
+					require.True(t, ok, "step %d: discovery manager channel is closed", i)
-						t.Fatalf("step %d: discovery manager channel is closed", i)
+					require.Equal(t, len(expected.tgs), len(tgs), "step %d: targets mismatch", i)
-					}
+
-					if len(tgs) != len(expected.tgs) {
-						t.Fatalf("step %d: target groups mismatch, got: %d, expected: %d\ngot: %#v\nexpected: %#v",
-							i, len(tgs), len(expected.tgs), tgs, expected.tgs)
-					}
 					for k := range expected.tgs {
-						if _, ok := tgs[k]; !ok {
+						_, ok := tgs[k]
-							t.Fatalf("step %d: target group not found: %s\ngot: %#v", i, k, tgs)
+						require.True(t, ok, "step %d: target group not found: %s", i, k)
-						}
 						assertEqualGroups(t, tgs[k], expected.tgs[k])
 					}
 				}
@@ -1408,7 +1450,11 @@ func (o onceProvider) Run(_ context.Context, ch chan<- []*targetgroup.Group) {
 func TestTargetSetTargetGroupsUpdateDuringApplyConfig(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -1470,6 +1516,11 @@ func newTestDiscoverer() *testDiscoverer {
 	}
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*testDiscoverer) NewDiscovererMetrics(prometheus.Registerer, RefreshMetricsInstantiator) DiscovererMetrics {
+	return &NoopDiscovererMetrics{}
+}
+
 // Name implements Config.
 func (t *testDiscoverer) Name() string {
 	return "test"

discovery/marathon/marathon.go

@@ -79,12 +79,19 @@ type SDConfig struct {
 	HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &marathonMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "marathon" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(*c, opts.Logger, opts.Registerer)
+	return NewDiscovery(*c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -133,7 +140,12 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Marathon Discovery.
-func NewDiscovery(conf SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(conf SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*marathonMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "marathon_sd")
 	if err != nil {
 		return nil, err
@@ -156,11 +168,11 @@ func NewDiscovery(conf SDConfig, logger log.Logger, reg prometheus.Registerer) (
 	}
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
+			Logger:              logger,
-			Mech:     "marathon",
+			Mech:                "marathon",
-			Interval: time.Duration(conf.RefreshInterval),
+			Interval:            time.Duration(conf.RefreshInterval),
-			RefreshF: d.refresh,
+			RefreshF:            d.refresh,
-			Registry: reg,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
 	return d, nil

discovery/marathon/marathon_test.go

@@ -23,7 +23,9 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/require"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -37,7 +39,19 @@ func testConfig() SDConfig {
 }
 
 func testUpdateServices(client appListClient) ([]*targetgroup.Group, error) {
-	md, err := NewDiscovery(testConfig(), nil, prometheus.NewRegistry())
+	cfg := testConfig()
+
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	err := metrics.Register()
+	if err != nil {
+		return nil, err
+	}
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	md, err := NewDiscovery(cfg, nil, metrics)
 	if err != nil {
 		return nil, err
 	}
@@ -55,23 +69,15 @@ func TestMarathonSDHandleError(t *testing.T) {
 		}
 	)
 	tgs, err := testUpdateServices(client)
-	if !errors.Is(err, errTesting) {
+	require.ErrorIs(t, err, errTesting)
-		t.Fatalf("Expected error: %s", err)
+	require.Empty(t, tgs, "Expected no target groups.")
-	}
-	if len(tgs) != 0 {
-		t.Fatalf("Got group: %s", tgs)
-	}
 }
 
 func TestMarathonSDEmptyList(t *testing.T) {
 	client := func(_ context.Context, _ *http.Client, _ string) (*appList, error) { return &appList{}, nil }
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Empty(t, tgs, "Expected no target groups.")
-	}
-	if len(tgs) > 0 {
-		t.Fatalf("Got group: %v", tgs)
-	}
 }
 
 func marathonTestAppList(labels map[string]string, runningTasks int) *appList {
@@ -105,66 +111,49 @@ func TestMarathonSDSendGroup(t *testing.T) {
 		return marathonTestAppList(marathonValidLabel, 1), nil
 	}
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 target group.")
-	}
-	if len(tgs) != 1 {
-		t.Fatal("Expected 1 target group, got", len(tgs))
-	}
 
 	tg := tgs[0]
+	require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
+	require.Len(t, tg.Targets, 1, "Expected 1 target.")
 
-	if tg.Source != "test-service" {
-		t.Fatalf("Wrong target group name: %s", tg.Source)
-	}
-	if len(tg.Targets) != 1 {
-		t.Fatalf("Wrong number of targets: %v", tg.Targets)
-	}
 	tgt := tg.Targets[0]
-	if tgt[model.AddressLabel] != "mesos-slave1:31000" {
+	require.Equal(t, "mesos-slave1:31000", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "yes", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.")
-	}
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" {
-		t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
-	}
 }
 
 func TestMarathonSDRemoveApp(t *testing.T) {
-	md, err := NewDiscovery(testConfig(), nil, prometheus.NewRegistry())
+	cfg := testConfig()
-	if err != nil {
+	reg := prometheus.NewRegistry()
-		t.Fatalf("%s", err)
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
-	}
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	md, err := NewDiscovery(cfg, nil, metrics)
+	require.NoError(t, err)
 
 	md.appsClient = func(_ context.Context, _ *http.Client, _ string) (*appList, error) {
 		return marathonTestAppList(marathonValidLabel, 1), nil
 	}
 	tgs, err := md.refresh(context.Background())
-	if err != nil {
+	require.NoError(t, err, "Got error on first update.")
-		t.Fatalf("Got error on first update: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 targetgroup.")
-	}
-	if len(tgs) != 1 {
-		t.Fatal("Expected 1 targetgroup, got", len(tgs))
-	}
 	tg1 := tgs[0]
 
 	md.appsClient = func(_ context.Context, _ *http.Client, _ string) (*appList, error) {
 		return marathonTestAppList(marathonValidLabel, 0), nil
 	}
 	tgs, err = md.refresh(context.Background())
-	if err != nil {
+	require.NoError(t, err, "Got error on second update.")
-		t.Fatalf("Got error on second update: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 targetgroup.")
-	}
+
-	if len(tgs) != 1 {
-		t.Fatal("Expected 1 targetgroup, got", len(tgs))
-	}
 	tg2 := tgs[0]
 
-	if tg2.Source != tg1.Source {
+	require.NotEmpty(t, tg2.Targets, "Got a non-empty target set.")
-		if len(tg2.Targets) > 0 {
+	require.Equal(t, tg1.Source, tg2.Source, "Source is different.")
-			t.Errorf("Got a non-empty target set: %s", tg2.Targets)
-		}
-		t.Fatalf("Source is different: %s != %s", tg1.Source, tg2.Source)
-	}
 }
 
 func marathonTestAppListWithMultiplePorts(labels map[string]string, runningTasks int) *appList {
@@ -199,34 +188,22 @@ func TestMarathonSDSendGroupWithMultiplePort(t *testing.T) {
 		return marathonTestAppListWithMultiplePorts(marathonValidLabel, 1), nil
 	}
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 target group.")
-	}
+
-	if len(tgs) != 1 {
+	tg := tgs[0]
-		t.Fatal("Expected 1 target group, got", len(tgs))
+	require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
-	}
+	require.Len(t, tg.Targets, 2, "Wrong number of targets.")
-	tg := tgs[0]
 
-	if tg.Source != "test-service" {
-		t.Fatalf("Wrong target group name: %s", tg.Source)
-	}
-	if len(tg.Targets) != 2 {
-		t.Fatalf("Wrong number of targets: %v", tg.Targets)
-	}
 	tgt := tg.Targets[0]
-	if tgt[model.AddressLabel] != "mesos-slave1:31000" {
+	require.Equal(t, "mesos-slave1:31000", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "yes", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]),
-	}
+		"Wrong portMappings label from the first port: %s", tgt[model.AddressLabel])
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" {
+
-		t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
-	}
 	tgt = tg.Targets[1]
-	if tgt[model.AddressLabel] != "mesos-slave1:32000" {
+	require.Equal(t, "mesos-slave1:32000", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]),
-	}
+		"Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
-	}
 }
 
 func marathonTestZeroTaskPortAppList(labels map[string]string, runningTasks int) *appList {
@@ -256,20 +233,12 @@ func TestMarathonZeroTaskPorts(t *testing.T) {
 		return marathonTestZeroTaskPortAppList(marathonValidLabel, 1), nil
 	}
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 target group.")
-	}
-	if len(tgs) != 1 {
-		t.Fatal("Expected 1 target group, got", len(tgs))
-	}
-	tg := tgs[0]
 
-	if tg.Source != "test-service-zero-ports" {
+	tg := tgs[0]
-		t.Fatalf("Wrong target group name: %s", tg.Source)
+	require.Equal(t, "test-service-zero-ports", tg.Source, "Wrong target group name.")
-	}
+	require.Empty(t, tg.Targets, "Wrong number of targets.")
-	if len(tg.Targets) != 0 {
-		t.Fatalf("Wrong number of targets: %v", tg.Targets)
-	}
 }
 
 func Test500ErrorHttpResponseWithValidJSONBody(t *testing.T) {
@@ -284,9 +253,7 @@ func Test500ErrorHttpResponseWithValidJSONBody(t *testing.T) {
 	defer ts.Close()
 	// Execute test case and validate behavior.
 	_, err := testUpdateServices(nil)
-	if err == nil {
+	require.Error(t, err, "Expected error for 5xx HTTP response from marathon server.")
-		t.Fatalf("Expected error for 5xx HTTP response from marathon server, got nil")
-	}
 }
 
 func marathonTestAppListWithPortDefinitions(labels map[string]string, runningTasks int) *appList {
@@ -324,40 +291,24 @@ func TestMarathonSDSendGroupWithPortDefinitions(t *testing.T) {
 		return marathonTestAppListWithPortDefinitions(marathonValidLabel, 1), nil
 	}
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 target group.")
-	}
+
-	if len(tgs) != 1 {
+	tg := tgs[0]
-		t.Fatal("Expected 1 target group, got", len(tgs))
+	require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
-	}
+	require.Len(t, tg.Targets, 2, "Wrong number of targets.")
-	tg := tgs[0]
 
-	if tg.Source != "test-service" {
-		t.Fatalf("Wrong target group name: %s", tg.Source)
-	}
-	if len(tg.Targets) != 2 {
-		t.Fatalf("Wrong number of targets: %v", tg.Targets)
-	}
 	tgt := tg.Targets[0]
-	if tgt[model.AddressLabel] != "mesos-slave1:1234" {
+	require.Equal(t, "mesos-slave1:1234", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]),
-	}
+		"Wrong portMappings label from the first port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]),
-		t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
+		"Wrong portDefinitions label from the first port.")
-	}
+
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
-	}
 	tgt = tg.Targets[1]
-	if tgt[model.AddressLabel] != "mesos-slave1:5678" {
+	require.Equal(t, "mesos-slave1:5678", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Empty(t, tgt[model.LabelName(portMappingLabelPrefix+"prometheus")], "Wrong portMappings label from the second port.")
-	}
+	require.Equal(t, "yes", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "yes" {
-		t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
-	}
 }
 
 func marathonTestAppListWithPortDefinitionsRequirePorts(labels map[string]string, runningTasks int) *appList {
@@ -394,40 +345,22 @@ func TestMarathonSDSendGroupWithPortDefinitionsRequirePorts(t *testing.T) {
 		return marathonTestAppListWithPortDefinitionsRequirePorts(marathonValidLabel, 1), nil
 	}
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 target group.")
-	}
+
-	if len(tgs) != 1 {
+	tg := tgs[0]
-		t.Fatal("Expected 1 target group, got", len(tgs))
+	require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
-	}
+	require.Len(t, tg.Targets, 2, "Wrong number of targets.")
-	tg := tgs[0]
 
-	if tg.Source != "test-service" {
-		t.Fatalf("Wrong target group name: %s", tg.Source)
-	}
-	if len(tg.Targets) != 2 {
-		t.Fatalf("Wrong number of targets: %v", tg.Targets)
-	}
 	tgt := tg.Targets[0]
-	if tgt[model.AddressLabel] != "mesos-slave1:31000" {
+	require.Equal(t, "mesos-slave1:31000", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.")
-	}
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
+
-		t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
-	}
 	tgt = tg.Targets[1]
-	if tgt[model.AddressLabel] != "mesos-slave1:32000" {
+	require.Equal(t, "mesos-slave1:32000", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.")
-	}
+	require.Equal(t, "yes", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "yes" {
-		t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
-	}
 }
 
 func marathonTestAppListWithPorts(labels map[string]string, runningTasks int) *appList {
@@ -459,40 +392,22 @@ func TestMarathonSDSendGroupWithPorts(t *testing.T) {
 		return marathonTestAppListWithPorts(marathonValidLabel, 1), nil
 	}
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 target group.")
-	}
+
-	if len(tgs) != 1 {
+	tg := tgs[0]
-		t.Fatal("Expected 1 target group, got", len(tgs))
+	require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
-	}
+	require.Len(t, tg.Targets, 2, "Wrong number of targets.")
-	tg := tgs[0]
 
-	if tg.Source != "test-service" {
-		t.Fatalf("Wrong target group name: %s", tg.Source)
-	}
-	if len(tg.Targets) != 2 {
-		t.Fatalf("Wrong number of targets: %v", tg.Targets)
-	}
 	tgt := tg.Targets[0]
-	if tgt[model.AddressLabel] != "mesos-slave1:31000" {
+	require.Equal(t, "mesos-slave1:31000", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.")
-	}
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
+
-		t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
-	}
 	tgt = tg.Targets[1]
-	if tgt[model.AddressLabel] != "mesos-slave1:32000" {
+	require.Equal(t, "mesos-slave1:32000", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.")
-	}
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
-	}
 }
 
 func marathonTestAppListWithContainerPortMappings(labels map[string]string, runningTasks int) *appList {
@@ -533,40 +448,22 @@ func TestMarathonSDSendGroupWithContainerPortMappings(t *testing.T) {
 		return marathonTestAppListWithContainerPortMappings(marathonValidLabel, 1), nil
 	}
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 target group.")
-	}
+
-	if len(tgs) != 1 {
+	tg := tgs[0]
-		t.Fatal("Expected 1 target group, got", len(tgs))
+	require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
-	}
+	require.Len(t, tg.Targets, 2, "Wrong number of targets.")
-	tg := tgs[0]
 
-	if tg.Source != "test-service" {
-		t.Fatalf("Wrong target group name: %s", tg.Source)
-	}
-	if len(tg.Targets) != 2 {
-		t.Fatalf("Wrong number of targets: %v", tg.Targets)
-	}
 	tgt := tg.Targets[0]
-	if tgt[model.AddressLabel] != "mesos-slave1:12345" {
+	require.Equal(t, "mesos-slave1:12345", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "yes", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.")
-	}
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" {
+
-		t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
-	}
 	tgt = tg.Targets[1]
-	if tgt[model.AddressLabel] != "mesos-slave1:32000" {
+	require.Equal(t, "mesos-slave1:32000", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.")
-	}
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
-	}
 }
 
 func marathonTestAppListWithDockerContainerPortMappings(labels map[string]string, runningTasks int) *appList {
@@ -607,40 +504,22 @@ func TestMarathonSDSendGroupWithDockerContainerPortMappings(t *testing.T) {
 		return marathonTestAppListWithDockerContainerPortMappings(marathonValidLabel, 1), nil
 	}
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 target group.")
-	}
+
-	if len(tgs) != 1 {
+	tg := tgs[0]
-		t.Fatal("Expected 1 target group, got", len(tgs))
+	require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
-	}
+	require.Len(t, tg.Targets, 2, "Wrong number of targets.")
-	tg := tgs[0]
 
-	if tg.Source != "test-service" {
-		t.Fatalf("Wrong target group name: %s", tg.Source)
-	}
-	if len(tg.Targets) != 2 {
-		t.Fatalf("Wrong number of targets: %v", tg.Targets)
-	}
 	tgt := tg.Targets[0]
-	if tgt[model.AddressLabel] != "mesos-slave1:31000" {
+	require.Equal(t, "mesos-slave1:31000", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "yes", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.")
-	}
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" {
+
-		t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
-	}
 	tgt = tg.Targets[1]
-	if tgt[model.AddressLabel] != "mesos-slave1:12345" {
+	require.Equal(t, "mesos-slave1:12345", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.")
-	}
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
-	}
 }
 
 func marathonTestAppListWithContainerNetworkAndPortMappings(labels map[string]string, runningTasks int) *appList {
@@ -685,38 +564,20 @@ func TestMarathonSDSendGroupWithContainerNetworkAndPortMapping(t *testing.T) {
 		return marathonTestAppListWithContainerNetworkAndPortMappings(marathonValidLabel, 1), nil
 	}
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 target group.")
-	}
+
-	if len(tgs) != 1 {
+	tg := tgs[0]
-		t.Fatal("Expected 1 target group, got", len(tgs))
+	require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
-	}
+	require.Len(t, tg.Targets, 2, "Wrong number of targets.")
-	tg := tgs[0]
 
-	if tg.Source != "test-service" {
-		t.Fatalf("Wrong target group name: %s", tg.Source)
-	}
-	if len(tg.Targets) != 2 {
-		t.Fatalf("Wrong number of targets: %v", tg.Targets)
-	}
 	tgt := tg.Targets[0]
-	if tgt[model.AddressLabel] != "1.2.3.4:8080" {
+	require.Equal(t, "1.2.3.4:8080", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "yes", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.")
-	}
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" {
+
-		t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
-	}
 	tgt = tg.Targets[1]
-	if tgt[model.AddressLabel] != "1.2.3.4:1234" {
+	require.Equal(t, "1.2.3.4:1234", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.")
-	}
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
-	}
 }

discovery/marathon/metrics.go

@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package marathon
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*marathonMetrics)(nil)
+
+type marathonMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *marathonMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *marathonMetrics) Unregister() {}

discovery/metrics.go

@@ -38,7 +38,7 @@ type Metrics struct {
 	SentUpdates       prometheus.Counter
 }
 
-func NewMetrics(registerer prometheus.Registerer, sdManagerName string) (*Metrics, error) {
+func NewManagerMetrics(registerer prometheus.Registerer, sdManagerName string) (*Metrics, error) {
 	m := &Metrics{}
 
 	m.FailedConfigs = prometheus.NewGauge(

discovery/metrics_refresh.go

@@ -0,0 +1,75 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package discovery
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Metric vectors for the "refresh" package.
+// We define them here in the "discovery" package in order to avoid a cyclic dependency between
+// "discovery" and "refresh".
+type RefreshMetricsVecs struct {
+	failuresVec *prometheus.CounterVec
+	durationVec *prometheus.SummaryVec
+
+	metricRegisterer MetricRegisterer
+}
+
+var _ RefreshMetricsManager = (*RefreshMetricsVecs)(nil)
+
+func NewRefreshMetrics(reg prometheus.Registerer) RefreshMetricsManager {
+	m := &RefreshMetricsVecs{
+		failuresVec: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_refresh_failures_total",
+				Help: "Number of refresh failures for the given SD mechanism.",
+			},
+			[]string{"mechanism"}),
+		durationVec: prometheus.NewSummaryVec(
+			prometheus.SummaryOpts{
+				Name:       "prometheus_sd_refresh_duration_seconds",
+				Help:       "The duration of a refresh in seconds for the given SD mechanism.",
+				Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+			},
+			[]string{"mechanism"}),
+	}
+
+	// The reason we register metric vectors instead of metrics is so that
+	// the metrics are not visible until they are recorded.
+	m.metricRegisterer = NewMetricRegisterer(reg, []prometheus.Collector{
+		m.failuresVec,
+		m.durationVec,
+	})
+
+	return m
+}
+
+// Instantiate returns metrics out of metric vectors.
+func (m *RefreshMetricsVecs) Instantiate(mech string) *RefreshMetrics {
+	return &RefreshMetrics{
+		Failures: m.failuresVec.WithLabelValues(mech),
+		Duration: m.durationVec.WithLabelValues(mech),
+	}
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *RefreshMetricsVecs) Register() error {
+	return m.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *RefreshMetricsVecs) Unregister() {
+	m.metricRegisterer.UnregisterMetrics()
+}

discovery/moby/docker.go

@@ -22,7 +22,7 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
 	"github.com/go-kit/log"
@@ -76,12 +76,19 @@ type DockerSDConfig struct {
 	RefreshInterval model.Duration `yaml:"refresh_interval"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*DockerSDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &dockerMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Config.
 func (*DockerSDConfig) Name() string { return "docker" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *DockerSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDockerDiscovery(c, opts.Logger, opts.Registerer)
+	return NewDockerDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -115,8 +122,11 @@ type DockerDiscovery struct {
 }
 
 // NewDockerDiscovery returns a new DockerDiscovery which periodically refreshes its targets.
-func NewDockerDiscovery(conf *DockerSDConfig, logger log.Logger, reg prometheus.Registerer) (*DockerDiscovery, error) {
+func NewDockerDiscovery(conf *DockerSDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*DockerDiscovery, error) {
-	var err error
+	m, ok := metrics.(*dockerMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
 
 	d := &DockerDiscovery{
 		port:               conf.Port,
@@ -167,11 +177,11 @@ func NewDockerDiscovery(conf *DockerSDConfig, logger log.Logger, reg prometheus.
 
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
+			Logger:              logger,
-			Mech:     "docker",
+			Mech:                "docker",
-			Interval: time.Duration(conf.RefreshInterval),
+			Interval:            time.Duration(conf.RefreshInterval),
-			RefreshF: d.refresh,
+			RefreshF:            d.refresh,
-			Registry: reg,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
 	return d, nil
@@ -182,7 +192,7 @@ func (d *DockerDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, er
 		Source: "Docker",
 	}
 
-	containers, err := d.client.ContainerList(ctx, types.ContainerListOptions{Filters: d.filters})
+	containers, err := d.client.ContainerList(ctx, container.ListOptions{Filters: d.filters})
 	if err != nil {
 		return nil, fmt.Errorf("error while listing containers: %w", err)
 	}

discovery/moby/docker_test.go

@@ -23,6 +23,8 @@ import (
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
+
+	"github.com/prometheus/prometheus/discovery"
 )
 
 func TestDockerSDRefresh(t *testing.T) {
@@ -38,7 +40,14 @@ host: %s
 	var cfg DockerSDConfig
 	require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg))
 
-	d, err := NewDockerDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	d, err := NewDockerDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()

discovery/moby/dockerswarm.go

@@ -70,12 +70,19 @@ type Filter struct {
 	Values []string `yaml:"values"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*DockerSwarmSDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &dockerswarmMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Config.
 func (*DockerSwarmSDConfig) Name() string { return "dockerswarm" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *DockerSwarmSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.Registerer)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -118,8 +125,11 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *DockerSwarmSDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(conf *DockerSwarmSDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
-	var err error
+	m, ok := metrics.(*dockerswarmMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
 
 	d := &Discovery{
 		port: conf.Port,
@@ -170,11 +180,11 @@ func NewDiscovery(conf *DockerSwarmSDConfig, logger log.Logger, reg prometheus.R
 
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
+			Logger:              logger,
-			Mech:     "dockerswarm",
+			Mech:                "dockerswarm",
-			Interval: time.Duration(conf.RefreshInterval),
+			Interval:            time.Duration(conf.RefreshInterval),
-			RefreshF: d.refresh,
+			RefreshF:            d.refresh,
-			Registry: reg,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
 	return d, nil

discovery/moby/metrics_docker.go

@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package moby
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*dockerMetrics)(nil)
+
+type dockerMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *dockerMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *dockerMetrics) Unregister() {}

discovery/moby/metrics_dockerswarm.go

@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package moby
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*dockerswarmMetrics)(nil)
+
+type dockerswarmMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *dockerswarmMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *dockerswarmMetrics) Unregister() {}

discovery/moby/nodes_test.go

@@ -23,6 +23,8 @@ import (
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
+
+	"github.com/prometheus/prometheus/discovery"
 )
 
 func TestDockerSwarmNodesSDRefresh(t *testing.T) {
@@ -39,7 +41,14 @@ host: %s
 	var cfg DockerSwarmSDConfig
 	require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg))
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()

discovery/moby/services_test.go

@@ -23,6 +23,8 @@ import (
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
+
+	"github.com/prometheus/prometheus/discovery"
 )
 
 func TestDockerSwarmSDServicesRefresh(t *testing.T) {
@@ -39,7 +41,14 @@ host: %s
 	var cfg DockerSwarmSDConfig
 	require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg))
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()
@@ -333,7 +342,14 @@ filters:
 	var cfg DockerSwarmSDConfig
 	require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg))
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()

discovery/moby/tasks_test.go

@@ -23,6 +23,8 @@ import (
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
+
+	"github.com/prometheus/prometheus/discovery"
 )
 
 func TestDockerSwarmTasksSDRefresh(t *testing.T) {
@@ -39,7 +41,14 @@ host: %s
 	var cfg DockerSwarmSDConfig
 	require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg))
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()

discovery/nomad/metrics.go

@@ -0,0 +1,57 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nomad
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*nomadMetrics)(nil)
+
+type nomadMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+
+	failuresCount prometheus.Counter
+
+	metricRegisterer discovery.MetricRegisterer
+}
+
+func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	m := &nomadMetrics{
+		refreshMetrics: rmi,
+		failuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_nomad_failures_total",
+				Help: "Number of nomad service discovery refresh failures.",
+			}),
+	}
+
+	m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
+		m.failuresCount,
+	})
+
+	return m
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *nomadMetrics) Register() error {
+	return m.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *nomadMetrics) Unregister() {
+	m.metricRegisterer.UnregisterMetrics()
+}

discovery/nomad/nomad.go

@@ -74,12 +74,17 @@ type SDConfig struct {
 	TagSeparator     string                  `yaml:"tag_separator,omitempty"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return newDiscovererMetrics(reg, rmi)
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "nomad" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.Registerer)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -112,11 +117,16 @@ type Discovery struct {
 	region          string
 	server          string
 	tagSeparator    string
-	failuresCount   prometheus.Counter
+	metrics         *nomadMetrics
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*nomadMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	d := &Discovery{
 		allowStale:      conf.AllowStale,
 		namespace:       conf.Namespace,
@@ -124,11 +134,7 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer)
 		region:          conf.Region,
 		server:          conf.Server,
 		tagSeparator:    conf.TagSeparator,
-		failuresCount: prometheus.NewCounter(
+		metrics:         m,
-			prometheus.CounterOpts{
-				Name: "prometheus_sd_nomad_failures_total",
-				Help: "Number of nomad service discovery refresh failures.",
-			}),
 	}
 
 	HTTPClient, err := config.NewClientFromConfig(conf.HTTPClientConfig, "nomad_sd")
@@ -151,12 +157,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer)
 
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
+			Logger:              logger,
-			Mech:     "nomad",
+			Mech:                "nomad",
-			Interval: time.Duration(conf.RefreshInterval),
+			Interval:            time.Duration(conf.RefreshInterval),
-			RefreshF: d.refresh,
+			RefreshF:            d.refresh,
-			Registry: reg,
+			MetricsInstantiator: m.refreshMetrics,
-			Metrics:  []prometheus.Collector{d.failuresCount},
 		},
 	)
 	return d, nil
@@ -168,7 +173,7 @@ func (d *Discovery) refresh(context.Context) ([]*targetgroup.Group, error) {
 	}
 	stubs, _, err := d.client.Services().List(opts)
 	if err != nil {
-		d.failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, err
 	}
 
@@ -180,7 +185,7 @@ func (d *Discovery) refresh(context.Context) ([]*targetgroup.Group, error) {
 		for _, service := range stub.Services {
 			instances, _, err := d.client.Services().Get(service.ServiceName, opts)
 			if err != nil {
-				d.failuresCount.Inc()
+				d.metrics.failuresCount.Inc()
 				return nil, fmt.Errorf("failed to fetch services: %w", err)
 			}
 

discovery/nomad/nomad_test.go

@@ -25,6 +25,8 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/prometheus/discovery"
 )
 
 type NomadSDTestSuite struct {
@@ -128,8 +130,16 @@ func TestConfiguredService(t *testing.T) {
 	conf := &SDConfig{
 		Server: "http://localhost:4646",
 	}
-	_, err := NewDiscovery(conf, nil, prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := conf.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+
+	_, err := NewDiscovery(conf, nil, metrics)
 	require.NoError(t, err)
+
+	metrics.Unregister()
 }
 
 func TestNomadSDRefresh(t *testing.T) {
@@ -142,7 +152,15 @@ func TestNomadSDRefresh(t *testing.T) {
 
 	cfg := DefaultSDConfig
 	cfg.Server = endpoint.String()
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 
 	tgs, err := d.refresh(context.Background())

discovery/openstack/metrics.go

@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package openstack
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+type openstackMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+var _ discovery.DiscovererMetrics = (*openstackMetrics)(nil)
+
+// Register implements discovery.DiscovererMetrics.
+func (m *openstackMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *openstackMetrics) Unregister() {}

discovery/openstack/mock_test.go

@@ -18,6 +18,8 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"testing"
+
+	"github.com/stretchr/testify/require"
 )
 
 // SDMock is the interface for the OpenStack mock.
@@ -49,15 +51,13 @@ func (m *SDMock) Setup() {
 const tokenID = "cbc36478b0bd8e67e89469c7749d4127"
 
 func testMethod(t *testing.T, r *http.Request, expected string) {
-	if expected != r.Method {
+	require.Equal(t, expected, r.Method, "Unexpected request method.")
-		t.Errorf("Request method = %v, expected %v", r.Method, expected)
-	}
 }
 
 func testHeader(t *testing.T, r *http.Request, header, expected string) {
-	if actual := r.Header.Get(header); expected != actual {
+	t.Helper()
-		t.Errorf("Header %s = %s, expected %s", header, actual, expected)
+	actual := r.Header.Get(header)
-	}
+	require.Equal(t, expected, actual, "Unexpected value for request header %s.", header)
 }
 
 // HandleVersionsSuccessfully mocks version call.

discovery/openstack/openstack.go

@@ -66,12 +66,19 @@ type SDConfig struct {
 	Availability                string           `yaml:"availability,omitempty"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &openstackMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "openstack" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.Registerer)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -135,18 +142,23 @@ type refresher interface {
 }
 
 // NewDiscovery returns a new OpenStack Discoverer which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, l log.Logger, reg prometheus.Registerer) (*refresh.Discovery, error) {
+func NewDiscovery(conf *SDConfig, l log.Logger, metrics discovery.DiscovererMetrics) (*refresh.Discovery, error) {
+	m, ok := metrics.(*openstackMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	r, err := newRefresher(conf, l)
 	if err != nil {
 		return nil, err
 	}
 	return refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   l,
+			Logger:              l,
-			Mech:     "openstack",
+			Mech:                "openstack",
-			Interval: time.Duration(conf.RefreshInterval),
+			Interval:            time.Duration(conf.RefreshInterval),
-			RefreshF: r.refresh,
+			RefreshF:            r.refresh,
-			Registry: reg,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	), nil
 }

discovery/ovhcloud/metrics.go

@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ovhcloud
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*ovhcloudMetrics)(nil)
+
+type ovhcloudMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *ovhcloudMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *ovhcloudMetrics) Unregister() {}

discovery/ovhcloud/ovhcloud.go

@@ -53,6 +53,13 @@ type SDConfig struct {
 	Service           string         `yaml:"service"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &ovhcloudMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name implements the Discoverer interface.
 func (c SDConfig) Name() string {
 	return "ovhcloud"
@@ -94,7 +101,7 @@ func createClient(config *SDConfig) (*ovh.Client, error) {
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(options discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, options.Logger, options.Registerer)
+	return NewDiscovery(c, options.Logger, options.Metrics)
 }
 
 func init() {
@@ -141,7 +148,12 @@ func newRefresher(conf *SDConfig, logger log.Logger) (refresher, error) {
 }
 
 // NewDiscovery returns a new OVHcloud Discoverer which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*refresh.Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*refresh.Discovery, error) {
+	m, ok := metrics.(*ovhcloudMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	r, err := newRefresher(conf, logger)
 	if err != nil {
 		return nil, err
@@ -149,11 +161,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer)
 
 	return refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
+			Logger:              logger,
-			Mech:     "ovhcloud",
+			Mech:                "ovhcloud",
-			Interval: time.Duration(conf.RefreshInterval),
+			Interval:            time.Duration(conf.RefreshInterval),
-			RefreshF: r.refresh,
+			RefreshF:            r.refresh,
-			Registry: reg,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	), nil
 }

discovery/ovhcloud/ovhcloud_test.go

@@ -122,9 +122,17 @@ func TestParseIPs(t *testing.T) {
 func TestDiscoverer(t *testing.T) {
 	conf, _ := getMockConf("vps")
 	logger := testutil.NewLogger(t)
+
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := conf.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
 	_, err := conf.NewDiscoverer(discovery.DiscovererOptions{
-		Logger:     logger,
+		Logger:  logger,
-		Registerer: prometheus.NewRegistry(),
+		Metrics: metrics,
 	})
 
 	require.NoError(t, err)

discovery/puppetdb/metrics.go

@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package puppetdb
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*puppetdbMetrics)(nil)
+
+type puppetdbMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *puppetdbMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *puppetdbMetrics) Unregister() {}

discovery/puppetdb/puppetdb.go

@@ -79,12 +79,19 @@ type SDConfig struct {
 	Port              int                     `yaml:"port"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &puppetdbMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "puppetdb" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.Registerer)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -131,7 +138,12 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new PuppetDB discovery for the given config.
-func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*puppetdbMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -158,11 +170,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer)
 
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
+			Logger:              logger,
-			Mech:     "http",
+			Mech:                "puppetdb",
-			Interval: time.Duration(conf.RefreshInterval),
+			Interval:            time.Duration(conf.RefreshInterval),
-			RefreshF: d.refresh,
+			RefreshF:            d.refresh,
-			Registry: reg,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
 	return d, nil

discovery/puppetdb/puppetdb_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -63,9 +64,17 @@ func TestPuppetSlashInURL(t *testing.T) {
 			Port:             80,
 			RefreshInterval:  model.Duration(30 * time.Second),
 		}
-		d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
+
+		reg := prometheus.NewRegistry()
+		refreshMetrics := discovery.NewRefreshMetrics(reg)
+		metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+		require.NoError(t, metrics.Register())
+
+		d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 		require.NoError(t, err)
 		require.Equal(t, apiURL, d.url)
+
+		metrics.Unregister()
 	}
 }
 
@@ -80,7 +89,12 @@ func TestPuppetDBRefresh(t *testing.T) {
 		RefreshInterval:  model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()
@@ -107,6 +121,8 @@ func TestPuppetDBRefresh(t *testing.T) {
 		},
 	}
 	require.Equal(t, expectedTargets, tgs)
+
+	metrics.Unregister()
 }
 
 func TestPuppetDBRefreshWithParameters(t *testing.T) {
@@ -121,7 +137,12 @@ func TestPuppetDBRefreshWithParameters(t *testing.T) {
 		RefreshInterval:   model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()
@@ -158,6 +179,8 @@ func TestPuppetDBRefreshWithParameters(t *testing.T) {
 		},
 	}
 	require.Equal(t, expectedTargets, tgs)
+
+	metrics.Unregister()
 }
 
 func TestPuppetDBInvalidCode(t *testing.T) {
@@ -173,12 +196,19 @@ func TestPuppetDBInvalidCode(t *testing.T) {
 		RefreshInterval:  model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()
 	_, err = d.refresh(ctx)
 	require.EqualError(t, err, "server returned HTTP status 400 Bad Request")
+
+	metrics.Unregister()
 }
 
 func TestPuppetDBInvalidFormat(t *testing.T) {
@@ -194,10 +224,17 @@ func TestPuppetDBInvalidFormat(t *testing.T) {
 		RefreshInterval:  model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()
 	_, err = d.refresh(ctx)
 	require.EqualError(t, err, "unsupported content type text/plain; charset=utf-8")
+
+	metrics.Unregister()
 }

discovery/refresh/refresh.go

@@ -20,19 +20,17 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
-	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
 type Options struct {
-	Logger   log.Logger
+	Logger              log.Logger
-	Mech     string
+	Mech                string
-	Interval time.Duration
+	Interval            time.Duration
-	RefreshF func(ctx context.Context) ([]*targetgroup.Group, error)
+	RefreshF            func(ctx context.Context) ([]*targetgroup.Group, error)
-	Registry prometheus.Registerer
+	MetricsInstantiator discovery.RefreshMetricsInstantiator
-	Metrics  []prometheus.Collector
 }
 
 // Discovery implements the Discoverer interface.
@@ -40,15 +38,13 @@ type Discovery struct {
 	logger   log.Logger
 	interval time.Duration
 	refreshf func(ctx context.Context) ([]*targetgroup.Group, error)
-
+	metrics  *discovery.RefreshMetrics
-	failures prometheus.Counter
-	duration prometheus.Summary
-
-	metricRegisterer discovery.MetricRegisterer
 }
 
 // NewDiscovery returns a Discoverer function that calls a refresh() function at every interval.
 func NewDiscovery(opts Options) *Discovery {
+	m := opts.MetricsInstantiator.Instantiate(opts.Mech)
+
 	var logger log.Logger
 	if opts.Logger == nil {
 		logger = log.NewNopLogger()
@@ -60,44 +56,14 @@ func NewDiscovery(opts Options) *Discovery {
 		logger:   logger,
 		interval: opts.Interval,
 		refreshf: opts.RefreshF,
-		failures: prometheus.NewCounter(
+		metrics:  m,
-			prometheus.CounterOpts{
-				Name: "prometheus_sd_refresh_failures_total",
-				Help: "Number of refresh failures for the given SD mechanism.",
-				ConstLabels: prometheus.Labels{
-					"mechanism": opts.Mech,
-				},
-			}),
-		duration: prometheus.NewSummary(
-			prometheus.SummaryOpts{
-				Name:       "prometheus_sd_refresh_duration_seconds",
-				Help:       "The duration of a refresh in seconds for the given SD mechanism.",
-				Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
-				ConstLabels: prometheus.Labels{
-					"mechanism": opts.Mech,
-				},
-			}),
 	}
 
-	metrics := []prometheus.Collector{d.failures, d.duration}
-	if opts.Metrics != nil {
-		metrics = append(metrics, opts.Metrics...)
-	}
-
-	d.metricRegisterer = discovery.NewMetricRegisterer(opts.Registry, metrics)
-
 	return &d
 }
 
 // Run implements the Discoverer interface.
 func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
-	err := d.metricRegisterer.RegisterMetrics()
-	if err != nil {
-		level.Error(d.logger).Log("msg", "Unable to register metrics", "err", err.Error())
-		return
-	}
-	defer d.metricRegisterer.UnregisterMetrics()
-
 	// Get an initial set right away.
 	tgs, err := d.refresh(ctx)
 	if err != nil {
@@ -140,12 +106,12 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 	now := time.Now()
 	defer func() {
-		d.duration.Observe(time.Since(now).Seconds())
+		d.metrics.Duration.Observe(time.Since(now).Seconds())
 	}()
 
 	tgs, err := d.refreshf(ctx)
 	if err != nil {
-		d.failures.Inc()
+		d.metrics.Failures.Inc()
 	}
 	return tgs, err
 }

discovery/refresh/refresh_test.go

@@ -24,6 +24,7 @@ import (
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -66,13 +67,18 @@ func TestRefresh(t *testing.T) {
 		return nil, fmt.Errorf("some error")
 	}
 	interval := time.Millisecond
+
+	metrics := discovery.NewRefreshMetrics(prometheus.NewRegistry())
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+
 	d := NewDiscovery(
 		Options{
-			Logger:   nil,
+			Logger:              nil,
-			Mech:     "test",
+			Mech:                "test",
-			Interval: interval,
+			Interval:            interval,
-			RefreshF: refresh,
+			RefreshF:            refresh,
-			Registry: prometheus.NewRegistry(),
+			MetricsInstantiator: metrics,
 		},
 	)
 
@@ -91,7 +97,7 @@ func TestRefresh(t *testing.T) {
 	defer tick.Stop()
 	select {
 	case <-ch:
-		t.Fatal("Unexpected target group")
+		require.FailNow(t, "Unexpected target group")
 	case <-tick.C:
 	}
 }

discovery/registry.go

@@ -24,6 +24,8 @@ import (
 
 	"gopkg.in/yaml.v2"
 
+	"github.com/prometheus/client_golang/prometheus"
+
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -258,3 +260,24 @@ func replaceYAMLTypeError(err error, oldTyp, newTyp reflect.Type) error {
 	}
 	return err
 }
+
+// RegisterSDMetrics registers the metrics used by service discovery mechanisms.
+// RegisterSDMetrics should be called only once during the lifetime of the Prometheus process.
+// There is no need for the Prometheus process to unregister the metrics.
+func RegisterSDMetrics(registerer prometheus.Registerer, rmm RefreshMetricsManager) (map[string]DiscovererMetrics, error) {
+	err := rmm.Register()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create service discovery refresh metrics")
+	}
+
+	metrics := make(map[string]DiscovererMetrics)
+	for _, conf := range configNames {
+		currentSdMetrics := conf.NewDiscovererMetrics(registerer, rmm)
+		err = currentSdMetrics.Register()
+		if err != nil {
+			return nil, fmt.Errorf("failed to create service discovery metrics")
+		}
+		metrics[conf.Name()] = currentSdMetrics
+	}
+	return metrics, nil
+}

discovery/scaleway/metrics.go

@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scaleway
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*scalewayMetrics)(nil)
+
+type scalewayMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *scalewayMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *scalewayMetrics) Unregister() {}

discovery/scaleway/scaleway.go

@@ -104,6 +104,13 @@ type SDConfig struct {
 	Role role `yaml:"role"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &scalewayMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 func (c SDConfig) Name() string {
 	return "scaleway"
 }
@@ -161,7 +168,7 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 }
 
 func (c SDConfig) NewDiscoverer(options discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(&c, options.Logger, options.Registerer)
+	return NewDiscovery(&c, options.Logger, options.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -178,7 +185,12 @@ func init() {
 // the Discoverer interface.
 type Discovery struct{}
 
-func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*refresh.Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*refresh.Discovery, error) {
+	m, ok := metrics.(*scalewayMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	r, err := newRefresher(conf)
 	if err != nil {
 		return nil, err
@@ -186,11 +198,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer)
 
 	return refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
+			Logger:              logger,
-			Mech:     "scaleway",
+			Mech:                "scaleway",
-			Interval: time.Duration(conf.RefreshInterval),
+			Interval:            time.Duration(conf.RefreshInterval),
-			RefreshF: r.refresh,
+			RefreshF:            r.refresh,
-			Registry: reg,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	), nil
 }

discovery/triton/metrics.go

@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package triton
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*tritonMetrics)(nil)
+
+type tritonMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *tritonMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *tritonMetrics) Unregister() {}

discovery/triton/triton.go

@@ -70,12 +70,19 @@ type SDConfig struct {
 	Version         int              `yaml:"version"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &tritonMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "triton" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return New(opts.Logger, c, opts.Registerer)
+	return New(opts.Logger, c, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -139,7 +146,12 @@ type Discovery struct {
 }
 
 // New returns a new Discovery which periodically refreshes its targets.
-func New(logger log.Logger, conf *SDConfig, reg prometheus.Registerer) (*Discovery, error) {
+func New(logger log.Logger, conf *SDConfig, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*tritonMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	tls, err := config.NewTLSConfig(&conf.TLSConfig)
 	if err != nil {
 		return nil, err
@@ -161,11 +173,11 @@ func New(logger log.Logger, conf *SDConfig, reg prometheus.Registerer) (*Discove
 	}
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
+			Logger:              logger,
-			Mech:     "triton",
+			Mech:                "triton",
-			Interval: time.Duration(conf.RefreshInterval),
+			Interval:            time.Duration(conf.RefreshInterval),
-			RefreshF: d.refresh,
+			RefreshF:            d.refresh,
-			Registry: reg,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
 	return d, nil

discovery/triton/triton_test.go

@@ -28,6 +28,8 @@ import (
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/prometheus/discovery"
 )
 
 var (
@@ -79,12 +81,26 @@ var (
 	}
 )
 
-func newTritonDiscovery(c SDConfig) (*Discovery, error) {
+func newTritonDiscovery(c SDConfig) (*Discovery, discovery.DiscovererMetrics, error) {
-	return New(nil, &c, prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	// TODO(ptodev): Add the ability to unregister refresh metrics.
+	metrics := c.NewDiscovererMetrics(reg, refreshMetrics)
+	err := metrics.Register()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	d, err := New(nil, &c, metrics)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return d, metrics, nil
 }
 
 func TestTritonSDNew(t *testing.T) {
-	td, err := newTritonDiscovery(conf)
+	td, m, err := newTritonDiscovery(conf)
 	require.NoError(t, err)
 	require.NotNil(t, td)
 	require.NotNil(t, td.client)
@@ -94,16 +110,17 @@ func TestTritonSDNew(t *testing.T) {
 	require.Equal(t, conf.DNSSuffix, td.sdConfig.DNSSuffix)
 	require.Equal(t, conf.Endpoint, td.sdConfig.Endpoint)
 	require.Equal(t, conf.Port, td.sdConfig.Port)
+	m.Unregister()
 }
 
 func TestTritonSDNewBadConfig(t *testing.T) {
-	td, err := newTritonDiscovery(badconf)
+	td, _, err := newTritonDiscovery(badconf)
 	require.Error(t, err)
 	require.Nil(t, td)
 }
 
 func TestTritonSDNewGroupsConfig(t *testing.T) {
-	td, err := newTritonDiscovery(groupsconf)
+	td, m, err := newTritonDiscovery(groupsconf)
 	require.NoError(t, err)
 	require.NotNil(t, td)
 	require.NotNil(t, td.client)
@@ -114,10 +131,11 @@ func TestTritonSDNewGroupsConfig(t *testing.T) {
 	require.Equal(t, groupsconf.Endpoint, td.sdConfig.Endpoint)
 	require.Equal(t, groupsconf.Groups, td.sdConfig.Groups)
 	require.Equal(t, groupsconf.Port, td.sdConfig.Port)
+	m.Unregister()
 }
 
 func TestTritonSDNewCNConfig(t *testing.T) {
-	td, err := newTritonDiscovery(cnconf)
+	td, m, err := newTritonDiscovery(cnconf)
 	require.NoError(t, err)
 	require.NotNil(t, td)
 	require.NotNil(t, td.client)
@@ -128,6 +146,7 @@ func TestTritonSDNewCNConfig(t *testing.T) {
 	require.Equal(t, cnconf.DNSSuffix, td.sdConfig.DNSSuffix)
 	require.Equal(t, cnconf.Endpoint, td.sdConfig.Endpoint)
 	require.Equal(t, cnconf.Port, td.sdConfig.Port)
+	m.Unregister()
 }
 
 func TestTritonSDRefreshNoTargets(t *testing.T) {
@@ -160,21 +179,23 @@ func TestTritonSDRefreshMultipleTargets(t *testing.T) {
 }
 
 func TestTritonSDRefreshNoServer(t *testing.T) {
-	td, _ := newTritonDiscovery(conf)
+	td, m, _ := newTritonDiscovery(conf)
 
 	_, err := td.refresh(context.Background())
 	require.Error(t, err)
 	require.True(t, strings.Contains(err.Error(), "an error occurred when requesting targets from the discovery endpoint"))
+	m.Unregister()
 }
 
 func TestTritonSDRefreshCancelled(t *testing.T) {
-	td, _ := newTritonDiscovery(conf)
+	td, m, _ := newTritonDiscovery(conf)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel()
 	_, err := td.refresh(ctx)
 	require.Error(t, err)
 	require.True(t, strings.Contains(err.Error(), context.Canceled.Error()))
+	m.Unregister()
 }
 
 func TestTritonSDRefreshCNsUUIDOnly(t *testing.T) {
@@ -211,8 +232,8 @@ func TestTritonSDRefreshCNsWithHostname(t *testing.T) {
 
 func testTritonSDRefresh(t *testing.T, c SDConfig, dstr string) []model.LabelSet {
 	var (
-		td, _ = newTritonDiscovery(c)
+		td, m, _ = newTritonDiscovery(c)
-		s     = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		s        = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			fmt.Fprintln(w, dstr)
 		}))
 	)
@@ -240,5 +261,7 @@ func testTritonSDRefresh(t *testing.T, c SDConfig, dstr string) []model.LabelSet
 	tg := tgs[0]
 	require.NotNil(t, tg)
 
+	m.Unregister()
+
 	return tg.Targets
 }

discovery/uyuni/metrics.go

@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package uyuni
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*uyuniMetrics)(nil)
+
+type uyuniMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *uyuniMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *uyuniMetrics) Unregister() {}

discovery/uyuni/uyuni.go

@@ -111,12 +111,19 @@ type Discovery struct {
 	logger          log.Logger
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &uyuniMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "uyuni" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.Registerer)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -204,7 +211,12 @@ func getEndpointInfoForSystems(
 }
 
 // NewDiscovery returns a uyuni discovery for the given configuration.
-func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*uyuniMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	apiURL, err := url.Parse(conf.Server)
 	if err != nil {
 		return nil, err
@@ -229,11 +241,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer)
 
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
+			Logger:              logger,
-			Mech:     "uyuni",
+			Mech:                "uyuni",
-			Interval: time.Duration(conf.RefreshInterval),
+			Interval:            time.Duration(conf.RefreshInterval),
-			RefreshF: d.refresh,
+			RefreshF:            d.refresh,
-			Registry: reg,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
 	return d, nil

discovery/uyuni/uyuni_test.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -37,7 +38,17 @@ func testUpdateServices(respHandler http.HandlerFunc) ([]*targetgroup.Group, err
 		Server: ts.URL,
 	}
 
-	md, err := NewDiscovery(&conf, nil, prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := conf.NewDiscovererMetrics(reg, refreshMetrics)
+	err := metrics.Register()
+	if err != nil {
+		return nil, err
+	}
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	md, err := NewDiscovery(&conf, nil, metrics)
 	if err != nil {
 		return nil, err
 	}
@@ -110,7 +121,14 @@ func TestUyuniSDSkipLogin(t *testing.T) {
 		Server: ts.URL,
 	}
 
-	md, err := NewDiscovery(&conf, nil, prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := conf.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	md, err := NewDiscovery(&conf, nil, metrics)
 	if err != nil {
 		t.Error(err)
 	}