Merge branch 'main' into merge-2.49.0-back

Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>

.github/workflows/ci.yml

@@ -151,6 +151,7 @@ jobs:
         uses: golangci/golangci-lint-action@3a919529898de77ec3da873e3063ca4b10e7f5cc # v3.7.0
         with:
           args: --verbose
+          # Make sure to sync this with Makefile.common and scripts/golangci-lint.yml.
           version: v1.55.2
   fuzzing:
     uses: ./.github/workflows/fuzzing.yml

.github/workflows/codeql-analysis.yml

@@ -30,12 +30,12 @@ jobs:
           go-version: 1.21.x
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@407ffafae6a767df3e0230c3df91b6443ae8df75 # v2.22.8
+        uses: github/codeql-action/init@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # v3.22.12
         with:
           languages: ${{ matrix.language }}
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@407ffafae6a767df3e0230c3df91b6443ae8df75 # v2.22.8
+        uses: github/codeql-action/autobuild@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # v3.22.12
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@407ffafae6a767df3e0230c3df91b6443ae8df75 # v2.22.8
+        uses: github/codeql-action/analyze@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # v3.22.12

.github/workflows/scorecards.yml

@@ -45,6 +45,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@407ffafae6a767df3e0230c3df91b6443ae8df75 # tag=v2.22.8
+        uses: github/codeql-action/upload-sarif@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # tag=v3.22.12
         with:
           sarif_file: results.sarif

.golangci.yml

@@ -36,13 +36,9 @@ issues:
     - path: _test.go
       linters:
         - errcheck
-    - path: tsdb/
+    - path: "tsdb/head_wal.go"
       linters:
         - errorlint
-    - path: tsdb/
-      text: "import 'github.com/pkg/errors' is not allowed"
-      linters:
-        - depguard
     - linters:
         - godot
       source: "^// ==="

RELEASE.md

@@ -54,7 +54,8 @@ Release cadence of first pre-releases being cut is 6 weeks.
 | v2.47          | 2023-08-23                                 | Bryan Boreham (GitHub: @bboreham)           |
 | v2.48          | 2023-10-04                                 | Levi Harrison (GitHub: @LeviHarrison)       |
 | v2.49          | 2023-12-05                                 | Bartek Plotka (GitHub: @bwplotka)           |
-| v2.50          | 2024-01-16                                 | **searching for volunteer**                 |
+| v2.50          | 2024-01-16                                 | Augustin Husson (GitHub: @nexucis)          |
+| v2.51          | 2024-02-13                                 | **searching for volunteer**                 |
 
 If you are interested in volunteering please create a pull request against the [prometheus/prometheus](https://github.com/prometheus/prometheus) repository and propose yourself for the release series of your choice.
 

cmd/prometheus/main.go

@@ -206,9 +206,15 @@ func (c *flagConfig) setFeatureListOptions(logger log.Logger) error {
 			case "native-histograms":
 				c.tsdb.EnableNativeHistograms = true
 				// Change relevant global variables. Hacky, but it's hard to pass a new option or default to unmarshallers.
-				config.DefaultConfig.GlobalConfig.ScrapeProtocols = config.DefaultNativeHistogramScrapeProtocols
-				config.DefaultGlobalConfig.ScrapeProtocols = config.DefaultNativeHistogramScrapeProtocols
+				config.DefaultConfig.GlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
+				config.DefaultGlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
 				level.Info(logger).Log("msg", "Experimental native histogram support enabled. Changed default scrape_protocols to prefer PrometheusProto format.", "global.scrape_protocols", fmt.Sprintf("%v", config.DefaultGlobalConfig.ScrapeProtocols))
+			case "created-timestamp-zero-ingestion":
+				c.scrape.EnableCreatedTimestampZeroIngestion = true
+				// Change relevant global variables. Hacky, but it's hard to pass a new option or default to unmarshallers.
+				config.DefaultConfig.GlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
+				config.DefaultGlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
+				level.Info(logger).Log("msg", "Experimental created timestamp zero ingestion enabled. Changed default scrape_protocols to prefer PrometheusProto format.", "global.scrape_protocols", fmt.Sprintf("%v", config.DefaultGlobalConfig.ScrapeProtocols))
 			case "":
 				continue
 			case "promql-at-modifier", "promql-negative-offset":
@@ -614,14 +620,52 @@ func main() {
 		discoveryManagerNotify  discoveryManager
 	)
 
+	// Kubernetes client metrics are used by Kubernetes SD.
+	// They are registered here in the main function, because SD mechanisms
+	// can only register metrics specific to a SD instance.
+	// Kubernetes client metrics are the same for the whole process -
+	// they are not specific to an SD instance.
+	err = discovery.RegisterK8sClientMetricsWithPrometheus(prometheus.DefaultRegisterer)
+	if err != nil {
+		level.Error(logger).Log("msg", "failed to register Kubernetes client metrics", "err", err)
+		os.Exit(1)
+	}
 	if cfg.enableNewSDManager {
-		discovery.RegisterMetrics()
-		discoveryManagerScrape = discovery.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), discovery.Name("scrape"))
-		discoveryManagerNotify = discovery.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), discovery.Name("notify"))
+		{
+			discMgr := discovery.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, discovery.Name("scrape"))
+			if discMgr == nil {
+				level.Error(logger).Log("msg", "failed to create a discovery manager scrape")
+				os.Exit(1)
+			}
+			discoveryManagerScrape = discMgr
+		}
+
+		{
+			discMgr := discovery.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, discovery.Name("notify"))
+			if discMgr == nil {
+				level.Error(logger).Log("msg", "failed to create a discovery manager notify")
+				os.Exit(1)
+			}
+			discoveryManagerNotify = discMgr
+		}
 	} else {
-		legacymanager.RegisterMetrics()
-		discoveryManagerScrape = legacymanager.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), legacymanager.Name("scrape"))
-		discoveryManagerNotify = legacymanager.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), legacymanager.Name("notify"))
+		{
+			discMgr := legacymanager.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, legacymanager.Name("scrape"))
+			if discMgr == nil {
+				level.Error(logger).Log("msg", "failed to create a discovery manager scrape")
+				os.Exit(1)
+			}
+			discoveryManagerScrape = discMgr
+		}
+
+		{
+			discMgr := legacymanager.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, legacymanager.Name("notify"))
+			if discMgr == nil {
+				level.Error(logger).Log("msg", "failed to create a discovery manager notify")
+				os.Exit(1)
+			}
+			discoveryManagerNotify = discMgr
+		}
 	}
 
 	scrapeManager, err := scrape.NewManager(
@@ -1449,6 +1493,10 @@ func (n notReadyAppender) UpdateMetadata(ref storage.SeriesRef, l labels.Labels,
 	return 0, tsdb.ErrNotReady
 }
 
+func (n notReadyAppender) AppendCTZeroSample(ref storage.SeriesRef, l labels.Labels, t, ct int64) (storage.SeriesRef, error) {
+	return 0, tsdb.ErrNotReady
+}
+
 func (n notReadyAppender) Commit() error { return tsdb.ErrNotReady }
 
 func (n notReadyAppender) Rollback() error { return tsdb.ErrNotReady }
@@ -1587,7 +1635,6 @@ func (opts tsdbOptions) ToTSDBOptions() tsdb.Options {
 		RetentionDuration:              int64(time.Duration(opts.RetentionDuration) / time.Millisecond),
 		MaxBytes:                       int64(opts.MaxBytes),
 		NoLockfile:                     opts.NoLockfile,
-		AllowOverlappingCompaction:     true,
 		WALCompression:                 wlog.ParseCompressionType(opts.WALCompression, opts.WALCompressionType),
 		HeadChunksWriteQueueSize:       opts.HeadChunksWriteQueueSize,
 		SamplesPerChunk:                opts.SamplesPerChunk,

cmd/promtool/analyze.go

@@ -0,0 +1,370 @@
+// Copyright 2023 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"net/url"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
+	"github.com/prometheus/common/model"
+
+	"github.com/prometheus/prometheus/model/labels"
+)
+
+var (
+	errNotNativeHistogram = fmt.Errorf("not a native histogram")
+	errNotEnoughData      = fmt.Errorf("not enough data")
+
+	outputHeader = `Bucket stats for each histogram series over time
+------------------------------------------------
+First the min, avg, and max number of populated buckets, followed by the total
+number of buckets (only if different from the max number of populated buckets
+which is typical for classic but not native histograms).`
+	outputFooter = `Aggregated bucket stats
+-----------------------
+Each line shows min/avg/max over the series above.`
+)
+
+type QueryAnalyzeConfig struct {
+	metricType string
+	duration   time.Duration
+	time       string
+	matchers   []string
+}
+
+// run retrieves metrics that look like conventional histograms (i.e. have _bucket
+// suffixes) or native histograms, depending on metricType flag.
+func (c *QueryAnalyzeConfig) run(url *url.URL, roundtripper http.RoundTripper) error {
+	if c.metricType != "histogram" {
+		return fmt.Errorf("analyze type is %s, must be 'histogram'", c.metricType)
+	}
+
+	ctx := context.Background()
+
+	api, err := newAPI(url, roundtripper, nil)
+	if err != nil {
+		return err
+	}
+
+	var endTime time.Time
+	if c.time != "" {
+		endTime, err = parseTime(c.time)
+		if err != nil {
+			return fmt.Errorf("error parsing time '%s': %w", c.time, err)
+		}
+	} else {
+		endTime = time.Now()
+	}
+
+	return c.getStatsFromMetrics(ctx, api, endTime, os.Stdout, c.matchers)
+}
+
+func (c *QueryAnalyzeConfig) getStatsFromMetrics(ctx context.Context, api v1.API, endTime time.Time, out io.Writer, matchers []string) error {
+	fmt.Fprintf(out, "%s\n\n", outputHeader)
+	metastatsNative := newMetaStatistics()
+	metastatsClassic := newMetaStatistics()
+	for _, matcher := range matchers {
+		seriesSel := seriesSelector(matcher, c.duration)
+		matrix, err := querySamples(ctx, api, seriesSel, endTime)
+		if err != nil {
+			return err
+		}
+
+		matrices := make(map[string]model.Matrix)
+		for _, series := range matrix {
+			// We do not handle mixed types. If there are float values, we assume it is a
+			// classic histogram, otherwise we assume it is a native histogram, and we
+			// ignore series with errors if they do not match the expected type.
+			if len(series.Values) == 0 {
+				stats, err := calcNativeBucketStatistics(series)
+				if err != nil {
+					if errors.Is(err, errNotNativeHistogram) || errors.Is(err, errNotEnoughData) {
+						continue
+					}
+					return err
+				}
+				fmt.Fprintf(out, "- %s (native): %v\n", series.Metric, *stats)
+				metastatsNative.update(stats)
+			} else {
+				lbs := model.LabelSet(series.Metric).Clone()
+				if _, ok := lbs["le"]; !ok {
+					continue
+				}
+				metricName := string(lbs[labels.MetricName])
+				if !strings.HasSuffix(metricName, "_bucket") {
+					continue
+				}
+				delete(lbs, labels.MetricName)
+				delete(lbs, "le")
+				key := formatSeriesName(metricName, lbs)
+				matrices[key] = append(matrices[key], series)
+			}
+		}
+
+		for key, matrix := range matrices {
+			stats, err := calcClassicBucketStatistics(matrix)
+			if err != nil {
+				if errors.Is(err, errNotEnoughData) {
+					continue
+				}
+				return err
+			}
+			fmt.Fprintf(out, "- %s (classic): %v\n", key, *stats)
+			metastatsClassic.update(stats)
+		}
+	}
+	fmt.Fprintf(out, "\n%s\n", outputFooter)
+	if metastatsNative.Count() > 0 {
+		fmt.Fprintf(out, "\nNative %s\n", metastatsNative)
+	}
+	if metastatsClassic.Count() > 0 {
+		fmt.Fprintf(out, "\nClassic %s\n", metastatsClassic)
+	}
+	return nil
+}
+
+func seriesSelector(metricName string, duration time.Duration) string {
+	builder := strings.Builder{}
+	builder.WriteString(metricName)
+	builder.WriteRune('[')
+	builder.WriteString(duration.String())
+	builder.WriteRune(']')
+	return builder.String()
+}
+
+func formatSeriesName(metricName string, lbs model.LabelSet) string {
+	builder := strings.Builder{}
+	builder.WriteString(metricName)
+	builder.WriteString(lbs.String())
+	return builder.String()
+}
+
+func querySamples(ctx context.Context, api v1.API, query string, end time.Time) (model.Matrix, error) {
+	values, _, err := api.Query(ctx, query, end)
+	if err != nil {
+		return nil, err
+	}
+
+	matrix, ok := values.(model.Matrix)
+	if !ok {
+		return nil, fmt.Errorf("query of buckets resulted in non-Matrix")
+	}
+
+	return matrix, nil
+}
+
+// minPop/avgPop/maxPop is for the number of populated (non-zero) buckets.
+// total is the total number of buckets across all samples in the series,
+// populated or not.
+type statistics struct {
+	minPop, maxPop, total int
+	avgPop                float64
+}
+
+func (s statistics) String() string {
+	if s.maxPop == s.total {
+		return fmt.Sprintf("%d/%.3f/%d", s.minPop, s.avgPop, s.maxPop)
+	}
+	return fmt.Sprintf("%d/%.3f/%d/%d", s.minPop, s.avgPop, s.maxPop, s.total)
+}
+
+func calcClassicBucketStatistics(matrix model.Matrix) (*statistics, error) {
+	numBuckets := len(matrix)
+
+	stats := &statistics{
+		minPop: math.MaxInt,
+		total:  numBuckets,
+	}
+
+	if numBuckets == 0 || len(matrix[0].Values) < 2 {
+		return stats, errNotEnoughData
+	}
+
+	numSamples := len(matrix[0].Values)
+
+	sortMatrix(matrix)
+
+	totalPop := 0
+	for timeIdx := 0; timeIdx < numSamples; timeIdx++ {
+		curr, err := getBucketCountsAtTime(matrix, numBuckets, timeIdx)
+		if err != nil {
+			return stats, err
+		}
+		countPop := 0
+		for _, b := range curr {
+			if b != 0 {
+				countPop++
+			}
+		}
+
+		totalPop += countPop
+		if stats.minPop > countPop {
+			stats.minPop = countPop
+		}
+		if stats.maxPop < countPop {
+			stats.maxPop = countPop
+		}
+	}
+	stats.avgPop = float64(totalPop) / float64(numSamples)
+	return stats, nil
+}
+
+func sortMatrix(matrix model.Matrix) {
+	sort.SliceStable(matrix, func(i, j int) bool {
+		return getLe(matrix[i]) < getLe(matrix[j])
+	})
+}
+
+func getLe(series *model.SampleStream) float64 {
+	lbs := model.LabelSet(series.Metric)
+	le, _ := strconv.ParseFloat(string(lbs["le"]), 64)
+	return le
+}
+
+func getBucketCountsAtTime(matrix model.Matrix, numBuckets, timeIdx int) ([]int, error) {
+	counts := make([]int, numBuckets)
+	if timeIdx >= len(matrix[0].Values) {
+		// Just return zeroes instead of erroring out so we can get partial results.
+		return counts, nil
+	}
+	counts[0] = int(matrix[0].Values[timeIdx].Value)
+	for i, bucket := range matrix[1:] {
+		if timeIdx >= len(bucket.Values) {
+			// Just return zeroes instead of erroring out so we can get partial results.
+			return counts, nil
+		}
+		curr := bucket.Values[timeIdx]
+		prev := matrix[i].Values[timeIdx]
+		// Assume the results are nicely aligned.
+		if curr.Timestamp != prev.Timestamp {
+			return counts, fmt.Errorf("matrix result is not time aligned")
+		}
+		counts[i+1] = int(curr.Value - prev.Value)
+	}
+	return counts, nil
+}
+
+type bucketBounds struct {
+	boundaries   int32
+	upper, lower float64
+}
+
+func makeBucketBounds(b *model.HistogramBucket) bucketBounds {
+	return bucketBounds{
+		boundaries: b.Boundaries,
+		upper:      float64(b.Upper),
+		lower:      float64(b.Lower),
+	}
+}
+
+func calcNativeBucketStatistics(series *model.SampleStream) (*statistics, error) {
+	stats := &statistics{
+		minPop: math.MaxInt,
+	}
+
+	overall := make(map[bucketBounds]struct{})
+	totalPop := 0
+	if len(series.Histograms) == 0 {
+		return nil, errNotNativeHistogram
+	}
+	if len(series.Histograms) == 1 {
+		return nil, errNotEnoughData
+	}
+	for _, histogram := range series.Histograms {
+		for _, bucket := range histogram.Histogram.Buckets {
+			bb := makeBucketBounds(bucket)
+			overall[bb] = struct{}{}
+		}
+		countPop := len(histogram.Histogram.Buckets)
+
+		totalPop += countPop
+		if stats.minPop > countPop {
+			stats.minPop = countPop
+		}
+		if stats.maxPop < countPop {
+			stats.maxPop = countPop
+		}
+	}
+	stats.avgPop = float64(totalPop) / float64(len(series.Histograms))
+	stats.total = len(overall)
+	return stats, nil
+}
+
+type distribution struct {
+	min, max, count int
+	avg             float64
+}
+
+func newDistribution() distribution {
+	return distribution{
+		min: math.MaxInt,
+	}
+}
+
+func (d *distribution) update(num int) {
+	if d.min > num {
+		d.min = num
+	}
+	if d.max < num {
+		d.max = num
+	}
+	d.count++
+	d.avg += float64(num)/float64(d.count) - d.avg/float64(d.count)
+}
+
+func (d distribution) String() string {
+	return fmt.Sprintf("%d/%.3f/%d", d.min, d.avg, d.max)
+}
+
+type metaStatistics struct {
+	minPop, avgPop, maxPop, total distribution
+}
+
+func newMetaStatistics() *metaStatistics {
+	return &metaStatistics{
+		minPop: newDistribution(),
+		avgPop: newDistribution(),
+		maxPop: newDistribution(),
+		total:  newDistribution(),
+	}
+}
+
+func (ms metaStatistics) Count() int {
+	return ms.minPop.count
+}
+
+func (ms metaStatistics) String() string {
+	if ms.maxPop == ms.total {
+		return fmt.Sprintf("histogram series (%d in total):\n- min populated: %v\n- avg populated: %v\n- max populated: %v", ms.Count(), ms.minPop, ms.avgPop, ms.maxPop)
+	}
+	return fmt.Sprintf("histogram series (%d in total):\n- min populated: %v\n- avg populated: %v\n- max populated: %v\n- total: %v", ms.Count(), ms.minPop, ms.avgPop, ms.maxPop, ms.total)
+}
+
+func (ms *metaStatistics) update(s *statistics) {
+	ms.minPop.update(s.minPop)
+	ms.avgPop.update(int(s.avgPop))
+	ms.maxPop.update(s.maxPop)
+	ms.total.update(s.total)
+}

cmd/promtool/analyze_test.go

@@ -0,0 +1,170 @@
+// Copyright 2023 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/common/model"
+)
+
+var (
+	exampleMatrix = model.Matrix{
+		&model.SampleStream{
+			Metric: model.Metric{
+				"le": "+Inf",
+			},
+			Values: []model.SamplePair{
+				{
+					Value:     31,
+					Timestamp: 100,
+				},
+				{
+					Value:     32,
+					Timestamp: 200,
+				},
+				{
+					Value:     40,
+					Timestamp: 300,
+				},
+			},
+		},
+		&model.SampleStream{
+			Metric: model.Metric{
+				"le": "0.5",
+			},
+			Values: []model.SamplePair{
+				{
+					Value:     10,
+					Timestamp: 100,
+				},
+				{
+					Value:     11,
+					Timestamp: 200,
+				},
+				{
+					Value:     11,
+					Timestamp: 300,
+				},
+			},
+		},
+		&model.SampleStream{
+			Metric: model.Metric{
+				"le": "10",
+			},
+			Values: []model.SamplePair{
+				{
+					Value:     30,
+					Timestamp: 100,
+				},
+				{
+					Value:     31,
+					Timestamp: 200,
+				},
+				{
+					Value:     37,
+					Timestamp: 300,
+				},
+			},
+		},
+		&model.SampleStream{
+			Metric: model.Metric{
+				"le": "2",
+			},
+			Values: []model.SamplePair{
+				{
+					Value:     25,
+					Timestamp: 100,
+				},
+				{
+					Value:     26,
+					Timestamp: 200,
+				},
+				{
+					Value:     27,
+					Timestamp: 300,
+				},
+			},
+		},
+	}
+	exampleMatrixLength = len(exampleMatrix)
+)
+
+func init() {
+	sortMatrix(exampleMatrix)
+}
+
+func TestGetBucketCountsAtTime(t *testing.T) {
+	cases := []struct {
+		matrix   model.Matrix
+		length   int
+		timeIdx  int
+		expected []int
+	}{
+		{
+			exampleMatrix,
+			exampleMatrixLength,
+			0,
+			[]int{10, 15, 5, 1},
+		},
+		{
+			exampleMatrix,
+			exampleMatrixLength,
+			1,
+			[]int{11, 15, 5, 1},
+		},
+		{
+			exampleMatrix,
+			exampleMatrixLength,
+			2,
+			[]int{11, 16, 10, 3},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(fmt.Sprintf("exampleMatrix@%d", c.timeIdx), func(t *testing.T) {
+			res, err := getBucketCountsAtTime(c.matrix, c.length, c.timeIdx)
+			require.NoError(t, err)
+			require.Equal(t, c.expected, res)
+		})
+	}
+}
+
+func TestCalcClassicBucketStatistics(t *testing.T) {
+	cases := []struct {
+		matrix   model.Matrix
+		expected *statistics
+	}{
+		{
+			exampleMatrix,
+			&statistics{
+				minPop: 4,
+				avgPop: 4,
+				maxPop: 4,
+				total:  4,
+			},
+		},
+	}
+
+	for i, c := range cases {
+		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
+			res, err := calcClassicBucketStatistics(c.matrix)
+			require.NoError(t, err)
+			require.Equal(t, c.expected, res)
+		})
+	}
+}

cmd/promtool/main.go

@@ -35,8 +35,7 @@ import (
 	"github.com/go-kit/log"
 	"github.com/google/pprof/profile"
 	"github.com/prometheus/client_golang/api"
-	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/testutil/promlint"
 	config_util "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
@@ -184,6 +183,14 @@ func main() {
 	queryLabelsEnd := queryLabelsCmd.Flag("end", "End time (RFC3339 or Unix timestamp).").String()
 	queryLabelsMatch := queryLabelsCmd.Flag("match", "Series selector. Can be specified multiple times.").Strings()
 
+	queryAnalyzeCfg := &QueryAnalyzeConfig{}
+	queryAnalyzeCmd := queryCmd.Command("analyze", "Run queries against your Prometheus to analyze the usage pattern of certain metrics.")
+	queryAnalyzeCmd.Flag("server", "Prometheus server to query.").Required().URLVar(&serverURL)
+	queryAnalyzeCmd.Flag("type", "Type of metric: histogram.").Required().StringVar(&queryAnalyzeCfg.metricType)
+	queryAnalyzeCmd.Flag("duration", "Time frame to analyze.").Default("1h").DurationVar(&queryAnalyzeCfg.duration)
+	queryAnalyzeCmd.Flag("time", "Query time (RFC3339 or Unix timestamp), defaults to now.").StringVar(&queryAnalyzeCfg.time)
+	queryAnalyzeCmd.Flag("match", "Series selector. Can be specified multiple times.").Required().StringsVar(&queryAnalyzeCfg.matchers)
+
 	pushCmd := app.Command("push", "Push to a Prometheus server.")
 	pushCmd.Flag("http.config.file", "HTTP client configuration file for promtool to connect to Prometheus.").PlaceHolder("<filename>").ExistingFileVar(&httpConfigFilePath)
 	pushMetricsCmd := pushCmd.Command("metrics", "Push metrics to a prometheus remote write (for testing purpose only).")
@@ -229,7 +236,7 @@ func main() {
 	dumpPath := tsdbDumpCmd.Arg("db path", "Database path (default is "+defaultDBPath+").").Default(defaultDBPath).String()
 	dumpMinTime := tsdbDumpCmd.Flag("min-time", "Minimum timestamp to dump.").Default(strconv.FormatInt(math.MinInt64, 10)).Int64()
 	dumpMaxTime := tsdbDumpCmd.Flag("max-time", "Maximum timestamp to dump.").Default(strconv.FormatInt(math.MaxInt64, 10)).Int64()
-	dumpMatch := tsdbDumpCmd.Flag("match", "Series selector.").Default("{__name__=~'(?s:.*)'}").String()
+	dumpMatch := tsdbDumpCmd.Flag("match", "Series selector. Can be specified multiple times.").Default("{__name__=~'(?s:.*)'}").Strings()
 
 	importCmd := tsdbCmd.Command("create-blocks-from", "[Experimental] Import samples from input and produce TSDB blocks. Please refer to the storage docs for more details.")
 	importHumanReadable := importCmd.Flag("human-readable", "Print human readable values.").Short('r').Bool()
@@ -317,7 +324,7 @@ func main() {
 
 	switch parsedCmd {
 	case sdCheckCmd.FullCommand():
-		os.Exit(CheckSD(*sdConfigFile, *sdJobName, *sdTimeout, noDefaultScrapePort))
+		os.Exit(CheckSD(*sdConfigFile, *sdJobName, *sdTimeout, noDefaultScrapePort, prometheus.DefaultRegisterer))
 
 	case checkConfigCmd.FullCommand():
 		os.Exit(CheckConfig(*agentMode, *checkConfigSyntaxOnly, newLintConfig(*checkConfigLint, *checkConfigLintFatal), *configFiles...))
@@ -389,6 +396,9 @@ func main() {
 	case importRulesCmd.FullCommand():
 		os.Exit(checkErr(importRules(serverURL, httpRoundTripper, *importRulesStart, *importRulesEnd, *importRulesOutputDir, *importRulesEvalInterval, *maxBlockDuration, *importRulesFiles...)))
 
+	case queryAnalyzeCmd.FullCommand():
+		os.Exit(checkErr(queryAnalyzeCfg.run(serverURL, httpRoundTripper)))
+
 	case documentationCmd.FullCommand():
 		os.Exit(checkErr(documentcli.GenerateMarkdown(app.Model(), os.Stdout)))
 
@@ -996,246 +1006,6 @@ func checkMetricsExtended(r io.Reader) ([]metricStat, int, error) {
 	return stats, total, nil
 }
 
-// QueryInstant performs an instant query against a Prometheus server.
-func QueryInstant(url *url.URL, roundTripper http.RoundTripper, query, evalTime string, p printer) int {
-	if url.Scheme == "" {
-		url.Scheme = "http"
-	}
-	config := api.Config{
-		Address:      url.String(),
-		RoundTripper: roundTripper,
-	}
-
-	// Create new client.
-	c, err := api.NewClient(config)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, "error creating API client:", err)
-		return failureExitCode
-	}
-
-	eTime := time.Now()
-	if evalTime != "" {
-		eTime, err = parseTime(evalTime)
-		if err != nil {
-			fmt.Fprintln(os.Stderr, "error parsing evaluation time:", err)
-			return failureExitCode
-		}
-	}
-
-	// Run query against client.
-	api := v1.NewAPI(c)
-
-	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
-	val, _, err := api.Query(ctx, query, eTime) // Ignoring warnings for now.
-	cancel()
-	if err != nil {
-		return handleAPIError(err)
-	}
-
-	p.printValue(val)
-
-	return successExitCode
-}
-
-// QueryRange performs a range query against a Prometheus server.
-func QueryRange(url *url.URL, roundTripper http.RoundTripper, headers map[string]string, query, start, end string, step time.Duration, p printer) int {
-	if url.Scheme == "" {
-		url.Scheme = "http"
-	}
-	config := api.Config{
-		Address:      url.String(),
-		RoundTripper: roundTripper,
-	}
-
-	if len(headers) > 0 {
-		config.RoundTripper = promhttp.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
-			for key, value := range headers {
-				req.Header.Add(key, value)
-			}
-			return roundTripper.RoundTrip(req)
-		})
-	}
-
-	// Create new client.
-	c, err := api.NewClient(config)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, "error creating API client:", err)
-		return failureExitCode
-	}
-
-	var stime, etime time.Time
-
-	if end == "" {
-		etime = time.Now()
-	} else {
-		etime, err = parseTime(end)
-		if err != nil {
-			fmt.Fprintln(os.Stderr, "error parsing end time:", err)
-			return failureExitCode
-		}
-	}
-
-	if start == "" {
-		stime = etime.Add(-5 * time.Minute)
-	} else {
-		stime, err = parseTime(start)
-		if err != nil {
-			fmt.Fprintln(os.Stderr, "error parsing start time:", err)
-			return failureExitCode
-		}
-	}
-
-	if !stime.Before(etime) {
-		fmt.Fprintln(os.Stderr, "start time is not before end time")
-		return failureExitCode
-	}
-
-	if step == 0 {
-		resolution := math.Max(math.Floor(etime.Sub(stime).Seconds()/250), 1)
-		// Convert seconds to nanoseconds such that time.Duration parses correctly.
-		step = time.Duration(resolution) * time.Second
-	}
-
-	// Run query against client.
-	api := v1.NewAPI(c)
-	r := v1.Range{Start: stime, End: etime, Step: step}
-	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
-	val, _, err := api.QueryRange(ctx, query, r) // Ignoring warnings for now.
-	cancel()
-
-	if err != nil {
-		return handleAPIError(err)
-	}
-
-	p.printValue(val)
-	return successExitCode
-}
-
-// QuerySeries queries for a series against a Prometheus server.
-func QuerySeries(url *url.URL, roundTripper http.RoundTripper, matchers []string, start, end string, p printer) int {
-	if url.Scheme == "" {
-		url.Scheme = "http"
-	}
-	config := api.Config{
-		Address:      url.String(),
-		RoundTripper: roundTripper,
-	}
-
-	// Create new client.
-	c, err := api.NewClient(config)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, "error creating API client:", err)
-		return failureExitCode
-	}
-
-	stime, etime, err := parseStartTimeAndEndTime(start, end)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		return failureExitCode
-	}
-
-	// Run query against client.
-	api := v1.NewAPI(c)
-	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
-	val, _, err := api.Series(ctx, matchers, stime, etime) // Ignoring warnings for now.
-	cancel()
-
-	if err != nil {
-		return handleAPIError(err)
-	}
-
-	p.printSeries(val)
-	return successExitCode
-}
-
-// QueryLabels queries for label values against a Prometheus server.
-func QueryLabels(url *url.URL, roundTripper http.RoundTripper, matchers []string, name, start, end string, p printer) int {
-	if url.Scheme == "" {
-		url.Scheme = "http"
-	}
-	config := api.Config{
-		Address:      url.String(),
-		RoundTripper: roundTripper,
-	}
-
-	// Create new client.
-	c, err := api.NewClient(config)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, "error creating API client:", err)
-		return failureExitCode
-	}
-
-	stime, etime, err := parseStartTimeAndEndTime(start, end)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		return failureExitCode
-	}
-
-	// Run query against client.
-	api := v1.NewAPI(c)
-	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
-	val, warn, err := api.LabelValues(ctx, name, matchers, stime, etime)
-	cancel()
-
-	for _, v := range warn {
-		fmt.Fprintln(os.Stderr, "query warning:", v)
-	}
-	if err != nil {
-		return handleAPIError(err)
-	}
-
-	p.printLabelValues(val)
-	return successExitCode
-}
-
-func handleAPIError(err error) int {
-	var apiErr *v1.Error
-	if errors.As(err, &apiErr) && apiErr.Detail != "" {
-		fmt.Fprintf(os.Stderr, "query error: %v (detail: %s)\n", apiErr, strings.TrimSpace(apiErr.Detail))
-	} else {
-		fmt.Fprintln(os.Stderr, "query error:", err)
-	}
-
-	return failureExitCode
-}
-
-func parseStartTimeAndEndTime(start, end string) (time.Time, time.Time, error) {
-	var (
-		minTime = time.Now().Add(-9999 * time.Hour)
-		maxTime = time.Now().Add(9999 * time.Hour)
-		err     error
-	)
-
-	stime := minTime
-	etime := maxTime
-
-	if start != "" {
-		stime, err = parseTime(start)
-		if err != nil {
-			return stime, etime, fmt.Errorf("error parsing start time: %w", err)
-		}
-	}
-
-	if end != "" {
-		etime, err = parseTime(end)
-		if err != nil {
-			return stime, etime, fmt.Errorf("error parsing end time: %w", err)
-		}
-	}
-	return stime, etime, nil
-}
-
-func parseTime(s string) (time.Time, error) {
-	if t, err := strconv.ParseFloat(s, 64); err == nil {
-		s, ns := math.Modf(t)
-		return time.Unix(int64(s), int64(ns*float64(time.Second))).UTC(), nil
-	}
-	if t, err := time.Parse(time.RFC3339Nano, s); err == nil {
-		return t, nil
-	}
-	return time.Time{}, fmt.Errorf("cannot parse %q to a valid timestamp", s)
-}
-
 type endpointsGroup struct {
 	urlToFilename map[string]string
 	postProcess   func(b []byte) ([]byte, error)
@@ -1389,15 +1159,12 @@ func importRules(url *url.URL, roundTripper http.RoundTripper, start, end, outpu
 		evalInterval:     evalInterval,
 		maxBlockDuration: maxBlockDuration,
 	}
-	client, err := api.NewClient(api.Config{
-		Address:      url.String(),
-		RoundTripper: roundTripper,
-	})
+	api, err := newAPI(url, roundTripper, nil)
 	if err != nil {
 		return fmt.Errorf("new api client error: %w", err)
 	}
 
-	ruleImporter := newRuleImporter(log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr)), cfg, v1.NewAPI(client))
+	ruleImporter := newRuleImporter(log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr)), cfg, api)
 	errs := ruleImporter.loadGroups(ctx, files)
 	for _, err := range errs {
 		if err != nil {

cmd/promtool/query.go

@@ -0,0 +1,251 @@
+// Copyright 2023 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math"
+	"net/http"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/prometheus/client_golang/api"
+	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
+	_ "github.com/prometheus/prometheus/plugins" // Register plugins.
+)
+
+func newAPI(url *url.URL, roundTripper http.RoundTripper, headers map[string]string) (v1.API, error) {
+	if url.Scheme == "" {
+		url.Scheme = "http"
+	}
+	config := api.Config{
+		Address:      url.String(),
+		RoundTripper: roundTripper,
+	}
+
+	if len(headers) > 0 {
+		config.RoundTripper = promhttp.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
+			for key, value := range headers {
+				req.Header.Add(key, value)
+			}
+			return roundTripper.RoundTrip(req)
+		})
+	}
+
+	// Create new client.
+	client, err := api.NewClient(config)
+	if err != nil {
+		return nil, err
+	}
+
+	api := v1.NewAPI(client)
+	return api, nil
+}
+
+// QueryInstant performs an instant query against a Prometheus server.
+func QueryInstant(url *url.URL, roundTripper http.RoundTripper, query, evalTime string, p printer) int {
+	api, err := newAPI(url, roundTripper, nil)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "error creating API client:", err)
+		return failureExitCode
+	}
+
+	eTime := time.Now()
+	if evalTime != "" {
+		eTime, err = parseTime(evalTime)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, "error parsing evaluation time:", err)
+			return failureExitCode
+		}
+	}
+
+	// Run query against client.
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	val, _, err := api.Query(ctx, query, eTime) // Ignoring warnings for now.
+	cancel()
+	if err != nil {
+		return handleAPIError(err)
+	}
+
+	p.printValue(val)
+
+	return successExitCode
+}
+
+// QueryRange performs a range query against a Prometheus server.
+func QueryRange(url *url.URL, roundTripper http.RoundTripper, headers map[string]string, query, start, end string, step time.Duration, p printer) int {
+	api, err := newAPI(url, roundTripper, headers)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "error creating API client:", err)
+		return failureExitCode
+	}
+
+	var stime, etime time.Time
+
+	if end == "" {
+		etime = time.Now()
+	} else {
+		etime, err = parseTime(end)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, "error parsing end time:", err)
+			return failureExitCode
+		}
+	}
+
+	if start == "" {
+		stime = etime.Add(-5 * time.Minute)
+	} else {
+		stime, err = parseTime(start)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, "error parsing start time:", err)
+			return failureExitCode
+		}
+	}
+
+	if !stime.Before(etime) {
+		fmt.Fprintln(os.Stderr, "start time is not before end time")
+		return failureExitCode
+	}
+
+	if step == 0 {
+		resolution := math.Max(math.Floor(etime.Sub(stime).Seconds()/250), 1)
+		// Convert seconds to nanoseconds such that time.Duration parses correctly.
+		step = time.Duration(resolution) * time.Second
+	}
+
+	// Run query against client.
+	r := v1.Range{Start: stime, End: etime, Step: step}
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	val, _, err := api.QueryRange(ctx, query, r) // Ignoring warnings for now.
+	cancel()
+
+	if err != nil {
+		return handleAPIError(err)
+	}
+
+	p.printValue(val)
+	return successExitCode
+}
+
+// QuerySeries queries for a series against a Prometheus server.
+func QuerySeries(url *url.URL, roundTripper http.RoundTripper, matchers []string, start, end string, p printer) int {
+	api, err := newAPI(url, roundTripper, nil)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "error creating API client:", err)
+		return failureExitCode
+	}
+
+	stime, etime, err := parseStartTimeAndEndTime(start, end)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		return failureExitCode
+	}
+
+	// Run query against client.
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	val, _, err := api.Series(ctx, matchers, stime, etime) // Ignoring warnings for now.
+	cancel()
+
+	if err != nil {
+		return handleAPIError(err)
+	}
+
+	p.printSeries(val)
+	return successExitCode
+}
+
+// QueryLabels queries for label values against a Prometheus server.
+func QueryLabels(url *url.URL, roundTripper http.RoundTripper, matchers []string, name, start, end string, p printer) int {
+	api, err := newAPI(url, roundTripper, nil)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "error creating API client:", err)
+		return failureExitCode
+	}
+
+	stime, etime, err := parseStartTimeAndEndTime(start, end)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		return failureExitCode
+	}
+
+	// Run query against client.
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	val, warn, err := api.LabelValues(ctx, name, matchers, stime, etime)
+	cancel()
+
+	for _, v := range warn {
+		fmt.Fprintln(os.Stderr, "query warning:", v)
+	}
+	if err != nil {
+		return handleAPIError(err)
+	}
+
+	p.printLabelValues(val)
+	return successExitCode
+}
+
+func handleAPIError(err error) int {
+	var apiErr *v1.Error
+	if errors.As(err, &apiErr) && apiErr.Detail != "" {
+		fmt.Fprintf(os.Stderr, "query error: %v (detail: %s)\n", apiErr, strings.TrimSpace(apiErr.Detail))
+	} else {
+		fmt.Fprintln(os.Stderr, "query error:", err)
+	}
+
+	return failureExitCode
+}
+
+func parseStartTimeAndEndTime(start, end string) (time.Time, time.Time, error) {
+	var (
+		minTime = time.Now().Add(-9999 * time.Hour)
+		maxTime = time.Now().Add(9999 * time.Hour)
+		err     error
+	)
+
+	stime := minTime
+	etime := maxTime
+
+	if start != "" {
+		stime, err = parseTime(start)
+		if err != nil {
+			return stime, etime, fmt.Errorf("error parsing start time: %w", err)
+		}
+	}
+
+	if end != "" {
+		etime, err = parseTime(end)
+		if err != nil {
+			return stime, etime, fmt.Errorf("error parsing end time: %w", err)
+		}
+	}
+	return stime, etime, nil
+}
+
+func parseTime(s string) (time.Time, error) {
+	if t, err := strconv.ParseFloat(s, 64); err == nil {
+		s, ns := math.Modf(t)
+		return time.Unix(int64(s), int64(ns*float64(time.Second))).UTC(), nil
+	}
+	if t, err := time.Parse(time.RFC3339Nano, s); err == nil {
+		return t, nil
+	}
+	return time.Time{}, fmt.Errorf("cannot parse %q to a valid timestamp", s)
+}

cmd/promtool/sd.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/discovery"
@@ -37,7 +38,7 @@ type sdCheckResult struct {
 }
 
 // CheckSD performs service discovery for the given job name and reports the results.
-func CheckSD(sdConfigFiles, sdJobName string, sdTimeout time.Duration, noDefaultScrapePort bool) int {
+func CheckSD(sdConfigFiles, sdJobName string, sdTimeout time.Duration, noDefaultScrapePort bool, registerer prometheus.Registerer) int {
 	logger := log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr))
 
 	cfg, err := config.LoadFile(sdConfigFiles, false, false, logger)
@@ -77,7 +78,7 @@ func CheckSD(sdConfigFiles, sdJobName string, sdTimeout time.Duration, noDefault
 	defer cancel()
 
 	for _, cfg := range scrapeConfig.ServiceDiscoveryConfigs {
-		d, err := cfg.NewDiscoverer(discovery.DiscovererOptions{Logger: logger})
+		d, err := cfg.NewDiscoverer(discovery.DiscovererOptions{Logger: logger, Registerer: registerer})
 		if err != nil {
 			fmt.Fprintln(os.Stderr, "Could not create new discoverer", err)
 			return failureExitCode

cmd/promtool/testdata/dump-test-1.prom

@@ -0,0 +1,15 @@
+{__name__="heavy_metric", foo="bar"} 5 0
+{__name__="heavy_metric", foo="bar"} 4 60000
+{__name__="heavy_metric", foo="bar"} 3 120000
+{__name__="heavy_metric", foo="bar"} 2 180000
+{__name__="heavy_metric", foo="bar"} 1 240000
+{__name__="heavy_metric", foo="foo"} 5 0
+{__name__="heavy_metric", foo="foo"} 4 60000
+{__name__="heavy_metric", foo="foo"} 3 120000
+{__name__="heavy_metric", foo="foo"} 2 180000
+{__name__="heavy_metric", foo="foo"} 1 240000
+{__name__="metric", baz="abc", foo="bar"} 1 0
+{__name__="metric", baz="abc", foo="bar"} 2 60000
+{__name__="metric", baz="abc", foo="bar"} 3 120000
+{__name__="metric", baz="abc", foo="bar"} 4 180000
+{__name__="metric", baz="abc", foo="bar"} 5 240000

cmd/promtool/testdata/dump-test-2.prom

@@ -0,0 +1,10 @@
+{__name__="heavy_metric", foo="foo"} 5 0
+{__name__="heavy_metric", foo="foo"} 4 60000
+{__name__="heavy_metric", foo="foo"} 3 120000
+{__name__="heavy_metric", foo="foo"} 2 180000
+{__name__="heavy_metric", foo="foo"} 1 240000
+{__name__="metric", baz="abc", foo="bar"} 1 0
+{__name__="metric", baz="abc", foo="bar"} 2 60000
+{__name__="metric", baz="abc", foo="bar"} 3 120000
+{__name__="metric", baz="abc", foo="bar"} 4 180000
+{__name__="metric", baz="abc", foo="bar"} 5 240000

cmd/promtool/testdata/dump-test-3.prom

@@ -0,0 +1,2 @@
+{__name__="metric", baz="abc", foo="bar"} 2 60000
+{__name__="metric", baz="abc", foo="bar"} 3 120000

cmd/promtool/tsdb.go

@@ -706,7 +706,7 @@ func analyzeCompaction(ctx context.Context, block tsdb.BlockReader, indexr tsdb.
 	return nil
 }
 
-func dumpSamples(ctx context.Context, path string, mint, maxt int64, match string) (err error) {
+func dumpSamples(ctx context.Context, path string, mint, maxt int64, match []string) (err error) {
 	db, err := tsdb.OpenDBReadOnly(path, nil)
 	if err != nil {
 		return err
@@ -720,11 +720,21 @@ func dumpSamples(ctx context.Context, path string, mint, maxt int64, match strin
 	}
 	defer q.Close()
 
-	matchers, err := parser.ParseMetricSelector(match)
+	matcherSets, err := parser.ParseMetricSelectors(match)
 	if err != nil {
 		return err
 	}
-	ss := q.Select(ctx, false, nil, matchers...)
+
+	var ss storage.SeriesSet
+	if len(matcherSets) > 1 {
+		var sets []storage.SeriesSet
+		for _, mset := range matcherSets {
+			sets = append(sets, q.Select(ctx, true, nil, mset...))
+		}
+		ss = storage.NewMergeSeriesSet(sets, storage.ChainedSeriesMerge)
+	} else {
+		ss = q.Select(ctx, false, nil, matcherSets[0]...)
+	}
 
 	for ss.Next() {
 		series := ss.At()

cmd/promtool/tsdb_test.go

@@ -14,9 +14,18 @@
 package main
 
 import (
+	"bytes"
+	"context"
+	"io"
+	"math"
+	"os"
+	"runtime"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/prometheus/promql"
 )
 
 func TestGenerateBucket(t *testing.T) {
@@ -41,3 +50,101 @@ func TestGenerateBucket(t *testing.T) {
 		require.Equal(t, tc.step, step)
 	}
 }
+
+// getDumpedSamples dumps samples and returns them.
+func getDumpedSamples(t *testing.T, path string, mint, maxt int64, match []string) string {
+	t.Helper()
+
+	oldStdout := os.Stdout
+	r, w, _ := os.Pipe()
+	os.Stdout = w
+
+	err := dumpSamples(
+		context.Background(),
+		path,
+		mint,
+		maxt,
+		match,
+	)
+	require.NoError(t, err)
+
+	w.Close()
+	os.Stdout = oldStdout
+
+	var buf bytes.Buffer
+	io.Copy(&buf, r)
+	return buf.String()
+}
+
+func TestTSDBDump(t *testing.T) {
+	storage := promql.LoadedStorage(t, `
+		load 1m
+			metric{foo="bar", baz="abc"} 1 2 3 4 5
+			heavy_metric{foo="bar"} 5 4 3 2 1
+			heavy_metric{foo="foo"} 5 4 3 2 1
+	`)
+
+	tests := []struct {
+		name         string
+		mint         int64
+		maxt         int64
+		match        []string
+		expectedDump string
+	}{
+		{
+			name:         "default match",
+			mint:         math.MinInt64,
+			maxt:         math.MaxInt64,
+			match:        []string{"{__name__=~'(?s:.*)'}"},
+			expectedDump: "testdata/dump-test-1.prom",
+		},
+		{
+			name:         "same matcher twice",
+			mint:         math.MinInt64,
+			maxt:         math.MaxInt64,
+			match:        []string{"{foo=~'.+'}", "{foo=~'.+'}"},
+			expectedDump: "testdata/dump-test-1.prom",
+		},
+		{
+			name:         "no duplication",
+			mint:         math.MinInt64,
+			maxt:         math.MaxInt64,
+			match:        []string{"{__name__=~'(?s:.*)'}", "{baz='abc'}"},
+			expectedDump: "testdata/dump-test-1.prom",
+		},
+		{
+			name:         "well merged",
+			mint:         math.MinInt64,
+			maxt:         math.MaxInt64,
+			match:        []string{"{__name__='heavy_metric'}", "{baz='abc'}"},
+			expectedDump: "testdata/dump-test-1.prom",
+		},
+		{
+			name:         "multi matchers",
+			mint:         math.MinInt64,
+			maxt:         math.MaxInt64,
+			match:        []string{"{__name__='heavy_metric',foo='foo'}", "{__name__='metric'}"},
+			expectedDump: "testdata/dump-test-2.prom",
+		},
+		{
+			name:         "with reduced mint and maxt",
+			mint:         int64(60000),
+			maxt:         int64(120000),
+			match:        []string{"{__name__='metric'}"},
+			expectedDump: "testdata/dump-test-3.prom",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			dumpedMetrics := getDumpedSamples(t, storage.Dir(), tt.mint, tt.maxt, tt.match)
+			expectedMetrics, err := os.ReadFile(tt.expectedDump)
+			require.NoError(t, err)
+			if strings.Contains(runtime.GOOS, "windows") {
+				// We use "/n" while dumping on windows as well.
+				expectedMetrics = bytes.ReplaceAll(expectedMetrics, []byte("\r\n"), []byte("\n"))
+			}
+			// even though in case of one matcher samples are not sorted, the order in the cases above should stay the same.
+			require.Equal(t, string(expectedMetrics), dumpedMetrics)
+		})
+	}
+}

config/config.go

@@ -454,12 +454,19 @@ var (
 		OpenMetricsText1_0_0: "application/openmetrics-text;version=1.0.0",
 	}
 
+	// DefaultScrapeProtocols is the set of scrape protocols that will be proposed
+	// to scrape target, ordered by priority.
 	DefaultScrapeProtocols = []ScrapeProtocol{
 		OpenMetricsText1_0_0,
 		OpenMetricsText0_0_1,
 		PrometheusText0_0_4,
 	}
-	DefaultNativeHistogramScrapeProtocols = []ScrapeProtocol{
+
+	// DefaultProtoFirstScrapeProtocols is like DefaultScrapeProtocols, but it
+	// favors protobuf Prometheus exposition format.
+	// Used by default for certain feature-flags like
+	// "native-histograms" and "created-timestamp-zero-ingestion".
+	DefaultProtoFirstScrapeProtocols = []ScrapeProtocol{
 		PrometheusProto,
 		OpenMetricsText1_0_0,
 		OpenMetricsText0_0_1,
@@ -1117,6 +1124,9 @@ type QueueConfig struct {
 	MinBackoff       model.Duration `yaml:"min_backoff,omitempty"`
 	MaxBackoff       model.Duration `yaml:"max_backoff,omitempty"`
 	RetryOnRateLimit bool           `yaml:"retry_on_http_429,omitempty"`
+
+	// Samples older than the limit will be dropped.
+	SampleAgeLimit model.Duration `yaml:"sample_age_limit,omitempty"`
 }
 
 // MetadataConfig is the configuration for sending metadata to remote

config/config_test.go

@@ -568,6 +568,7 @@ var expectedConf = &Config{
 			ServiceDiscoveryConfigs: discovery.Configs{
 				&xds.KumaSDConfig{
 					Server:           "http://kuma-control-plane.kuma-system.svc:5676",
+					ClientID:         "main-prometheus",
 					HTTPClientConfig: config.DefaultHTTPClientConfig,
 					RefreshInterval:  model.Duration(15 * time.Second),
 					FetchTimeout:     model.Duration(2 * time.Minute),

config/testdata/conf.good.yml

@@ -221,6 +221,7 @@ scrape_configs:
 
     kuma_sd_configs:
       - server: http://kuma-control-plane.kuma-system.svc:5676
+        client_id: main-prometheus
 
   - job_name: service-marathon
     marathon_sd_configs:

config/testdata/roundtrip.good.yml

@@ -108,6 +108,7 @@ scrape_configs:
 
     kuma_sd_configs:
       - server: http://kuma-control-plane.kuma-system.svc:5676
+        client_id: main-prometheus
 
     marathon_sd_configs:
       - servers:

discovery/README.md

@@ -234,6 +234,11 @@ type Config interface {
 
 type DiscovererOptions struct {
 	Logger log.Logger
+
+	// A registerer for the Discoverer's metrics.
+	Registerer prometheus.Registerer
+	
+	HTTPClientOptions []config.HTTPClientOption
 }
 ```
 

discovery/aws/ec2.go

@@ -30,6 +30,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -101,7 +102,7 @@ func (*EC2SDConfig) Name() string { return "ec2" }
 
 // NewDiscoverer returns a Discoverer for the EC2 Config.
 func (c *EC2SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewEC2Discovery(c, opts.Logger), nil
+	return NewEC2Discovery(c, opts.Logger, opts.Registerer), nil
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface for the EC2 Config.
@@ -147,7 +148,7 @@ type EC2Discovery struct {
 }
 
 // NewEC2Discovery returns a new EC2Discovery which periodically refreshes its targets.
-func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger) *EC2Discovery {
+func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger, reg prometheus.Registerer) *EC2Discovery {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -156,10 +157,13 @@ func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger) *EC2Discovery {
 		cfg:    conf,
 	}
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"ec2",
-		time.Duration(d.cfg.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "ec2",
+			Interval: time.Duration(d.cfg.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+		},
 	)
 	return d
 }

discovery/aws/lightsail.go

@@ -29,6 +29,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/lightsail"
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -84,7 +85,7 @@ func (*LightsailSDConfig) Name() string { return "lightsail" }
 
 // NewDiscoverer returns a Discoverer for the Lightsail Config.
 func (c *LightsailSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewLightsailDiscovery(c, opts.Logger), nil
+	return NewLightsailDiscovery(c, opts.Logger, opts.Registerer), nil
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface for the Lightsail Config.
@@ -121,7 +122,7 @@ type LightsailDiscovery struct {
 }
 
 // NewLightsailDiscovery returns a new LightsailDiscovery which periodically refreshes its targets.
-func NewLightsailDiscovery(conf *LightsailSDConfig, logger log.Logger) *LightsailDiscovery {
+func NewLightsailDiscovery(conf *LightsailSDConfig, logger log.Logger, reg prometheus.Registerer) *LightsailDiscovery {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -129,10 +130,13 @@ func NewLightsailDiscovery(conf *LightsailSDConfig, logger log.Logger) *Lightsai
 		cfg: conf,
 	}
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"lightsail",
-		time.Duration(d.cfg.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "lightsail",
+			Interval: time.Duration(d.cfg.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+		},
 	)
 	return d
 }

discovery/azure/azure.go

@@ -30,8 +30,8 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
 	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
-	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4"
-	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4"
 	cache "github.com/Code-Hex/go-generics-cache"
 	"github.com/Code-Hex/go-generics-cache/policy/lru"
 	"github.com/go-kit/log"
@@ -79,17 +79,6 @@ var (
 		AuthenticationMethod: authMethodOAuth,
 		HTTPClientConfig:     config_util.DefaultHTTPClientConfig,
 	}
-
-	failuresCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_azure_failures_total",
-			Help: "Number of Azure service discovery refresh failures.",
-		})
-	cacheHitCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_azure_cache_hit_total",
-			Help: "Number of cache hit during refresh.",
-		})
 )
 
 var environments = map[string]cloud.Configuration{
@@ -114,8 +103,6 @@ func CloudConfigurationFromName(name string) (cloud.Configuration, error) {
 
 func init() {
 	discovery.RegisterConfig(&SDConfig{})
-	prometheus.MustRegister(failuresCount)
-	prometheus.MustRegister(cacheHitCount)
 }
 
 // SDConfig is the configuration for Azure based service discovery.
@@ -138,7 +125,7 @@ func (*SDConfig) Name() string { return "azure" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger), nil
+	return NewDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 func validateAuthParam(param, name string) error {
@@ -181,14 +168,16 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 
 type Discovery struct {
 	*refresh.Discovery
-	logger log.Logger
-	cfg    *SDConfig
-	port   int
-	cache  *cache.Cache[string, *armnetwork.Interface]
+	logger        log.Logger
+	cfg           *SDConfig
+	port          int
+	cache         *cache.Cache[string, *armnetwork.Interface]
+	failuresCount prometheus.Counter
+	cacheHitCount prometheus.Counter
 }
 
 // NewDiscovery returns a new AzureDiscovery which periodically refreshes its targets.
-func NewDiscovery(cfg *SDConfig, logger log.Logger) *Discovery {
+func NewDiscovery(cfg *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -198,16 +187,30 @@ func NewDiscovery(cfg *SDConfig, logger log.Logger) *Discovery {
 		port:   cfg.Port,
 		logger: logger,
 		cache:  l,
+		failuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_azure_failures_total",
+				Help: "Number of Azure service discovery refresh failures.",
+			}),
+		cacheHitCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_azure_cache_hit_total",
+				Help: "Number of cache hit during refresh.",
+			}),
 	}
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"azure",
-		time.Duration(cfg.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "azure",
+			Interval: time.Duration(cfg.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+			Metrics:  []prometheus.Collector{d.failuresCount, d.cacheHitCount},
+		},
 	)
 
-	return d
+	return d, nil
 }
 
 // azureClient represents multiple Azure Resource Manager providers.
@@ -330,14 +333,14 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 
 	client, err := createAzureClient(*d.cfg)
 	if err != nil {
-		failuresCount.Inc()
+		d.failuresCount.Inc()
 		return nil, fmt.Errorf("could not create Azure client: %w", err)
 	}
 	client.logger = d.logger
 
 	machines, err := client.getVMs(ctx, d.cfg.ResourceGroup)
 	if err != nil {
-		failuresCount.Inc()
+		d.failuresCount.Inc()
 		return nil, fmt.Errorf("could not get virtual machines: %w", err)
 	}
 
@@ -346,14 +349,14 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 	// Load the vms managed by scale sets.
 	scaleSets, err := client.getScaleSets(ctx, d.cfg.ResourceGroup)
 	if err != nil {
-		failuresCount.Inc()
+		d.failuresCount.Inc()
 		return nil, fmt.Errorf("could not get virtual machine scale sets: %w", err)
 	}
 
 	for _, scaleSet := range scaleSets {
 		scaleSetVms, err := client.getScaleSetVMs(ctx, scaleSet)
 		if err != nil {
-			failuresCount.Inc()
+			d.failuresCount.Inc()
 			return nil, fmt.Errorf("could not get virtual machine scale set vms: %w", err)
 		}
 		machines = append(machines, scaleSetVms...)
@@ -404,18 +407,18 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 				var networkInterface *armnetwork.Interface
 				if v, ok := d.getFromCache(nicID); ok {
 					networkInterface = v
-					cacheHitCount.Add(1)
+					d.cacheHitCount.Add(1)
 				} else {
 					if vm.ScaleSet == "" {
 						networkInterface, err = client.getVMNetworkInterfaceByID(ctx, nicID)
-						if err != nil {
-							if errors.Is(err, errorNotFound) {
-								level.Warn(d.logger).Log("msg", "Network interface does not exist", "name", nicID, "err", err)
-							} else {
-								ch <- target{labelSet: nil, err: err}
-							}
-							// Get out of this routine because we cannot continue without a network interface.
-							return
+					} else {
+						networkInterface, err = client.getVMScaleSetVMNetworkInterfaceByID(ctx, nicID, vm.ScaleSet, vm.InstanceID)
+					}
+					if err != nil {
+						if errors.Is(err, errorNotFound) {
+							level.Warn(d.logger).Log("msg", "Network interface does not exist", "name", nicID, "err", err)
+						} else {
+							ch <- target{labelSet: nil, err: err}
 						}
 						d.addToCache(nicID, networkInterface)
 					} else {
@@ -477,7 +480,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 	var tg targetgroup.Group
 	for tgt := range ch {
 		if tgt.err != nil {
-			failuresCount.Inc()
+			d.failuresCount.Inc()
 			return nil, fmt.Errorf("unable to complete Azure service discovery: %w", tgt.err)
 		}
 		if tgt.labelSet != nil {

discovery/azure/azure_test.go

@@ -17,7 +17,7 @@ import (
 	"testing"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/arm"
-	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
 )

discovery/consul/consul.go

@@ -71,41 +71,18 @@ const (
 	namespace = "prometheus"
 )
 
-var (
-	rpcFailuresCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Namespace: namespace,
-			Name:      "sd_consul_rpc_failures_total",
-			Help:      "The number of Consul RPC call failures.",
-		})
-	rpcDuration = prometheus.NewSummaryVec(
-		prometheus.SummaryOpts{
-			Namespace:  namespace,
-			Name:       "sd_consul_rpc_duration_seconds",
-			Help:       "The duration of a Consul RPC call in seconds.",
-			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
-		},
-		[]string{"endpoint", "call"},
-	)
-
-	// Initialize metric vectors.
-	servicesRPCDuration = rpcDuration.WithLabelValues("catalog", "services")
-	serviceRPCDuration  = rpcDuration.WithLabelValues("catalog", "service")
-
-	// DefaultSDConfig is the default Consul SD configuration.
-	DefaultSDConfig = SDConfig{
-		TagSeparator:     ",",
-		Scheme:           "http",
-		Server:           "localhost:8500",
-		AllowStale:       true,
-		RefreshInterval:  model.Duration(30 * time.Second),
-		HTTPClientConfig: config.DefaultHTTPClientConfig,
-	}
-)
+// DefaultSDConfig is the default Consul SD configuration.
+var DefaultSDConfig = SDConfig{
+	TagSeparator:     ",",
+	Scheme:           "http",
+	Server:           "localhost:8500",
+	AllowStale:       true,
+	RefreshInterval:  model.Duration(30 * time.Second),
+	HTTPClientConfig: config.DefaultHTTPClientConfig,
+}
 
 func init() {
 	discovery.RegisterConfig(&SDConfig{})
-	prometheus.MustRegister(rpcFailuresCount, rpcDuration)
 }
 
 // SDConfig is the configuration for Consul service discovery.
@@ -147,7 +124,7 @@ func (*SDConfig) Name() string { return "consul" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -184,22 +161,27 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 // Discovery retrieves target information from a Consul server
 // and updates them via watches.
 type Discovery struct {
-	client           *consul.Client
-	clientDatacenter string
-	clientNamespace  string
-	clientPartition  string
-	tagSeparator     string
-	watchedServices  []string // Set of services which will be discovered.
-	watchedTags      []string // Tags used to filter instances of a service.
-	watchedNodeMeta  map[string]string
-	allowStale       bool
-	refreshInterval  time.Duration
-	finalizer        func()
-	logger           log.Logger
+	client              *consul.Client
+	clientDatacenter    string
+	clientNamespace     string
+	clientPartition     string
+	tagSeparator        string
+	watchedServices     []string // Set of services which will be discovered.
+	watchedTags         []string // Tags used to filter instances of a service.
+	watchedNodeMeta     map[string]string
+	allowStale          bool
+	refreshInterval     time.Duration
+	finalizer           func()
+	logger              log.Logger
+	rpcFailuresCount    prometheus.Counter
+	rpcDuration         *prometheus.SummaryVec
+	servicesRPCDuration prometheus.Observer
+	serviceRPCDuration  prometheus.Observer
+	metricRegisterer    discovery.MetricRegisterer
 }
 
 // NewDiscovery returns a new Discovery for the given config.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -237,7 +219,35 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 		clientPartition:  conf.Partition,
 		finalizer:        wrapper.CloseIdleConnections,
 		logger:           logger,
+		rpcFailuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      "sd_consul_rpc_failures_total",
+				Help:      "The number of Consul RPC call failures.",
+			}),
+		rpcDuration: prometheus.NewSummaryVec(
+			prometheus.SummaryOpts{
+				Namespace:  namespace,
+				Name:       "sd_consul_rpc_duration_seconds",
+				Help:       "The duration of a Consul RPC call in seconds.",
+				Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+			},
+			[]string{"endpoint", "call"},
+		),
 	}
+
+	cd.metricRegisterer = discovery.NewMetricRegisterer(
+		reg,
+		[]prometheus.Collector{
+			cd.rpcFailuresCount,
+			cd.rpcDuration,
+		},
+	)
+
+	// Initialize metric vectors.
+	cd.servicesRPCDuration = cd.rpcDuration.WithLabelValues("catalog", "services")
+	cd.serviceRPCDuration = cd.rpcDuration.WithLabelValues("catalog", "service")
+
 	return cd, nil
 }
 
@@ -293,7 +303,7 @@ func (d *Discovery) getDatacenter() error {
 	info, err := d.client.Agent().Self()
 	if err != nil {
 		level.Error(d.logger).Log("msg", "Error retrieving datacenter name", "err", err)
-		rpcFailuresCount.Inc()
+		d.rpcFailuresCount.Inc()
 		return err
 	}
 
@@ -334,6 +344,13 @@ func (d *Discovery) initialize(ctx context.Context) {
 
 // Run implements the Discoverer interface.
 func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
+	err := d.metricRegisterer.RegisterMetrics()
+	if err != nil {
+		level.Error(d.logger).Log("msg", "Unable to register metrics", "err", err.Error())
+		return
+	}
+	defer d.metricRegisterer.UnregisterMetrics()
+
 	if d.finalizer != nil {
 		defer d.finalizer()
 	}
@@ -382,7 +399,7 @@ func (d *Discovery) watchServices(ctx context.Context, ch chan<- []*targetgroup.
 	t0 := time.Now()
 	srvs, meta, err := catalog.Services(opts.WithContext(ctx))
 	elapsed := time.Since(t0)
-	servicesRPCDuration.Observe(elapsed.Seconds())
+	d.servicesRPCDuration.Observe(elapsed.Seconds())
 
 	// Check the context before in order to exit early.
 	select {
@@ -393,7 +410,7 @@ func (d *Discovery) watchServices(ctx context.Context, ch chan<- []*targetgroup.
 
 	if err != nil {
 		level.Error(d.logger).Log("msg", "Error refreshing service list", "err", err)
-		rpcFailuresCount.Inc()
+		d.rpcFailuresCount.Inc()
 		time.Sleep(retryInterval)
 		return
 	}
@@ -449,13 +466,15 @@ func (d *Discovery) watchServices(ctx context.Context, ch chan<- []*targetgroup.
 
 // consulService contains data belonging to the same service.
 type consulService struct {
-	name         string
-	tags         []string
-	labels       model.LabelSet
-	discovery    *Discovery
-	client       *consul.Client
-	tagSeparator string
-	logger       log.Logger
+	name               string
+	tags               []string
+	labels             model.LabelSet
+	discovery          *Discovery
+	client             *consul.Client
+	tagSeparator       string
+	logger             log.Logger
+	rpcFailuresCount   prometheus.Counter
+	serviceRPCDuration prometheus.Observer
 }
 
 // Start watching a service.
@@ -469,8 +488,10 @@ func (d *Discovery) watchService(ctx context.Context, ch chan<- []*targetgroup.G
 			serviceLabel:    model.LabelValue(name),
 			datacenterLabel: model.LabelValue(d.clientDatacenter),
 		},
-		tagSeparator: d.tagSeparator,
-		logger:       d.logger,
+		tagSeparator:       d.tagSeparator,
+		logger:             d.logger,
+		rpcFailuresCount:   d.rpcFailuresCount,
+		serviceRPCDuration: d.serviceRPCDuration,
 	}
 
 	go func() {
@@ -508,7 +529,7 @@ func (srv *consulService) watch(ctx context.Context, ch chan<- []*targetgroup.Gr
 	t0 := time.Now()
 	serviceNodes, meta, err := health.ServiceMultipleTags(srv.name, srv.tags, false, opts.WithContext(ctx))
 	elapsed := time.Since(t0)
-	serviceRPCDuration.Observe(elapsed.Seconds())
+	srv.serviceRPCDuration.Observe(elapsed.Seconds())
 
 	// Check the context before in order to exit early.
 	select {
@@ -520,7 +541,7 @@ func (srv *consulService) watch(ctx context.Context, ch chan<- []*targetgroup.Gr
 
 	if err != nil {
 		level.Error(srv.logger).Log("msg", "Error refreshing service", "service", srv.name, "tags", strings.Join(srv.tags, ","), "err", err)
-		rpcFailuresCount.Inc()
+		srv.rpcFailuresCount.Inc()
 		time.Sleep(retryInterval)
 		return
 	}

discovery/consul/consul_test.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
@@ -39,7 +40,7 @@ func TestConfiguredService(t *testing.T) {
 	conf := &SDConfig{
 		Services: []string{"configuredServiceName"},
 	}
-	consulDiscovery, err := NewDiscovery(conf, nil)
+	consulDiscovery, err := NewDiscovery(conf, nil, prometheus.NewRegistry())
 	if err != nil {
 		t.Errorf("Unexpected error when initializing discovery %v", err)
 	}
@@ -56,7 +57,7 @@ func TestConfiguredServiceWithTag(t *testing.T) {
 		Services:    []string{"configuredServiceName"},
 		ServiceTags: []string{"http"},
 	}
-	consulDiscovery, err := NewDiscovery(conf, nil)
+	consulDiscovery, err := NewDiscovery(conf, nil, prometheus.NewRegistry())
 	if err != nil {
 		t.Errorf("Unexpected error when initializing discovery %v", err)
 	}
@@ -151,7 +152,7 @@ func TestConfiguredServiceWithTags(t *testing.T) {
 	}
 
 	for _, tc := range cases {
-		consulDiscovery, err := NewDiscovery(tc.conf, nil)
+		consulDiscovery, err := NewDiscovery(tc.conf, nil, prometheus.NewRegistry())
 		if err != nil {
 			t.Errorf("Unexpected error when initializing discovery %v", err)
 		}
@@ -165,7 +166,7 @@ func TestConfiguredServiceWithTags(t *testing.T) {
 
 func TestNonConfiguredService(t *testing.T) {
 	conf := &SDConfig{}
-	consulDiscovery, err := NewDiscovery(conf, nil)
+	consulDiscovery, err := NewDiscovery(conf, nil, prometheus.NewRegistry())
 	if err != nil {
 		t.Errorf("Unexpected error when initializing discovery %v", err)
 	}
@@ -262,7 +263,7 @@ func newServer(t *testing.T) (*httptest.Server, *SDConfig) {
 
 func newDiscovery(t *testing.T, config *SDConfig) *Discovery {
 	logger := log.NewNopLogger()
-	d, err := NewDiscovery(config, logger)
+	d, err := NewDiscovery(config, logger, prometheus.NewRegistry())
 	require.NoError(t, err)
 	return d
 }

discovery/digitalocean/digitalocean.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/digitalocean/godo"
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/version"
@@ -75,7 +76,7 @@ func (*SDConfig) Name() string { return "digitalocean" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -103,7 +104,7 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	d := &Discovery{
 		port: conf.Port,
 	}
@@ -125,10 +126,13 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 	}
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"digitalocean",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "digitalocean",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+		},
 	)
 	return d, nil
 }

discovery/digitalocean/digitalocean_test.go

@@ -20,6 +20,7 @@ import (
 	"testing"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 )
@@ -46,7 +47,7 @@ func TestDigitalOceanSDRefresh(t *testing.T) {
 
 	cfg := DefaultSDConfig
 	cfg.HTTPClientConfig.BearerToken = tokenID
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 	require.NoError(t, err)
 	endpoint, err := url.Parse(sdmock.Mock.Endpoint())
 	require.NoError(t, err)

discovery/discovery.go

@@ -18,6 +18,7 @@ import (
 	"reflect"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 
 	"github.com/prometheus/prometheus/discovery/targetgroup"
@@ -42,6 +43,15 @@ type Discoverer interface {
 type DiscovererOptions struct {
 	Logger log.Logger
 
+	// A registerer for the Discoverer's metrics.
+	// Some Discoverers may ignore this registerer and use the global one instead.
+	// For now this will work, because the Prometheus `main` function uses the global registry.
+	// However, in the future the Prometheus `main` function will be updated to not use the global registry.
+	// Hence, if a discoverer wants its metrics to be visible via the Prometheus executable's
+	// `/metrics` endpoint, it should use this explicit registerer.
+	// TODO(ptodev): Update this comment once the Prometheus `main` function does not use the global registry.
+	Registerer prometheus.Registerer
+
 	// Extra HTTP client options to expose to Discoverers. This field may be
 	// ignored; Discoverer implementations must opt-in to reading it.
 	HTTPClientOptions []config.HTTPClientOption

discovery/dns/dns.go

@@ -49,30 +49,14 @@ const (
 	namespace = "prometheus"
 )
 
-var (
-	dnsSDLookupsCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Namespace: namespace,
-			Name:      "sd_dns_lookups_total",
-			Help:      "The number of DNS-SD lookups.",
-		})
-	dnsSDLookupFailuresCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Namespace: namespace,
-			Name:      "sd_dns_lookup_failures_total",
-			Help:      "The number of DNS-SD lookup failures.",
-		})
-
-	// DefaultSDConfig is the default DNS SD configuration.
-	DefaultSDConfig = SDConfig{
-		RefreshInterval: model.Duration(30 * time.Second),
-		Type:            "SRV",
-	}
-)
+// DefaultSDConfig is the default DNS SD configuration.
+var DefaultSDConfig = SDConfig{
+	RefreshInterval: model.Duration(30 * time.Second),
+	Type:            "SRV",
+}
 
 func init() {
 	discovery.RegisterConfig(&SDConfig{})
-	prometheus.MustRegister(dnsSDLookupFailuresCount, dnsSDLookupsCount)
 }
 
 // SDConfig is the configuration for DNS based service discovery.
@@ -88,7 +72,7 @@ func (*SDConfig) Name() string { return "dns" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(*c, opts.Logger), nil
+	return NewDiscovery(*c, opts.Logger, opts.Registerer)
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
@@ -118,16 +102,18 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 // the Discoverer interface.
 type Discovery struct {
 	*refresh.Discovery
-	names  []string
-	port   int
-	qtype  uint16
-	logger log.Logger
+	names                    []string
+	port                     int
+	qtype                    uint16
+	logger                   log.Logger
+	dnsSDLookupsCount        prometheus.Counter
+	dnsSDLookupFailuresCount prometheus.Counter
 
 	lookupFn func(name string, qtype uint16, logger log.Logger) (*dns.Msg, error)
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf SDConfig, logger log.Logger) *Discovery {
+func NewDiscovery(conf SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -151,14 +137,32 @@ func NewDiscovery(conf SDConfig, logger log.Logger) *Discovery {
 		port:     conf.Port,
 		logger:   logger,
 		lookupFn: lookupWithSearchPath,
+		dnsSDLookupsCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      "sd_dns_lookups_total",
+				Help:      "The number of DNS-SD lookups.",
+			}),
+		dnsSDLookupFailuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      "sd_dns_lookup_failures_total",
+				Help:      "The number of DNS-SD lookup failures.",
+			}),
 	}
+
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"dns",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "dns",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: prometheus.NewRegistry(),
+			Metrics:  []prometheus.Collector{d.dnsSDLookupsCount, d.dnsSDLookupFailuresCount},
+		},
 	)
-	return d
+
+	return d, nil
 }
 
 func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
@@ -191,9 +195,9 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 
 func (d *Discovery) refreshOne(ctx context.Context, name string, ch chan<- *targetgroup.Group) error {
 	response, err := d.lookupFn(name, d.qtype, d.logger)
-	dnsSDLookupsCount.Inc()
+	d.dnsSDLookupsCount.Inc()
 	if err != nil {
-		dnsSDLookupFailuresCount.Inc()
+		d.dnsSDLookupFailuresCount.Inc()
 		return err
 	}
 

discovery/dns/dns_test.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/miekg/dns"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
@@ -252,7 +253,8 @@ func TestDNS(t *testing.T) {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
-			sd := NewDiscovery(tc.config, nil)
+			sd, err := NewDiscovery(tc.config, nil, prometheus.NewRegistry())
+			require.NoError(t, err)
 			sd.lookupFn = tc.lookup
 
 			tgs, err := sd.refresh(context.Background())

discovery/eureka/eureka.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -80,7 +81,7 @@ func (*SDConfig) Name() string { return "eureka" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -117,7 +118,7 @@ type Discovery struct {
 }
 
 // NewDiscovery creates a new Eureka discovery for the given role.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "eureka_sd")
 	if err != nil {
 		return nil, err
@@ -128,10 +129,13 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 		server: conf.Server,
 	}
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"eureka",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "eureka",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+		},
 	)
 	return d, nil
 }

discovery/eureka/eureka_test.go

@@ -20,6 +20,7 @@ import (
 	"net/http/httptest"
 	"testing"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 
@@ -35,7 +36,7 @@ func testUpdateServices(respHandler http.HandlerFunc) ([]*targetgroup.Group, err
 		Server: ts.URL,
 	}
 
-	md, err := NewDiscovery(&conf, nil)
+	md, err := NewDiscovery(&conf, nil, prometheus.NewRegistry())
 	if err != nil {
 		return nil, err
 	}

discovery/file/file.go

@@ -39,24 +39,6 @@ import (
 )
 
 var (
-	fileSDReadErrorsCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_file_read_errors_total",
-			Help: "The number of File-SD read errors.",
-		})
-	fileSDScanDuration = prometheus.NewSummary(
-		prometheus.SummaryOpts{
-			Name:       "prometheus_sd_file_scan_duration_seconds",
-			Help:       "The duration of the File-SD scan in seconds.",
-			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
-		})
-	fileSDTimeStamp        = NewTimestampCollector()
-	fileWatcherErrorsCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_file_watcher_errors_total",
-			Help: "The number of File-SD errors caused by filesystem watch failures.",
-		})
-
 	patFileSDName = regexp.MustCompile(`^[^*]*(\*[^/]*)?\.(json|yml|yaml|JSON|YML|YAML)$`)
 
 	// DefaultSDConfig is the default file SD configuration.
@@ -67,7 +49,6 @@ var (
 
 func init() {
 	discovery.RegisterConfig(&SDConfig{})
-	prometheus.MustRegister(fileSDReadErrorsCount, fileSDScanDuration, fileSDTimeStamp, fileWatcherErrorsCount)
 }
 
 // SDConfig is the configuration for file based discovery.
@@ -81,7 +62,7 @@ func (*SDConfig) Name() string { return "file" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger), nil
+	return NewDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -187,10 +168,17 @@ type Discovery struct {
 	// This is used to detect deleted target groups.
 	lastRefresh map[string]int
 	logger      log.Logger
+
+	fileSDReadErrorsCount  prometheus.Counter
+	fileSDScanDuration     prometheus.Summary
+	fileWatcherErrorsCount prometheus.Counter
+	fileSDTimeStamp        *TimestampCollector
+
+	metricRegisterer discovery.MetricRegisterer
 }
 
 // NewDiscovery returns a new file discovery for the given paths.
-func NewDiscovery(conf *SDConfig, logger log.Logger) *Discovery {
+func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -200,9 +188,35 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) *Discovery {
 		interval:   time.Duration(conf.RefreshInterval),
 		timestamps: make(map[string]float64),
 		logger:     logger,
+		fileSDReadErrorsCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_file_read_errors_total",
+				Help: "The number of File-SD read errors.",
+			}),
+		fileSDScanDuration: prometheus.NewSummary(
+			prometheus.SummaryOpts{
+				Name:       "prometheus_sd_file_scan_duration_seconds",
+				Help:       "The duration of the File-SD scan in seconds.",
+				Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+			}),
+		fileWatcherErrorsCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_file_watcher_errors_total",
+				Help: "The number of File-SD errors caused by filesystem watch failures.",
+			}),
+		fileSDTimeStamp: NewTimestampCollector(),
 	}
-	fileSDTimeStamp.addDiscoverer(disc)
-	return disc
+
+	disc.fileSDTimeStamp.addDiscoverer(disc)
+
+	disc.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
+		disc.fileSDReadErrorsCount,
+		disc.fileSDScanDuration,
+		disc.fileWatcherErrorsCount,
+		disc.fileSDTimeStamp,
+	})
+
+	return disc, nil
 }
 
 // listFiles returns a list of all files that match the configured patterns.
@@ -239,10 +253,17 @@ func (d *Discovery) watchFiles() {
 
 // Run implements the Discoverer interface.
 func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
+	err := d.metricRegisterer.RegisterMetrics()
+	if err != nil {
+		level.Error(d.logger).Log("msg", "Unable to register metrics", "err", err.Error())
+		return
+	}
+	defer d.metricRegisterer.UnregisterMetrics()
+
 	watcher, err := fsnotify.NewWatcher()
 	if err != nil {
 		level.Error(d.logger).Log("msg", "Error adding file watcher", "err", err)
-		fileWatcherErrorsCount.Inc()
+		d.fileWatcherErrorsCount.Inc()
 		return
 	}
 	d.watcher = watcher
@@ -306,7 +327,7 @@ func (d *Discovery) stop() {
 	done := make(chan struct{})
 	defer close(done)
 
-	fileSDTimeStamp.removeDiscoverer(d)
+	d.fileSDTimeStamp.removeDiscoverer(d)
 
 	// Closing the watcher will deadlock unless all events and errors are drained.
 	go func() {
@@ -332,13 +353,13 @@ func (d *Discovery) stop() {
 func (d *Discovery) refresh(ctx context.Context, ch chan<- []*targetgroup.Group) {
 	t0 := time.Now()
 	defer func() {
-		fileSDScanDuration.Observe(time.Since(t0).Seconds())
+		d.fileSDScanDuration.Observe(time.Since(t0).Seconds())
 	}()
 	ref := map[string]int{}
 	for _, p := range d.listFiles() {
 		tgroups, err := d.readFile(p)
 		if err != nil {
-			fileSDReadErrorsCount.Inc()
+			d.fileSDReadErrorsCount.Inc()
 
 			level.Error(d.logger).Log("msg", "Error reading file", "path", p, "err", err)
 			// Prevent deletion down below.

discovery/file/file_test.go

@@ -24,6 +24,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
@@ -143,7 +144,7 @@ func (t *testRunner) run(files ...string) {
 	ctx, cancel := context.WithCancel(context.Background())
 	t.cancelSD = cancel
 	go func() {
-		NewDiscovery(
+		d, err := NewDiscovery(
 			&SDConfig{
 				Files: files,
 				// Setting a high refresh interval to make sure that the tests only
@@ -151,7 +152,11 @@ func (t *testRunner) run(files ...string) {
 				RefreshInterval: model.Duration(1 * time.Hour),
 			},
 			nil,
-		).Run(ctx, t.ch)
+			prometheus.NewRegistry(),
+		)
+		require.NoError(t, err)
+
+		d.Run(ctx, t.ch)
 	}()
 }
 

discovery/gce/gce.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"golang.org/x/oauth2/google"
 	"google.golang.org/api/compute/v1"
@@ -86,7 +87,7 @@ func (*SDConfig) Name() string { return "gce" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(*c, opts.Logger)
+	return NewDiscovery(*c, opts.Logger, opts.Registerer)
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
@@ -121,7 +122,7 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf SDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	d := &Discovery{
 		project:      conf.Project,
 		zone:         conf.Zone,
@@ -141,10 +142,13 @@ func NewDiscovery(conf SDConfig, logger log.Logger) (*Discovery, error) {
 	d.isvc = compute.NewInstancesService(d.svc)
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"gce",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "gce",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+		},
 	)
 	return d, nil
 }

discovery/hetzner/hetzner.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/hetznercloud/hcloud-go/v2/hcloud"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -67,7 +68,7 @@ func (*SDConfig) Name() string { return "hetzner" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 type refresher interface {
@@ -127,17 +128,20 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*refresh.Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*refresh.Discovery, error) {
 	r, err := newRefresher(conf, logger)
 	if err != nil {
 		return nil, err
 	}
 
 	return refresh.NewDiscovery(
-		logger,
-		"hetzner",
-		time.Duration(conf.RefreshInterval),
-		r.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "hetzner",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: r.refresh,
+			Registry: reg,
+		},
 	), nil
 }
 

discovery/http/http.go

@@ -45,17 +45,10 @@ var (
 	}
 	userAgent        = fmt.Sprintf("Prometheus/%s", version.Version)
 	matchContentType = regexp.MustCompile(`^(?i:application\/json(;\s*charset=("utf-8"|utf-8))?)$`)
-
-	failuresCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_http_failures_total",
-			Help: "Number of HTTP service discovery refresh failures.",
-		})
 )
 
 func init() {
 	discovery.RegisterConfig(&SDConfig{})
-	prometheus.MustRegister(failuresCount)
 }
 
 // SDConfig is the configuration for HTTP based discovery.
@@ -70,7 +63,7 @@ func (*SDConfig) Name() string { return "http" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.HTTPClientOptions)
+	return NewDiscovery(c, opts.Logger, opts.HTTPClientOptions, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -112,10 +105,11 @@ type Discovery struct {
 	client          *http.Client
 	refreshInterval time.Duration
 	tgLastLength    int
+	failuresCount   prometheus.Counter
 }
 
 // NewDiscovery returns a new HTTP discovery for the given config.
-func NewDiscovery(conf *SDConfig, logger log.Logger, clientOpts []config.HTTPClientOption) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, clientOpts []config.HTTPClientOption, reg prometheus.Registerer) (*Discovery, error) {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -130,13 +124,22 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, clientOpts []config.HTTPCli
 		url:             conf.URL,
 		client:          client,
 		refreshInterval: time.Duration(conf.RefreshInterval), // Stored to be sent as headers.
+		failuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_http_failures_total",
+				Help: "Number of HTTP service discovery refresh failures.",
+			}),
 	}
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"http",
-		time.Duration(conf.RefreshInterval),
-		d.Refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "http",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.Refresh,
+			Registry: reg,
+			Metrics:  []prometheus.Collector{d.failuresCount},
+		},
 	)
 	return d, nil
 }
@@ -152,7 +155,7 @@ func (d *Discovery) Refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 
 	resp, err := d.client.Do(req.WithContext(ctx))
 	if err != nil {
-		failuresCount.Inc()
+		d.failuresCount.Inc()
 		return nil, err
 	}
 	defer func() {
@@ -161,31 +164,31 @@ func (d *Discovery) Refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 	}()
 
 	if resp.StatusCode != http.StatusOK {
-		failuresCount.Inc()
+		d.failuresCount.Inc()
 		return nil, fmt.Errorf("server returned HTTP status %s", resp.Status)
 	}
 
 	if !matchContentType.MatchString(strings.TrimSpace(resp.Header.Get("Content-Type"))) {
-		failuresCount.Inc()
+		d.failuresCount.Inc()
 		return nil, fmt.Errorf("unsupported content type %q", resp.Header.Get("Content-Type"))
 	}
 
 	b, err := io.ReadAll(resp.Body)
 	if err != nil {
-		failuresCount.Inc()
+		d.failuresCount.Inc()
 		return nil, err
 	}
 
 	var targetGroups []*targetgroup.Group
 
 	if err := json.Unmarshal(b, &targetGroups); err != nil {
-		failuresCount.Inc()
+		d.failuresCount.Inc()
 		return nil, err
 	}
 
 	for i, tg := range targetGroups {
 		if tg == nil {
-			failuresCount.Inc()
+			d.failuresCount.Inc()
 			err = errors.New("nil target group item found")
 			return nil, err
 		}

discovery/http/http_test.go

@@ -41,7 +41,7 @@ func TestHTTPValidRefresh(t *testing.T) {
 		RefreshInterval:  model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil)
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, prometheus.NewRegistry())
 	require.NoError(t, err)
 
 	ctx := context.Background()
@@ -63,7 +63,7 @@ func TestHTTPValidRefresh(t *testing.T) {
 		},
 	}
 	require.Equal(t, expectedTargets, tgs)
-	require.Equal(t, 0.0, getFailureCount())
+	require.Equal(t, 0.0, getFailureCount(d.failuresCount))
 }
 
 func TestHTTPInvalidCode(t *testing.T) {
@@ -79,13 +79,13 @@ func TestHTTPInvalidCode(t *testing.T) {
 		RefreshInterval:  model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil)
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, prometheus.NewRegistry())
 	require.NoError(t, err)
 
 	ctx := context.Background()
 	_, err = d.Refresh(ctx)
 	require.EqualError(t, err, "server returned HTTP status 400 Bad Request")
-	require.Equal(t, 1.0, getFailureCount())
+	require.Equal(t, 1.0, getFailureCount(d.failuresCount))
 }
 
 func TestHTTPInvalidFormat(t *testing.T) {
@@ -101,18 +101,16 @@ func TestHTTPInvalidFormat(t *testing.T) {
 		RefreshInterval:  model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil)
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, prometheus.NewRegistry())
 	require.NoError(t, err)
 
 	ctx := context.Background()
 	_, err = d.Refresh(ctx)
 	require.EqualError(t, err, `unsupported content type "text/plain; charset=utf-8"`)
-	require.Equal(t, 1.0, getFailureCount())
+	require.Equal(t, 1.0, getFailureCount(d.failuresCount))
 }
 
-var lastFailureCount float64
-
-func getFailureCount() float64 {
+func getFailureCount(failuresCount prometheus.Counter) float64 {
 	failureChan := make(chan prometheus.Metric)
 
 	go func() {
@@ -129,10 +127,7 @@ func getFailureCount() float64 {
 		metric.Write(&counter)
 	}
 
-	// account for failures in prior tests
-	count := *counter.Counter.Value - lastFailureCount
-	lastFailureCount = *counter.Counter.Value
-	return count
+	return *counter.Counter.Value
 }
 
 func TestContentTypeRegex(t *testing.T) {
@@ -417,7 +412,7 @@ func TestSourceDisappeared(t *testing.T) {
 		URL:              ts.URL,
 		RefreshInterval:  model.Duration(1 * time.Second),
 	}
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil)
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, prometheus.NewRegistry())
 	require.NoError(t, err)
 	for _, test := range cases {
 		ctx := context.Background()

discovery/ionos/ionos.go

@@ -23,6 +23,8 @@ import (
 
 	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/refresh"
+
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 const (
@@ -41,7 +43,7 @@ func init() {
 type Discovery struct{}
 
 // NewDiscovery returns a new refresh.Discovery for IONOS Cloud.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*refresh.Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*refresh.Discovery, error) {
 	if conf.ionosEndpoint == "" {
 		conf.ionosEndpoint = "https://api.ionos.com"
 	}
@@ -52,10 +54,13 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*refresh.Discovery, error)
 	}
 
 	return refresh.NewDiscovery(
-		logger,
-		"ionos",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "ionos",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+		},
 	), nil
 }
 
@@ -86,7 +91,7 @@ func (c SDConfig) Name() string {
 
 // NewDiscoverer returns a new discovery.Discoverer for IONOS Cloud.
 func (c SDConfig) NewDiscoverer(options discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(&c, options.Logger)
+	return NewDiscovery(&c, options.Logger, options.Registerer)
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.

discovery/kubernetes/endpoints.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	apiv1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/tools/cache"
@@ -30,12 +31,6 @@ import (
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
-var (
-	epAddCount    = eventCount.WithLabelValues("endpoints", "add")
-	epUpdateCount = eventCount.WithLabelValues("endpoints", "update")
-	epDeleteCount = eventCount.WithLabelValues("endpoints", "delete")
-)
-
 // Endpoints discovers new endpoint targets.
 type Endpoints struct {
 	logger log.Logger
@@ -54,10 +49,19 @@ type Endpoints struct {
 }
 
 // NewEndpoints returns a new endpoints discovery.
-func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node cache.SharedInformer) *Endpoints {
+func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node cache.SharedInformer, eventCount *prometheus.CounterVec) *Endpoints {
 	if l == nil {
 		l = log.NewNopLogger()
 	}
+
+	epAddCount := eventCount.WithLabelValues(RoleEndpoint.String(), MetricLabelRoleAdd)
+	epUpdateCount := eventCount.WithLabelValues(RoleEndpoint.String(), MetricLabelRoleUpdate)
+	epDeleteCount := eventCount.WithLabelValues(RoleEndpoint.String(), MetricLabelRoleDelete)
+
+	svcAddCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleAdd)
+	svcUpdateCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleUpdate)
+	svcDeleteCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleDelete)
+
 	e := &Endpoints{
 		logger:           l,
 		endpointsInf:     eps,
@@ -68,7 +72,7 @@ func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node ca
 		podStore:         pod.GetStore(),
 		nodeInf:          node,
 		withNodeMetadata: node != nil,
-		queue:            workqueue.NewNamed("endpoints"),
+		queue:            workqueue.NewNamed(RoleEndpoint.String()),
 	}
 
 	_, err := e.endpointsInf.AddEventHandler(cache.ResourceEventHandlerFuncs{

discovery/kubernetes/endpointslice.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	apiv1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/discovery/v1"
@@ -33,12 +34,6 @@ import (
 	"github.com/prometheus/prometheus/util/strutil"
 )
 
-var (
-	epslAddCount    = eventCount.WithLabelValues("endpointslice", "add")
-	epslUpdateCount = eventCount.WithLabelValues("endpointslice", "update")
-	epslDeleteCount = eventCount.WithLabelValues("endpointslice", "delete")
-)
-
 // EndpointSlice discovers new endpoint targets.
 type EndpointSlice struct {
 	logger log.Logger
@@ -57,10 +52,19 @@ type EndpointSlice struct {
 }
 
 // NewEndpointSlice returns a new endpointslice discovery.
-func NewEndpointSlice(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node cache.SharedInformer) *EndpointSlice {
+func NewEndpointSlice(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node cache.SharedInformer, eventCount *prometheus.CounterVec) *EndpointSlice {
 	if l == nil {
 		l = log.NewNopLogger()
 	}
+
+	epslAddCount := eventCount.WithLabelValues(RoleEndpointSlice.String(), MetricLabelRoleAdd)
+	epslUpdateCount := eventCount.WithLabelValues(RoleEndpointSlice.String(), MetricLabelRoleUpdate)
+	epslDeleteCount := eventCount.WithLabelValues(RoleEndpointSlice.String(), MetricLabelRoleDelete)
+
+	svcAddCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleAdd)
+	svcUpdateCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleUpdate)
+	svcDeleteCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleDelete)
+
 	e := &EndpointSlice{
 		logger:             l,
 		endpointSliceInf:   eps,
@@ -71,7 +75,7 @@ func NewEndpointSlice(l log.Logger, eps cache.SharedIndexInformer, svc, pod, nod
 		podStore:           pod.GetStore(),
 		nodeInf:            node,
 		withNodeMetadata:   node != nil,
-		queue:              workqueue.NewNamed("endpointSlice"),
+		queue:              workqueue.NewNamed(RoleEndpointSlice.String()),
 	}
 
 	_, err := e.endpointSliceInf.AddEventHandler(cache.ResourceEventHandlerFuncs{

discovery/kubernetes/ingress.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	v1 "k8s.io/api/networking/v1"
 	"k8s.io/api/networking/v1beta1"
@@ -30,12 +31,6 @@ import (
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
-var (
-	ingressAddCount    = eventCount.WithLabelValues("ingress", "add")
-	ingressUpdateCount = eventCount.WithLabelValues("ingress", "update")
-	ingressDeleteCount = eventCount.WithLabelValues("ingress", "delete")
-)
-
 // Ingress implements discovery of Kubernetes ingress.
 type Ingress struct {
 	logger   log.Logger
@@ -45,8 +40,18 @@ type Ingress struct {
 }
 
 // NewIngress returns a new ingress discovery.
-func NewIngress(l log.Logger, inf cache.SharedInformer) *Ingress {
-	s := &Ingress{logger: l, informer: inf, store: inf.GetStore(), queue: workqueue.NewNamed("ingress")}
+func NewIngress(l log.Logger, inf cache.SharedInformer, eventCount *prometheus.CounterVec) *Ingress {
+	ingressAddCount := eventCount.WithLabelValues(RoleIngress.String(), MetricLabelRoleAdd)
+	ingressUpdateCount := eventCount.WithLabelValues(RoleIngress.String(), MetricLabelRoleUpdate)
+	ingressDeleteCount := eventCount.WithLabelValues(RoleIngress.String(), MetricLabelRoleDelete)
+
+	s := &Ingress{
+		logger:   l,
+		informer: inf,
+		store:    inf.GetStore(),
+		queue:    workqueue.NewNamed(RoleIngress.String()),
+	}
+
 	_, err := s.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(o interface{}) {
 			ingressAddCount.Inc()

discovery/kubernetes/kubernetes.go

@@ -58,24 +58,14 @@ import (
 const (
 	// metaLabelPrefix is the meta prefix used for all meta labels.
 	// in this discovery.
-	metaLabelPrefix  = model.MetaLabelPrefix + "kubernetes_"
-	namespaceLabel   = metaLabelPrefix + "namespace"
-	metricsNamespace = "prometheus_sd_kubernetes"
-	presentValue     = model.LabelValue("true")
+	metaLabelPrefix = model.MetaLabelPrefix + "kubernetes_"
+	namespaceLabel  = metaLabelPrefix + "namespace"
+	presentValue    = model.LabelValue("true")
 )
 
 var (
 	// Http header.
 	userAgent = fmt.Sprintf("Prometheus/%s", version.Version)
-	// Custom events metric.
-	eventCount = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Name:      "events_total",
-			Help:      "The number of Kubernetes events handled.",
-		},
-		[]string{"role", "event"},
-	)
 	// DefaultSDConfig is the default Kubernetes SD configuration.
 	DefaultSDConfig = SDConfig{
 		HTTPClientConfig: config.DefaultHTTPClientConfig,
@@ -84,15 +74,6 @@ var (
 
 func init() {
 	discovery.RegisterConfig(&SDConfig{})
-	prometheus.MustRegister(eventCount)
-	// Initialize metric vectors.
-	for _, role := range []string{"endpointslice", "endpoints", "node", "pod", "service", "ingress"} {
-		for _, evt := range []string{"add", "delete", "update"} {
-			eventCount.WithLabelValues(role, evt)
-		}
-	}
-	(&clientGoRequestMetricAdapter{}).Register(prometheus.DefaultRegisterer)
-	(&clientGoWorkqueueMetricsProvider{}).Register(prometheus.DefaultRegisterer)
 }
 
 // Role is role of the service in Kubernetes.
@@ -121,6 +102,16 @@ func (c *Role) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	}
 }
 
+func (c Role) String() string {
+	return string(c)
+}
+
+const (
+	MetricLabelRoleAdd    = "add"
+	MetricLabelRoleDelete = "delete"
+	MetricLabelRoleUpdate = "update"
+)
+
 // SDConfig is the configuration for Kubernetes service discovery.
 type SDConfig struct {
 	APIServer          config.URL              `yaml:"api_server,omitempty"`
@@ -137,7 +128,7 @@ func (*SDConfig) Name() string { return "kubernetes" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return New(opts.Logger, c)
+	return New(opts.Logger, opts.Registerer, c)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -274,6 +265,8 @@ type Discovery struct {
 	selectors          roleSelector
 	ownNamespace       string
 	attachMetadata     AttachMetadataConfig
+	eventCount         *prometheus.CounterVec
+	metricRegisterer   discovery.MetricRegisterer
 }
 
 func (d *Discovery) getNamespaces() []string {
@@ -292,7 +285,7 @@ func (d *Discovery) getNamespaces() []string {
 }
 
 // New creates a new Kubernetes discovery for the given role.
-func New(l log.Logger, conf *SDConfig) (*Discovery, error) {
+func New(l log.Logger, reg prometheus.Registerer, conf *SDConfig) (*Discovery, error) {
 	if l == nil {
 		l = log.NewNopLogger()
 	}
@@ -346,7 +339,7 @@ func New(l log.Logger, conf *SDConfig) (*Discovery, error) {
 		return nil, err
 	}
 
-	return &Discovery{
+	d := &Discovery{
 		client:             c,
 		logger:             l,
 		role:               conf.Role,
@@ -355,7 +348,37 @@ func New(l log.Logger, conf *SDConfig) (*Discovery, error) {
 		selectors:          mapSelector(conf.Selectors),
 		ownNamespace:       ownNamespace,
 		attachMetadata:     conf.AttachMetadata,
-	}, nil
+		eventCount: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Namespace: discovery.KubernetesMetricsNamespace,
+				Name:      "events_total",
+				Help:      "The number of Kubernetes events handled.",
+			},
+			[]string{"role", "event"},
+		),
+	}
+
+	d.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{d.eventCount})
+
+	// Initialize metric vectors.
+	for _, role := range []string{
+		RoleEndpointSlice.String(),
+		RoleEndpoint.String(),
+		RoleNode.String(),
+		RolePod.String(),
+		RoleService.String(),
+		RoleIngress.String(),
+	} {
+		for _, evt := range []string{
+			MetricLabelRoleAdd,
+			MetricLabelRoleDelete,
+			MetricLabelRoleUpdate,
+		} {
+			d.eventCount.WithLabelValues(role, evt)
+		}
+	}
+
+	return d, nil
 }
 
 func mapSelector(rawSelector []SelectorConfig) roleSelector {
@@ -391,6 +414,14 @@ const resyncDisabled = 0
 // Run implements the discoverer interface.
 func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 	d.Lock()
+
+	err := d.metricRegisterer.RegisterMetrics()
+	if err != nil {
+		level.Error(d.logger).Log("msg", "Unable to register metrics", "err", err.Error())
+		return
+	}
+	defer d.metricRegisterer.UnregisterMetrics()
+
 	namespaces := d.getNamespaces()
 
 	switch d.role {
@@ -482,6 +513,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 				cache.NewSharedInformer(slw, &apiv1.Service{}, resyncDisabled),
 				cache.NewSharedInformer(plw, &apiv1.Pod{}, resyncDisabled),
 				nodeInf,
+				d.eventCount,
 			)
 			d.discoverers = append(d.discoverers, eps)
 			go eps.endpointSliceInf.Run(ctx.Done())
@@ -541,6 +573,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 				cache.NewSharedInformer(slw, &apiv1.Service{}, resyncDisabled),
 				cache.NewSharedInformer(plw, &apiv1.Pod{}, resyncDisabled),
 				nodeInf,
+				d.eventCount,
 			)
 			d.discoverers = append(d.discoverers, eps)
 			go eps.endpointsInf.Run(ctx.Done())
@@ -572,6 +605,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 				log.With(d.logger, "role", "pod"),
 				d.newPodsByNodeInformer(plw),
 				nodeInformer,
+				d.eventCount,
 			)
 			d.discoverers = append(d.discoverers, pod)
 			go pod.podInf.Run(ctx.Done())
@@ -594,6 +628,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 			svc := NewService(
 				log.With(d.logger, "role", "service"),
 				cache.NewSharedInformer(slw, &apiv1.Service{}, resyncDisabled),
+				d.eventCount,
 			)
 			d.discoverers = append(d.discoverers, svc)
 			go svc.informer.Run(ctx.Done())
@@ -651,13 +686,14 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 			ingress := NewIngress(
 				log.With(d.logger, "role", "ingress"),
 				informer,
+				d.eventCount,
 			)
 			d.discoverers = append(d.discoverers, ingress)
 			go ingress.informer.Run(ctx.Done())
 		}
 	case RoleNode:
 		nodeInformer := d.newNodeInformer(ctx)
-		node := NewNode(log.With(d.logger, "role", "node"), nodeInformer)
+		node := NewNode(log.With(d.logger, "role", "node"), nodeInformer, d.eventCount)
 		d.discoverers = append(d.discoverers, node)
 		go node.informer.Run(ctx.Done())
 	default:

discovery/kubernetes/kubernetes_test.go

@@ -29,6 +29,8 @@ import (
 	"k8s.io/client-go/kubernetes/fake"
 	"k8s.io/client-go/tools/cache"
 
+	"github.com/prometheus/client_golang/prometheus"
+
 	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 	"github.com/prometheus/prometheus/util/testutil"
@@ -49,13 +51,25 @@ func makeDiscoveryWithVersion(role Role, nsDiscovery NamespaceDiscovery, k8sVer
 	fakeDiscovery, _ := clientset.Discovery().(*fakediscovery.FakeDiscovery)
 	fakeDiscovery.FakedServerVersion = &version.Info{GitVersion: k8sVer}
 
-	return &Discovery{
+	d := &Discovery{
 		client:             clientset,
 		logger:             log.NewNopLogger(),
 		role:               role,
 		namespaceDiscovery: &nsDiscovery,
 		ownNamespace:       "own-ns",
-	}, clientset
+		eventCount: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Namespace: discovery.KubernetesMetricsNamespace,
+				Name:      "events_total",
+				Help:      "The number of Kubernetes events handled.",
+			},
+			[]string{"role", "event"},
+		),
+	}
+
+	d.metricRegisterer = discovery.NewMetricRegisterer(prometheus.NewRegistry(), []prometheus.Collector{d.eventCount})
+
+	return d, clientset
 }
 
 // makeDiscoveryWithMetadata creates a kubernetes.Discovery instance with the specified metadata config.

discovery/kubernetes/node.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	apiv1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/tools/cache"
@@ -35,12 +36,6 @@ const (
 	NodeLegacyHostIP = "LegacyHostIP"
 )
 
-var (
-	nodeAddCount    = eventCount.WithLabelValues("node", "add")
-	nodeUpdateCount = eventCount.WithLabelValues("node", "update")
-	nodeDeleteCount = eventCount.WithLabelValues("node", "delete")
-)
-
 // Node discovers Kubernetes nodes.
 type Node struct {
 	logger   log.Logger
@@ -50,11 +45,22 @@ type Node struct {
 }
 
 // NewNode returns a new node discovery.
-func NewNode(l log.Logger, inf cache.SharedInformer) *Node {
+func NewNode(l log.Logger, inf cache.SharedInformer, eventCount *prometheus.CounterVec) *Node {
 	if l == nil {
 		l = log.NewNopLogger()
 	}
-	n := &Node{logger: l, informer: inf, store: inf.GetStore(), queue: workqueue.NewNamed("node")}
+
+	nodeAddCount := eventCount.WithLabelValues(RoleNode.String(), MetricLabelRoleAdd)
+	nodeUpdateCount := eventCount.WithLabelValues(RoleNode.String(), MetricLabelRoleUpdate)
+	nodeDeleteCount := eventCount.WithLabelValues(RoleNode.String(), MetricLabelRoleDelete)
+
+	n := &Node{
+		logger:   l,
+		informer: inf,
+		store:    inf.GetStore(),
+		queue:    workqueue.NewNamed(RoleNode.String()),
+	}
+
 	_, err := n.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(o interface{}) {
 			nodeAddCount.Inc()

discovery/kubernetes/pod.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	apiv1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -34,12 +35,6 @@ import (
 
 const nodeIndex = "node"
 
-var (
-	podAddCount    = eventCount.WithLabelValues("pod", "add")
-	podUpdateCount = eventCount.WithLabelValues("pod", "update")
-	podDeleteCount = eventCount.WithLabelValues("pod", "delete")
-)
-
 // Pod discovers new pod targets.
 type Pod struct {
 	podInf           cache.SharedIndexInformer
@@ -51,18 +46,22 @@ type Pod struct {
 }
 
 // NewPod creates a new pod discovery.
-func NewPod(l log.Logger, pods cache.SharedIndexInformer, nodes cache.SharedInformer) *Pod {
+func NewPod(l log.Logger, pods cache.SharedIndexInformer, nodes cache.SharedInformer, eventCount *prometheus.CounterVec) *Pod {
 	if l == nil {
 		l = log.NewNopLogger()
 	}
 
+	podAddCount := eventCount.WithLabelValues(RolePod.String(), MetricLabelRoleAdd)
+	podDeleteCount := eventCount.WithLabelValues(RolePod.String(), MetricLabelRoleDelete)
+	podUpdateCount := eventCount.WithLabelValues(RolePod.String(), MetricLabelRoleUpdate)
+
 	p := &Pod{
 		podInf:           pods,
 		nodeInf:          nodes,
 		withNodeMetadata: nodes != nil,
 		store:            pods.GetStore(),
 		logger:           l,
-		queue:            workqueue.NewNamed("pod"),
+		queue:            workqueue.NewNamed(RolePod.String()),
 	}
 	_, err := p.podInf.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(o interface{}) {

discovery/kubernetes/service.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	apiv1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/tools/cache"
@@ -30,12 +31,6 @@ import (
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
-var (
-	svcAddCount    = eventCount.WithLabelValues("service", "add")
-	svcUpdateCount = eventCount.WithLabelValues("service", "update")
-	svcDeleteCount = eventCount.WithLabelValues("service", "delete")
-)
-
 // Service implements discovery of Kubernetes services.
 type Service struct {
 	logger   log.Logger
@@ -45,11 +40,22 @@ type Service struct {
 }
 
 // NewService returns a new service discovery.
-func NewService(l log.Logger, inf cache.SharedInformer) *Service {
+func NewService(l log.Logger, inf cache.SharedInformer, eventCount *prometheus.CounterVec) *Service {
 	if l == nil {
 		l = log.NewNopLogger()
 	}
-	s := &Service{logger: l, informer: inf, store: inf.GetStore(), queue: workqueue.NewNamed("service")}
+
+	svcAddCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleAdd)
+	svcUpdateCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleUpdate)
+	svcDeleteCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleDelete)
+
+	s := &Service{
+		logger:   l,
+		informer: inf,
+		store:    inf.GetStore(),
+		queue:    workqueue.NewNamed(RoleService.String()),
+	}
+
 	_, err := s.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(o interface{}) {
 			svcAddCount.Inc()

discovery/legacymanager/manager.go

@@ -28,48 +28,6 @@ import (
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
-var (
-	failedConfigs = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "prometheus_sd_failed_configs",
-			Help: "Current number of service discovery configurations that failed to load.",
-		},
-		[]string{"name"},
-	)
-	discoveredTargets = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "prometheus_sd_discovered_targets",
-			Help: "Current number of discovered targets.",
-		},
-		[]string{"name", "config"},
-	)
-	receivedUpdates = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_received_updates_total",
-			Help: "Total number of update events received from the SD providers.",
-		},
-		[]string{"name"},
-	)
-	delayedUpdates = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_updates_delayed_total",
-			Help: "Total number of update events that couldn't be sent immediately.",
-		},
-		[]string{"name"},
-	)
-	sentUpdates = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_updates_total",
-			Help: "Total number of update events sent to the SD consumers.",
-		},
-		[]string{"name"},
-	)
-)
-
-func RegisterMetrics() {
-	prometheus.MustRegister(failedConfigs, discoveredTargets, receivedUpdates, delayedUpdates, sentUpdates)
-}
-
 type poolKey struct {
 	setName  string
 	provider string
@@ -84,7 +42,7 @@ type provider struct {
 }
 
 // NewManager is the Discovery Manager constructor.
-func NewManager(ctx context.Context, logger log.Logger, options ...func(*Manager)) *Manager {
+func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Registerer, options ...func(*Manager)) *Manager {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -96,10 +54,21 @@ func NewManager(ctx context.Context, logger log.Logger, options ...func(*Manager
 		ctx:            ctx,
 		updatert:       5 * time.Second,
 		triggerSend:    make(chan struct{}, 1),
+		registerer:     registerer,
 	}
 	for _, option := range options {
 		option(mgr)
 	}
+
+	// Register the metrics.
+	// We have to do this after setting all options, so that the name of the Manager is set.
+	if metrics, err := discovery.NewMetrics(registerer, mgr.name); err == nil {
+		mgr.metrics = metrics
+	} else {
+		level.Error(logger).Log("msg", "Failed to create discovery manager metrics", "manager", mgr.name, "err", err)
+		return nil
+	}
+
 	return mgr
 }
 
@@ -135,6 +104,11 @@ type Manager struct {
 
 	// The triggerSend channel signals to the manager that new updates have been received from providers.
 	triggerSend chan struct{}
+
+	// A registerer for all service discovery metrics.
+	registerer prometheus.Registerer
+
+	metrics *discovery.Metrics
 }
 
 // Run starts the background processing.
@@ -157,7 +131,7 @@ func (m *Manager) ApplyConfig(cfg map[string]discovery.Configs) error {
 
 	for pk := range m.targets {
 		if _, ok := cfg[pk.setName]; !ok {
-			discoveredTargets.DeleteLabelValues(m.name, pk.setName)
+			m.metrics.DiscoveredTargets.DeleteLabelValues(m.name, pk.setName)
 		}
 	}
 	m.cancelDiscoverers()
@@ -168,9 +142,9 @@ func (m *Manager) ApplyConfig(cfg map[string]discovery.Configs) error {
 	failedCount := 0
 	for name, scfg := range cfg {
 		failedCount += m.registerProviders(scfg, name)
-		discoveredTargets.WithLabelValues(m.name, name).Set(0)
+		m.metrics.DiscoveredTargets.WithLabelValues(name).Set(0)
 	}
-	failedConfigs.WithLabelValues(m.name).Set(float64(failedCount))
+	m.metrics.FailedConfigs.Set(float64(failedCount))
 
 	for _, prov := range m.providers {
 		m.startProvider(m.ctx, prov)
@@ -207,7 +181,7 @@ func (m *Manager) updater(ctx context.Context, p *provider, updates chan []*targ
 		case <-ctx.Done():
 			return
 		case tgs, ok := <-updates:
-			receivedUpdates.WithLabelValues(m.name).Inc()
+			m.metrics.ReceivedUpdates.Inc()
 			if !ok {
 				level.Debug(m.logger).Log("msg", "Discoverer channel closed", "provider", p.name)
 				return
@@ -236,11 +210,11 @@ func (m *Manager) sender() {
 		case <-ticker.C: // Some discoverers send updates too often so we throttle these with the ticker.
 			select {
 			case <-m.triggerSend:
-				sentUpdates.WithLabelValues(m.name).Inc()
+				m.metrics.SentUpdates.Inc()
 				select {
 				case m.syncCh <- m.allGroups():
 				default:
-					delayedUpdates.WithLabelValues(m.name).Inc()
+					m.metrics.DelayedUpdates.Inc()
 					level.Debug(m.logger).Log("msg", "Discovery receiver's channel was full so will retry the next cycle")
 					select {
 					case m.triggerSend <- struct{}{}:
@@ -288,7 +262,7 @@ func (m *Manager) allGroups() map[string][]*targetgroup.Group {
 		}
 	}
 	for setName, v := range n {
-		discoveredTargets.WithLabelValues(m.name, setName).Set(float64(v))
+		m.metrics.DiscoveredTargets.WithLabelValues(setName).Set(float64(v))
 	}
 	return tSets
 }
@@ -309,7 +283,8 @@ func (m *Manager) registerProviders(cfgs discovery.Configs, setName string) int
 		}
 		typ := cfg.Name()
 		d, err := cfg.NewDiscoverer(discovery.DiscovererOptions{
-			Logger: log.With(m.logger, "discovery", typ, "config", setName),
+			Logger:     log.With(m.logger, "discovery", typ, "config", setName),
+			Registerer: m.registerer,
 		})
 		if err != nil {
 			level.Error(m.logger).Log("msg", "Cannot create service discovery", "err", err, "type", typ, "config", setName)

discovery/legacymanager/manager_test.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	client_testutil "github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
@@ -664,7 +665,8 @@ func TestTargetUpdatesOrder(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel()
 
-			discoveryManager := NewManager(ctx, log.NewNopLogger())
+			discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+			require.NotNil(t, discoveryManager)
 			discoveryManager.updatert = 100 * time.Millisecond
 
 			var totalUpdatesCount int
@@ -746,7 +748,8 @@ func verifyPresence(t *testing.T, tSets map[poolKey]map[string]*targetgroup.Grou
 func TestTargetSetRecreatesTargetGroupsEveryRun(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -774,7 +777,8 @@ func TestTargetSetRecreatesTargetGroupsEveryRun(t *testing.T) {
 func TestDiscovererConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -798,7 +802,8 @@ func TestDiscovererConfigs(t *testing.T) {
 func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -837,7 +842,8 @@ func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 func TestIdenticalConfigurationsAreCoalesced(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, nil)
+	discoveryManager := NewManager(ctx, nil, prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -868,7 +874,8 @@ func TestApplyConfigDoesNotModifyStaticTargets(t *testing.T) {
 	}
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -893,7 +900,8 @@ func (e errorConfig) NewDiscoverer(discovery.DiscovererOptions) (discovery.Disco
 func TestGaugeFailedConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -907,7 +915,7 @@ func TestGaugeFailedConfigs(t *testing.T) {
 	discoveryManager.ApplyConfig(c)
 	<-discoveryManager.SyncCh()
 
-	failedCount := client_testutil.ToFloat64(failedConfigs)
+	failedCount := client_testutil.ToFloat64(discoveryManager.metrics.FailedConfigs)
 	if failedCount != 3 {
 		t.Fatalf("Expected to have 3 failed configs, got: %v", failedCount)
 	}
@@ -918,7 +926,7 @@ func TestGaugeFailedConfigs(t *testing.T) {
 	discoveryManager.ApplyConfig(c)
 	<-discoveryManager.SyncCh()
 
-	failedCount = client_testutil.ToFloat64(failedConfigs)
+	failedCount = client_testutil.ToFloat64(discoveryManager.metrics.FailedConfigs)
 	if failedCount != 0 {
 		t.Fatalf("Expected to get no failed config, got: %v", failedCount)
 	}
@@ -1049,7 +1057,8 @@ func TestCoordinationWithReceiver(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel()
 
-			mgr := NewManager(ctx, nil)
+			mgr := NewManager(ctx, nil, prometheus.NewRegistry())
+			require.NotNil(t, mgr)
 			mgr.updatert = updateDelay
 			go mgr.Run()
 

discovery/linode/linode.go

@@ -67,24 +67,15 @@ const (
 )
 
 // DefaultSDConfig is the default Linode SD configuration.
-var (
-	DefaultSDConfig = SDConfig{
-		TagSeparator:     ",",
-		Port:             80,
-		RefreshInterval:  model.Duration(60 * time.Second),
-		HTTPClientConfig: config.DefaultHTTPClientConfig,
-	}
-
-	failuresCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_linode_failures_total",
-			Help: "Number of Linode service discovery refresh failures.",
-		})
-)
+var DefaultSDConfig = SDConfig{
+	TagSeparator:     ",",
+	Port:             80,
+	RefreshInterval:  model.Duration(60 * time.Second),
+	HTTPClientConfig: config.DefaultHTTPClientConfig,
+}
 
 func init() {
 	discovery.RegisterConfig(&SDConfig{})
-	prometheus.MustRegister(failuresCount)
 }
 
 // SDConfig is the configuration for Linode based service discovery.
@@ -101,7 +92,7 @@ func (*SDConfig) Name() string { return "linode" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -131,16 +122,22 @@ type Discovery struct {
 	pollCount            int
 	lastResults          []*targetgroup.Group
 	eventPollingEnabled  bool
+	failuresCount        prometheus.Counter
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	d := &Discovery{
 		port:                 conf.Port,
 		tagSeparator:         conf.TagSeparator,
 		pollCount:            0,
 		lastRefreshTimestamp: time.Now().UTC(),
 		eventPollingEnabled:  true,
+		failuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_linode_failures_total",
+				Help: "Number of Linode service discovery refresh failures.",
+			}),
 	}
 
 	rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "linode_sd")
@@ -158,10 +155,14 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 	d.client = &client
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"linode",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "linode",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+			Metrics:  []prometheus.Collector{d.failuresCount},
+		},
 	)
 	return d, nil
 }
@@ -222,14 +223,14 @@ func (d *Discovery) refreshData(ctx context.Context) ([]*targetgroup.Group, erro
 	// Gather all linode instances.
 	instances, err := d.client.ListInstances(ctx, &linodego.ListOptions{PageSize: 500})
 	if err != nil {
-		failuresCount.Inc()
+		d.failuresCount.Inc()
 		return nil, err
 	}
 
 	// Gather detailed IP address info for all IPs on all linode instances.
 	detailedIPs, err := d.client.ListIPAddresses(ctx, &linodego.ListOptions{PageSize: 500})
 	if err != nil {
-		failuresCount.Inc()
+		d.failuresCount.Inc()
 		return nil, err
 	}
 

discovery/linode/linode_test.go

@@ -20,6 +20,7 @@ import (
 	"testing"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
@@ -52,7 +53,7 @@ func TestLinodeSDRefresh(t *testing.T) {
 		Credentials: tokenID,
 		Type:        "Bearer",
 	}
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 	require.NoError(t, err)
 	endpoint, err := url.Parse(sdmock.Mock.Endpoint())
 	require.NoError(t, err)

discovery/manager.go

@@ -28,48 +28,6 @@ import (
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
-var (
-	failedConfigs = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "prometheus_sd_failed_configs",
-			Help: "Current number of service discovery configurations that failed to load.",
-		},
-		[]string{"name"},
-	)
-	discoveredTargets = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "prometheus_sd_discovered_targets",
-			Help: "Current number of discovered targets.",
-		},
-		[]string{"name", "config"},
-	)
-	receivedUpdates = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_received_updates_total",
-			Help: "Total number of update events received from the SD providers.",
-		},
-		[]string{"name"},
-	)
-	delayedUpdates = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_updates_delayed_total",
-			Help: "Total number of update events that couldn't be sent immediately.",
-		},
-		[]string{"name"},
-	)
-	sentUpdates = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_updates_total",
-			Help: "Total number of update events sent to the SD consumers.",
-		},
-		[]string{"name"},
-	)
-)
-
-func RegisterMetrics() {
-	prometheus.MustRegister(failedConfigs, discoveredTargets, receivedUpdates, delayedUpdates, sentUpdates)
-}
-
 type poolKey struct {
 	setName  string
 	provider string
@@ -107,7 +65,7 @@ func (p *Provider) Config() interface{} {
 }
 
 // NewManager is the Discovery Manager constructor.
-func NewManager(ctx context.Context, logger log.Logger, options ...func(*Manager)) *Manager {
+func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Registerer, options ...func(*Manager)) *Manager {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -118,10 +76,21 @@ func NewManager(ctx context.Context, logger log.Logger, options ...func(*Manager
 		ctx:         ctx,
 		updatert:    5 * time.Second,
 		triggerSend: make(chan struct{}, 1),
+		registerer:  registerer,
 	}
 	for _, option := range options {
 		option(mgr)
 	}
+
+	// Register the metrics.
+	// We have to do this after setting all options, so that the name of the Manager is set.
+	if metrics, err := NewMetrics(registerer, mgr.name); err == nil {
+		mgr.metrics = metrics
+	} else {
+		level.Error(logger).Log("msg", "Failed to create discovery manager metrics", "manager", mgr.name, "err", err)
+		return nil
+	}
+
 	return mgr
 }
 
@@ -170,6 +139,11 @@ type Manager struct {
 
 	// lastProvider counts providers registered during Manager's lifetime.
 	lastProvider uint
+
+	// A registerer for all service discovery metrics.
+	registerer prometheus.Registerer
+
+	metrics *Metrics
 }
 
 // Providers returns the currently configured SD providers.
@@ -200,7 +174,7 @@ func (m *Manager) ApplyConfig(cfg map[string]Configs) error {
 	for name, scfg := range cfg {
 		failedCount += m.registerProviders(scfg, name)
 	}
-	failedConfigs.WithLabelValues(m.name).Set(float64(failedCount))
+	m.metrics.FailedConfigs.Set(float64(failedCount))
 
 	var (
 		wg sync.WaitGroup
@@ -230,13 +204,13 @@ func (m *Manager) ApplyConfig(cfg map[string]Configs) error {
 			// Remove obsolete subs' targets.
 			if _, ok := prov.newSubs[s]; !ok {
 				delete(m.targets, poolKey{s, prov.name})
-				discoveredTargets.DeleteLabelValues(m.name, s)
+				m.metrics.DiscoveredTargets.DeleteLabelValues(m.name, s)
 			}
 		}
 		// Set metrics and targets for new subs.
 		for s := range prov.newSubs {
 			if _, ok := prov.subs[s]; !ok {
-				discoveredTargets.WithLabelValues(m.name, s).Set(0)
+				m.metrics.DiscoveredTargets.WithLabelValues(s).Set(0)
 			}
 			if l := len(refTargets); l > 0 {
 				m.targets[poolKey{s, prov.name}] = make(map[string]*targetgroup.Group, l)
@@ -316,7 +290,7 @@ func (m *Manager) updater(ctx context.Context, p *Provider, updates chan []*targ
 		case <-ctx.Done():
 			return
 		case tgs, ok := <-updates:
-			receivedUpdates.WithLabelValues(m.name).Inc()
+			m.metrics.ReceivedUpdates.Inc()
 			if !ok {
 				level.Debug(m.logger).Log("msg", "Discoverer channel closed", "provider", p.name)
 				// Wait for provider cancellation to ensure targets are cleaned up when expected.
@@ -349,11 +323,11 @@ func (m *Manager) sender() {
 		case <-ticker.C: // Some discoverers send updates too often, so we throttle these with the ticker.
 			select {
 			case <-m.triggerSend:
-				sentUpdates.WithLabelValues(m.name).Inc()
+				m.metrics.SentUpdates.Inc()
 				select {
 				case m.syncCh <- m.allGroups():
 				default:
-					delayedUpdates.WithLabelValues(m.name).Inc()
+					m.metrics.DelayedUpdates.Inc()
 					level.Debug(m.logger).Log("msg", "Discovery receiver's channel was full so will retry the next cycle")
 					select {
 					case m.triggerSend <- struct{}{}:
@@ -405,7 +379,7 @@ func (m *Manager) allGroups() map[string][]*targetgroup.Group {
 		}
 	}
 	for setName, v := range n {
-		discoveredTargets.WithLabelValues(m.name, setName).Set(float64(v))
+		m.metrics.DiscoveredTargets.WithLabelValues(setName).Set(float64(v))
 	}
 	return tSets
 }
@@ -428,6 +402,7 @@ func (m *Manager) registerProviders(cfgs Configs, setName string) int {
 		d, err := cfg.NewDiscoverer(DiscovererOptions{
 			Logger:            log.With(m.logger, "discovery", typ, "config", setName),
 			HTTPClientOptions: m.httpOpts,
+			Registerer:        m.registerer,
 		})
 		if err != nil {
 			level.Error(m.logger).Log("msg", "Cannot create service discovery", "err", err, "type", typ, "config", setName)

discovery/manager_test.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	client_testutil "github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
@@ -664,7 +665,8 @@ func TestTargetUpdatesOrder(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel()
 
-			discoveryManager := NewManager(ctx, log.NewNopLogger())
+			discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+			require.NotNil(t, discoveryManager)
 			discoveryManager.updatert = 100 * time.Millisecond
 
 			var totalUpdatesCount int
@@ -778,7 +780,8 @@ func pk(provider, setName string, n int) poolKey {
 func TestTargetSetTargetGroupsPresentOnConfigReload(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -810,7 +813,8 @@ func TestTargetSetTargetGroupsPresentOnConfigReload(t *testing.T) {
 func TestTargetSetTargetGroupsPresentOnConfigRename(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -845,7 +849,8 @@ func TestTargetSetTargetGroupsPresentOnConfigRename(t *testing.T) {
 func TestTargetSetTargetGroupsPresentOnConfigDuplicateAndDeleteOriginal(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -883,7 +888,8 @@ func TestTargetSetTargetGroupsPresentOnConfigDuplicateAndDeleteOriginal(t *testi
 func TestTargetSetTargetGroupsPresentOnConfigChange(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -944,7 +950,8 @@ func TestTargetSetTargetGroupsPresentOnConfigChange(t *testing.T) {
 func TestTargetSetRecreatesTargetGroupsOnConfigChange(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -983,7 +990,8 @@ func TestTargetSetRecreatesTargetGroupsOnConfigChange(t *testing.T) {
 func TestDiscovererConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -1015,7 +1023,8 @@ func TestDiscovererConfigs(t *testing.T) {
 func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -1062,7 +1071,8 @@ func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 func TestIdenticalConfigurationsAreCoalesced(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, nil)
+	discoveryManager := NewManager(ctx, nil, prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -1098,7 +1108,8 @@ func TestApplyConfigDoesNotModifyStaticTargets(t *testing.T) {
 	}
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -1144,7 +1155,8 @@ func (s lockStaticDiscoverer) Run(ctx context.Context, up chan<- []*targetgroup.
 func TestGaugeFailedConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -1158,7 +1170,7 @@ func TestGaugeFailedConfigs(t *testing.T) {
 	discoveryManager.ApplyConfig(c)
 	<-discoveryManager.SyncCh()
 
-	failedCount := client_testutil.ToFloat64(failedConfigs)
+	failedCount := client_testutil.ToFloat64(discoveryManager.metrics.FailedConfigs)
 	if failedCount != 3 {
 		t.Fatalf("Expected to have 3 failed configs, got: %v", failedCount)
 	}
@@ -1169,7 +1181,7 @@ func TestGaugeFailedConfigs(t *testing.T) {
 	discoveryManager.ApplyConfig(c)
 	<-discoveryManager.SyncCh()
 
-	failedCount = client_testutil.ToFloat64(failedConfigs)
+	failedCount = client_testutil.ToFloat64(discoveryManager.metrics.FailedConfigs)
 	if failedCount != 0 {
 		t.Fatalf("Expected to get no failed config, got: %v", failedCount)
 	}
@@ -1300,7 +1312,8 @@ func TestCoordinationWithReceiver(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel()
 
-			mgr := NewManager(ctx, nil)
+			mgr := NewManager(ctx, nil, prometheus.NewRegistry())
+			require.NotNil(t, mgr)
 			mgr.updatert = updateDelay
 			go mgr.Run()
 
@@ -1392,10 +1405,11 @@ func (o onceProvider) Run(_ context.Context, ch chan<- []*targetgroup.Group) {
 
 // TestTargetSetTargetGroupsUpdateDuringApplyConfig is used to detect races when
 // ApplyConfig happens at the same time as targets update.
-func TestTargetSetTargetGroupsUpdateDuringApplyConfig(*testing.T) {
+func TestTargetSetTargetGroupsUpdateDuringApplyConfig(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 

discovery/marathon/marathon.go

@@ -28,6 +28,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -83,7 +84,7 @@ func (*SDConfig) Name() string { return "marathon" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(*c, opts.Logger)
+	return NewDiscovery(*c, opts.Logger, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -132,7 +133,7 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Marathon Discovery.
-func NewDiscovery(conf SDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "marathon_sd")
 	if err != nil {
 		return nil, err
@@ -154,10 +155,13 @@ func NewDiscovery(conf SDConfig, logger log.Logger) (*Discovery, error) {
 		appsClient: fetchApps,
 	}
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"marathon",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "marathon",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+		},
 	)
 	return d, nil
 }

discovery/marathon/marathon_test.go

@@ -21,6 +21,7 @@ import (
 	"net/http/httptest"
 	"testing"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 
 	"github.com/prometheus/prometheus/discovery/targetgroup"
@@ -36,7 +37,7 @@ func testConfig() SDConfig {
 }
 
 func testUpdateServices(client appListClient) ([]*targetgroup.Group, error) {
-	md, err := NewDiscovery(testConfig(), nil)
+	md, err := NewDiscovery(testConfig(), nil, prometheus.NewRegistry())
 	if err != nil {
 		return nil, err
 	}
@@ -129,7 +130,7 @@ func TestMarathonSDSendGroup(t *testing.T) {
 }
 
 func TestMarathonSDRemoveApp(t *testing.T) {
-	md, err := NewDiscovery(testConfig(), nil)
+	md, err := NewDiscovery(testConfig(), nil, prometheus.NewRegistry())
 	if err != nil {
 		t.Fatalf("%s", err)
 	}

discovery/metrics.go

@@ -0,0 +1,101 @@
+// Copyright 2016 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package discovery
+
+import (
+	"fmt"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	clientGoRequestMetrics  = &clientGoRequestMetricAdapter{}
+	clientGoWorkloadMetrics = &clientGoWorkqueueMetricsProvider{}
+)
+
+func init() {
+	clientGoRequestMetrics.RegisterWithK8sGoClient()
+	clientGoWorkloadMetrics.RegisterWithK8sGoClient()
+}
+
+// Metrics to be used with a discovery manager.
+type Metrics struct {
+	FailedConfigs     prometheus.Gauge
+	DiscoveredTargets *prometheus.GaugeVec
+	ReceivedUpdates   prometheus.Counter
+	DelayedUpdates    prometheus.Counter
+	SentUpdates       prometheus.Counter
+}
+
+func NewMetrics(registerer prometheus.Registerer, sdManagerName string) (*Metrics, error) {
+	m := &Metrics{}
+
+	m.FailedConfigs = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name:        "prometheus_sd_failed_configs",
+			Help:        "Current number of service discovery configurations that failed to load.",
+			ConstLabels: prometheus.Labels{"name": sdManagerName},
+		},
+	)
+
+	m.DiscoveredTargets = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name:        "prometheus_sd_discovered_targets",
+			Help:        "Current number of discovered targets.",
+			ConstLabels: prometheus.Labels{"name": sdManagerName},
+		},
+		[]string{"config"},
+	)
+
+	m.ReceivedUpdates = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name:        "prometheus_sd_received_updates_total",
+			Help:        "Total number of update events received from the SD providers.",
+			ConstLabels: prometheus.Labels{"name": sdManagerName},
+		},
+	)
+
+	m.DelayedUpdates = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name:        "prometheus_sd_updates_delayed_total",
+			Help:        "Total number of update events that couldn't be sent immediately.",
+			ConstLabels: prometheus.Labels{"name": sdManagerName},
+		},
+	)
+
+	m.SentUpdates = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name:        "prometheus_sd_updates_total",
+			Help:        "Total number of update events sent to the SD consumers.",
+			ConstLabels: prometheus.Labels{"name": sdManagerName},
+		},
+	)
+
+	metrics := []prometheus.Collector{
+		m.FailedConfigs,
+		m.DiscoveredTargets,
+		m.ReceivedUpdates,
+		m.DelayedUpdates,
+		m.SentUpdates,
+	}
+
+	for _, collector := range metrics {
+		err := registerer.Register(collector)
+		if err != nil {
+			return nil, fmt.Errorf("failed to register discovery manager metrics: %w", err)
+		}
+	}
+
+	return m, nil
+}

discovery/metrics_k8s_client.go

@@ -11,10 +11,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package kubernetes
+package discovery
 
 import (
 	"context"
+	"fmt"
 	"net/url"
 	"time"
 
@@ -23,13 +24,22 @@ import (
 	"k8s.io/client-go/util/workqueue"
 )
 
-const workqueueMetricsNamespace = metricsNamespace + "_workqueue"
+// This file registers metrics used by the Kubernetes Go client (k8s.io/client-go).
+// Unfortunately, k8s.io/client-go metrics are global.
+// If we instantiate multiple k8s SD instances, their k8s/client-go metrics will overlap.
+// To prevent us from displaying misleading metrics, we register k8s.io/client-go metrics
+// outside of the Kubernetes SD.
+
+const (
+	KubernetesMetricsNamespace = "prometheus_sd_kubernetes"
+	workqueueMetricsNamespace  = KubernetesMetricsNamespace + "_workqueue"
+)
 
 var (
 	// Metrics for client-go's HTTP requests.
 	clientGoRequestResultMetricVec = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
+			Namespace: KubernetesMetricsNamespace,
 			Name:      "http_request_total",
 			Help:      "Total number of HTTP requests to the Kubernetes API by status code.",
 		},
@@ -37,7 +47,7 @@ var (
 	)
 	clientGoRequestLatencyMetricVec = prometheus.NewSummaryVec(
 		prometheus.SummaryOpts{
-			Namespace:  metricsNamespace,
+			Namespace:  KubernetesMetricsNamespace,
 			Name:       "http_request_duration_seconds",
 			Help:       "Summary of latencies for HTTP requests to the Kubernetes API by endpoint.",
 			Objectives: map[float64]float64{},
@@ -109,17 +119,38 @@ func (noopMetric) Set(float64)     {}
 // Definition of client-go metrics adapters for HTTP requests observation.
 type clientGoRequestMetricAdapter struct{}
 
-func (f *clientGoRequestMetricAdapter) Register(registerer prometheus.Registerer) {
+// Returns all of the Prometheus metrics derived from k8s.io/client-go.
+// This may be used tu register and unregister the metrics.
+func clientGoMetrics() []prometheus.Collector {
+	return []prometheus.Collector{
+		clientGoRequestResultMetricVec,
+		clientGoRequestLatencyMetricVec,
+		clientGoWorkqueueDepthMetricVec,
+		clientGoWorkqueueAddsMetricVec,
+		clientGoWorkqueueLatencyMetricVec,
+		clientGoWorkqueueUnfinishedWorkSecondsMetricVec,
+		clientGoWorkqueueLongestRunningProcessorMetricVec,
+		clientGoWorkqueueWorkDurationMetricVec,
+	}
+}
+
+func RegisterK8sClientMetricsWithPrometheus(registerer prometheus.Registerer) error {
+	for _, collector := range clientGoMetrics() {
+		err := registerer.Register(collector)
+		if err != nil {
+			return fmt.Errorf("failed to register Kubernetes Go Client metrics: %w", err)
+		}
+	}
+	return nil
+}
+
+func (f *clientGoRequestMetricAdapter) RegisterWithK8sGoClient() {
 	metrics.Register(
 		metrics.RegisterOpts{
 			RequestLatency: f,
 			RequestResult:  f,
 		},
 	)
-	registerer.MustRegister(
-		clientGoRequestResultMetricVec,
-		clientGoRequestLatencyMetricVec,
-	)
 }
 
 func (clientGoRequestMetricAdapter) Increment(_ context.Context, code, _, _ string) {
@@ -133,16 +164,8 @@ func (clientGoRequestMetricAdapter) Observe(_ context.Context, _ string, u url.U
 // Definition of client-go workqueue metrics provider definition.
 type clientGoWorkqueueMetricsProvider struct{}
 
-func (f *clientGoWorkqueueMetricsProvider) Register(registerer prometheus.Registerer) {
+func (f *clientGoWorkqueueMetricsProvider) RegisterWithK8sGoClient() {
 	workqueue.SetProvider(f)
-	registerer.MustRegister(
-		clientGoWorkqueueDepthMetricVec,
-		clientGoWorkqueueAddsMetricVec,
-		clientGoWorkqueueLatencyMetricVec,
-		clientGoWorkqueueWorkDurationMetricVec,
-		clientGoWorkqueueUnfinishedWorkSecondsMetricVec,
-		clientGoWorkqueueLongestRunningProcessorMetricVec,
-	)
 }
 
 func (f *clientGoWorkqueueMetricsProvider) NewDepthMetric(name string) workqueue.GaugeMetric {

discovery/moby/docker.go

@@ -26,6 +26,7 @@ import (
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -80,7 +81,7 @@ func (*DockerSDConfig) Name() string { return "docker" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *DockerSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDockerDiscovery(c, opts.Logger)
+	return NewDockerDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -114,7 +115,7 @@ type DockerDiscovery struct {
 }
 
 // NewDockerDiscovery returns a new DockerDiscovery which periodically refreshes its targets.
-func NewDockerDiscovery(conf *DockerSDConfig, logger log.Logger) (*DockerDiscovery, error) {
+func NewDockerDiscovery(conf *DockerSDConfig, logger log.Logger, reg prometheus.Registerer) (*DockerDiscovery, error) {
 	var err error
 
 	d := &DockerDiscovery{
@@ -165,10 +166,13 @@ func NewDockerDiscovery(conf *DockerSDConfig, logger log.Logger) (*DockerDiscove
 	}
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"docker",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "docker",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+		},
 	)
 	return d, nil
 }

discovery/moby/docker_test.go

@@ -19,6 +19,7 @@ import (
 	"testing"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
@@ -37,7 +38,7 @@ host: %s
 	var cfg DockerSDConfig
 	require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg))
 
-	d, err := NewDockerDiscovery(&cfg, log.NewNopLogger())
+	d, err := NewDockerDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 	require.NoError(t, err)
 
 	ctx := context.Background()

discovery/moby/dockerswarm.go

@@ -23,6 +23,7 @@ import (
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/version"
@@ -74,7 +75,7 @@ func (*DockerSwarmSDConfig) Name() string { return "dockerswarm" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *DockerSwarmSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -117,7 +118,7 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *DockerSwarmSDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf *DockerSwarmSDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	var err error
 
 	d := &Discovery{
@@ -168,10 +169,13 @@ func NewDiscovery(conf *DockerSwarmSDConfig, logger log.Logger) (*Discovery, err
 	}
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"dockerswarm",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "dockerswarm",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+		},
 	)
 	return d, nil
 }

discovery/moby/nodes_test.go

@@ -19,6 +19,7 @@ import (
 	"testing"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
@@ -38,7 +39,7 @@ host: %s
 	var cfg DockerSwarmSDConfig
 	require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg))
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 	require.NoError(t, err)
 
 	ctx := context.Background()

discovery/moby/services_test.go

@@ -19,6 +19,7 @@ import (
 	"testing"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
@@ -38,7 +39,7 @@ host: %s
 	var cfg DockerSwarmSDConfig
 	require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg))
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 	require.NoError(t, err)
 
 	ctx := context.Background()
@@ -332,7 +333,7 @@ filters:
 	var cfg DockerSwarmSDConfig
 	require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg))
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 	require.NoError(t, err)
 
 	ctx := context.Background()

discovery/moby/tasks_test.go

@@ -19,6 +19,7 @@ import (
 	"testing"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
@@ -38,7 +39,7 @@ host: %s
 	var cfg DockerSwarmSDConfig
 	require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg))
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 	require.NoError(t, err)
 
 	ctx := context.Background()

discovery/nomad/nomad.go

@@ -49,27 +49,18 @@ const (
 )
 
 // DefaultSDConfig is the default nomad SD configuration.
-var (
-	DefaultSDConfig = SDConfig{
-		AllowStale:       true,
-		HTTPClientConfig: config.DefaultHTTPClientConfig,
-		Namespace:        "default",
-		RefreshInterval:  model.Duration(60 * time.Second),
-		Region:           "global",
-		Server:           "http://localhost:4646",
-		TagSeparator:     ",",
-	}
-
-	failuresCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_nomad_failures_total",
-			Help: "Number of nomad service discovery refresh failures.",
-		})
-)
+var DefaultSDConfig = SDConfig{
+	AllowStale:       true,
+	HTTPClientConfig: config.DefaultHTTPClientConfig,
+	Namespace:        "default",
+	RefreshInterval:  model.Duration(60 * time.Second),
+	Region:           "global",
+	Server:           "http://localhost:4646",
+	TagSeparator:     ",",
+}
 
 func init() {
 	discovery.RegisterConfig(&SDConfig{})
-	prometheus.MustRegister(failuresCount)
 }
 
 // SDConfig is the configuration for nomad based service discovery.
@@ -88,7 +79,7 @@ func (*SDConfig) Name() string { return "nomad" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -121,10 +112,11 @@ type Discovery struct {
 	region          string
 	server          string
 	tagSeparator    string
+	failuresCount   prometheus.Counter
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	d := &Discovery{
 		allowStale:      conf.AllowStale,
 		namespace:       conf.Namespace,
@@ -132,6 +124,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 		region:          conf.Region,
 		server:          conf.Server,
 		tagSeparator:    conf.TagSeparator,
+		failuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_nomad_failures_total",
+				Help: "Number of nomad service discovery refresh failures.",
+			}),
 	}
 
 	HTTPClient, err := config.NewClientFromConfig(conf.HTTPClientConfig, "nomad_sd")
@@ -153,10 +150,14 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 	d.client = client
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"nomad",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "nomad",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+			Metrics:  []prometheus.Collector{d.failuresCount},
+		},
 	)
 	return d, nil
 }
@@ -167,7 +168,7 @@ func (d *Discovery) refresh(context.Context) ([]*targetgroup.Group, error) {
 	}
 	stubs, _, err := d.client.Services().List(opts)
 	if err != nil {
-		failuresCount.Inc()
+		d.failuresCount.Inc()
 		return nil, err
 	}
 
@@ -179,7 +180,7 @@ func (d *Discovery) refresh(context.Context) ([]*targetgroup.Group, error) {
 		for _, service := range stub.Services {
 			instances, _, err := d.client.Services().Get(service.ServiceName, opts)
 			if err != nil {
-				failuresCount.Inc()
+				d.failuresCount.Inc()
 				return nil, fmt.Errorf("failed to fetch services: %w", err)
 			}
 

discovery/nomad/nomad_test.go

@@ -22,6 +22,7 @@ import (
 	"testing"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 )
@@ -127,7 +128,7 @@ func TestConfiguredService(t *testing.T) {
 	conf := &SDConfig{
 		Server: "http://localhost:4646",
 	}
-	_, err := NewDiscovery(conf, nil)
+	_, err := NewDiscovery(conf, nil, prometheus.NewRegistry())
 	require.NoError(t, err)
 }
 
@@ -141,7 +142,7 @@ func TestNomadSDRefresh(t *testing.T) {
 
 	cfg := DefaultSDConfig
 	cfg.Server = endpoint.String()
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 	require.NoError(t, err)
 
 	tgs, err := d.refresh(context.Background())

discovery/openstack/openstack.go

@@ -24,6 +24,7 @@ import (
 	"github.com/gophercloud/gophercloud"
 	"github.com/gophercloud/gophercloud/openstack"
 	"github.com/mwitkow/go-conntrack"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -70,7 +71,7 @@ func (*SDConfig) Name() string { return "openstack" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -134,16 +135,19 @@ type refresher interface {
 }
 
 // NewDiscovery returns a new OpenStack Discoverer which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, l log.Logger) (*refresh.Discovery, error) {
+func NewDiscovery(conf *SDConfig, l log.Logger, reg prometheus.Registerer) (*refresh.Discovery, error) {
 	r, err := newRefresher(conf, l)
 	if err != nil {
 		return nil, err
 	}
 	return refresh.NewDiscovery(
-		l,
-		"openstack",
-		time.Duration(conf.RefreshInterval),
-		r.refresh,
+		refresh.Options{
+			Logger:   l,
+			Mech:     "openstack",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: r.refresh,
+			Registry: reg,
+		},
 	), nil
 }
 

discovery/ovhcloud/ovhcloud.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/ovh/go-ovh/ovh"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -93,7 +94,7 @@ func createClient(config *SDConfig) (*ovh.Client, error) {
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(options discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, options.Logger)
+	return NewDiscovery(c, options.Logger, options.Registerer)
 }
 
 func init() {
@@ -140,16 +141,19 @@ func newRefresher(conf *SDConfig, logger log.Logger) (refresher, error) {
 }
 
 // NewDiscovery returns a new OVHcloud Discoverer which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*refresh.Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*refresh.Discovery, error) {
 	r, err := newRefresher(conf, logger)
 	if err != nil {
 		return nil, err
 	}
 
 	return refresh.NewDiscovery(
-		logger,
-		"ovhcloud",
-		time.Duration(conf.RefreshInterval),
-		r.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "ovhcloud",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: r.refresh,
+			Registry: reg,
+		},
 	), nil
 }

discovery/ovhcloud/ovhcloud_test.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
@@ -122,7 +123,8 @@ func TestDiscoverer(t *testing.T) {
 	conf, _ := getMockConf("vps")
 	logger := testutil.NewLogger(t)
 	_, err := conf.NewDiscoverer(discovery.DiscovererOptions{
-		Logger: logger,
+		Logger:     logger,
+		Registerer: prometheus.NewRegistry(),
 	})
 
 	require.NoError(t, err)

discovery/puppetdb/puppetdb.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/grafana/regexp"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/version"
@@ -83,7 +84,7 @@ func (*SDConfig) Name() string { return "puppetdb" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -130,7 +131,7 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new PuppetDB discovery for the given config.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -156,10 +157,13 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 	}
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"http",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "http",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+		},
 	)
 	return d, nil
 }

discovery/puppetdb/puppetdb_test.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
@@ -62,7 +63,7 @@ func TestPuppetSlashInURL(t *testing.T) {
 			Port:             80,
 			RefreshInterval:  model.Duration(30 * time.Second),
 		}
-		d, err := NewDiscovery(&cfg, log.NewNopLogger())
+		d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 		require.NoError(t, err)
 		require.Equal(t, apiURL, d.url)
 	}
@@ -79,7 +80,7 @@ func TestPuppetDBRefresh(t *testing.T) {
 		RefreshInterval:  model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 	require.NoError(t, err)
 
 	ctx := context.Background()
@@ -120,7 +121,7 @@ func TestPuppetDBRefreshWithParameters(t *testing.T) {
 		RefreshInterval:   model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 	require.NoError(t, err)
 
 	ctx := context.Background()
@@ -172,7 +173,7 @@ func TestPuppetDBInvalidCode(t *testing.T) {
 		RefreshInterval:  model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 	require.NoError(t, err)
 
 	ctx := context.Background()
@@ -193,7 +194,7 @@ func TestPuppetDBInvalidFormat(t *testing.T) {
 		RefreshInterval:  model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 	require.NoError(t, err)
 
 	ctx := context.Background()

discovery/refresh/refresh.go

@@ -22,29 +22,17 @@ import (
 	"github.com/go-kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
-var (
-	failuresCount = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_refresh_failures_total",
-			Help: "Number of refresh failures for the given SD mechanism.",
-		},
-		[]string{"mechanism"},
-	)
-	duration = prometheus.NewSummaryVec(
-		prometheus.SummaryOpts{
-			Name:       "prometheus_sd_refresh_duration_seconds",
-			Help:       "The duration of a refresh in seconds for the given SD mechanism.",
-			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
-		},
-		[]string{"mechanism"},
-	)
-)
-
-func init() {
-	prometheus.MustRegister(duration, failuresCount)
+type Options struct {
+	Logger   log.Logger
+	Mech     string
+	Interval time.Duration
+	RefreshF func(ctx context.Context) ([]*targetgroup.Group, error)
+	Registry prometheus.Registerer
+	Metrics  []prometheus.Collector
 }
 
 // Discovery implements the Discoverer interface.
@@ -54,25 +42,62 @@ type Discovery struct {
 	refreshf func(ctx context.Context) ([]*targetgroup.Group, error)
 
 	failures prometheus.Counter
-	duration prometheus.Observer
+	duration prometheus.Summary
+
+	metricRegisterer discovery.MetricRegisterer
 }
 
 // NewDiscovery returns a Discoverer function that calls a refresh() function at every interval.
-func NewDiscovery(l log.Logger, mech string, interval time.Duration, refreshf func(ctx context.Context) ([]*targetgroup.Group, error)) *Discovery {
-	if l == nil {
-		l = log.NewNopLogger()
+func NewDiscovery(opts Options) *Discovery {
+	var logger log.Logger
+	if opts.Logger == nil {
+		logger = log.NewNopLogger()
+	} else {
+		logger = opts.Logger
 	}
-	return &Discovery{
-		logger:   l,
-		interval: interval,
-		refreshf: refreshf,
-		failures: failuresCount.WithLabelValues(mech),
-		duration: duration.WithLabelValues(mech),
+
+	d := Discovery{
+		logger:   logger,
+		interval: opts.Interval,
+		refreshf: opts.RefreshF,
+		failures: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_refresh_failures_total",
+				Help: "Number of refresh failures for the given SD mechanism.",
+				ConstLabels: prometheus.Labels{
+					"mechanism": opts.Mech,
+				},
+			}),
+		duration: prometheus.NewSummary(
+			prometheus.SummaryOpts{
+				Name:       "prometheus_sd_refresh_duration_seconds",
+				Help:       "The duration of a refresh in seconds for the given SD mechanism.",
+				Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+				ConstLabels: prometheus.Labels{
+					"mechanism": opts.Mech,
+				},
+			}),
 	}
+
+	metrics := []prometheus.Collector{d.failures, d.duration}
+	if opts.Metrics != nil {
+		metrics = append(metrics, opts.Metrics...)
+	}
+
+	d.metricRegisterer = discovery.NewMetricRegisterer(opts.Registry, metrics)
+
+	return &d
 }
 
 // Run implements the Discoverer interface.
 func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
+	err := d.metricRegisterer.RegisterMetrics()
+	if err != nil {
+		level.Error(d.logger).Log("msg", "Unable to register metrics", "err", err.Error())
+		return
+	}
+	defer d.metricRegisterer.UnregisterMetrics()
+
 	// Get an initial set right away.
 	tgs, err := d.refresh(ctx)
 	if err != nil {

discovery/refresh/refresh_test.go

@@ -19,6 +19,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
@@ -65,7 +66,15 @@ func TestRefresh(t *testing.T) {
 		return nil, fmt.Errorf("some error")
 	}
 	interval := time.Millisecond
-	d := NewDiscovery(nil, "test", interval, refresh)
+	d := NewDiscovery(
+		Options{
+			Logger:   nil,
+			Mech:     "test",
+			Interval: interval,
+			RefreshF: refresh,
+			Registry: prometheus.NewRegistry(),
+		},
+	)
 
 	ch := make(chan []*targetgroup.Group)
 	ctx, cancel := context.WithCancel(context.Background())

discovery/scaleway/scaleway.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/scaleway/scaleway-sdk-go/scw"
@@ -160,7 +161,7 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 }
 
 func (c SDConfig) NewDiscoverer(options discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(&c, options.Logger)
+	return NewDiscovery(&c, options.Logger, options.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -177,17 +178,20 @@ func init() {
 // the Discoverer interface.
 type Discovery struct{}
 
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*refresh.Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*refresh.Discovery, error) {
 	r, err := newRefresher(conf)
 	if err != nil {
 		return nil, err
 	}
 
 	return refresh.NewDiscovery(
-		logger,
-		"scaleway",
-		time.Duration(conf.RefreshInterval),
-		r.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "scaleway",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: r.refresh,
+			Registry: reg,
+		},
 	), nil
 }
 

discovery/triton/triton.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/mwitkow/go-conntrack"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -74,7 +75,7 @@ func (*SDConfig) Name() string { return "triton" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return New(opts.Logger, c)
+	return New(opts.Logger, c, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -138,7 +139,7 @@ type Discovery struct {
 }
 
 // New returns a new Discovery which periodically refreshes its targets.
-func New(logger log.Logger, conf *SDConfig) (*Discovery, error) {
+func New(logger log.Logger, conf *SDConfig, reg prometheus.Registerer) (*Discovery, error) {
 	tls, err := config.NewTLSConfig(&conf.TLSConfig)
 	if err != nil {
 		return nil, err
@@ -159,10 +160,13 @@ func New(logger log.Logger, conf *SDConfig) (*Discovery, error) {
 		sdConfig: conf,
 	}
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"triton",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "triton",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+		},
 	)
 	return d, nil
 }

discovery/triton/triton_test.go

@@ -24,6 +24,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
@@ -79,7 +80,7 @@ var (
 )
 
 func newTritonDiscovery(c SDConfig) (*Discovery, error) {
-	return New(nil, &c)
+	return New(nil, &c, prometheus.NewRegistry())
 }
 
 func TestTritonSDNew(t *testing.T) {

discovery/util.go

@@ -0,0 +1,72 @@
+// Copyright 2020 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package discovery
+
+import (
+	"fmt"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// A utility to be used by implementations of discovery.Discoverer
+// which need to manage the lifetime of their metrics.
+type MetricRegisterer interface {
+	RegisterMetrics() error
+	UnregisterMetrics()
+}
+
+// metricRegistererImpl is an implementation of MetricRegisterer.
+type metricRegistererImpl struct {
+	reg     prometheus.Registerer
+	metrics []prometheus.Collector
+}
+
+var _ MetricRegisterer = &metricRegistererImpl{}
+
+// Creates an instance of a MetricRegisterer.
+// Typically called inside the implementation of the NewDiscoverer() method.
+func NewMetricRegisterer(reg prometheus.Registerer, metrics []prometheus.Collector) MetricRegisterer {
+	return &metricRegistererImpl{
+		reg:     reg,
+		metrics: metrics,
+	}
+}
+
+// RegisterMetrics registers the metrics with a Prometheus registerer.
+// If any metric fails to register, it will unregister all metrics that
+// were registered so far, and return an error.
+// Typically called at the start of the SD's Run() method.
+func (rh *metricRegistererImpl) RegisterMetrics() error {
+	for _, collector := range rh.metrics {
+		err := rh.reg.Register(collector)
+		if err != nil {
+			// Unregister all metrics that were registered so far.
+			// This is so that if RegisterMetrics() gets called again,
+			// there will not be an error due to a duplicate registration.
+			rh.UnregisterMetrics()
+
+			return fmt.Errorf("failed to register metric: %w", err)
+		}
+	}
+	return nil
+}
+
+// UnregisterMetrics unregisters the metrics from the same Prometheus
+// registerer which was used to register them.
+// Typically called at the end of the SD's Run() method by a defer statement.
+func (rh *metricRegistererImpl) UnregisterMetrics() {
+	for _, collector := range rh.metrics {
+		rh.reg.Unregister(collector)
+	}
+}

discovery/uyuni/uyuni.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/kolo/xmlrpc"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -115,7 +116,7 @@ func (*SDConfig) Name() string { return "uyuni" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -203,7 +204,7 @@ func getEndpointInfoForSystems(
 }
 
 // NewDiscovery returns a uyuni discovery for the given configuration.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	apiURL, err := url.Parse(conf.Server)
 	if err != nil {
 		return nil, err
@@ -227,10 +228,13 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 	}
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"uyuni",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "uyuni",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+		},
 	)
 	return d, nil
 }

discovery/uyuni/uyuni_test.go

@@ -23,6 +23,8 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/prometheus/client_golang/prometheus"
+
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -35,7 +37,7 @@ func testUpdateServices(respHandler http.HandlerFunc) ([]*targetgroup.Group, err
 		Server: ts.URL,
 	}
 
-	md, err := NewDiscovery(&conf, nil)
+	md, err := NewDiscovery(&conf, nil, prometheus.NewRegistry())
 	if err != nil {
 		return nil, err
 	}
@@ -108,7 +110,7 @@ func TestUyuniSDSkipLogin(t *testing.T) {
 		Server: ts.URL,
 	}
 
-	md, err := NewDiscovery(&conf, nil)
+	md, err := NewDiscovery(&conf, nil, prometheus.NewRegistry())
 	if err != nil {
 		t.Error(err)
 	}

discovery/vultr/vultr.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/version"
@@ -78,7 +79,7 @@ func (*SDConfig) Name() string { return "vultr" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -106,7 +107,7 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	d := &Discovery{
 		port: conf.Port,
 	}
@@ -128,10 +129,13 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 	}
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"vultr",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "vultr",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+		},
 	)
 	return d, nil
 }

discovery/vultr/vultr_test.go

@@ -20,6 +20,7 @@ import (
 	"testing"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 )
@@ -46,7 +47,7 @@ func TestVultrSDRefresh(t *testing.T) {
 
 	cfg := DefaultSDConfig
 	cfg.HTTPClientConfig.BearerToken = APIKey
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 	require.NoError(t, err)
 	endpoint, err := url.Parse(sdMock.Mock.Endpoint())
 	require.NoError(t, err)

discovery/xds/kuma.go

@@ -30,35 +30,12 @@ import (
 	"github.com/prometheus/prometheus/util/strutil"
 )
 
-var (
-	// DefaultKumaSDConfig is the default Kuma MADS SD configuration.
-	DefaultKumaSDConfig = KumaSDConfig{
-		HTTPClientConfig: config.DefaultHTTPClientConfig,
-		RefreshInterval:  model.Duration(15 * time.Second),
-		FetchTimeout:     model.Duration(2 * time.Minute),
-	}
-
-	kumaFetchFailuresCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Namespace: namespace,
-			Name:      "sd_kuma_fetch_failures_total",
-			Help:      "The number of Kuma MADS fetch call failures.",
-		})
-	kumaFetchSkipUpdateCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Namespace: namespace,
-			Name:      "sd_kuma_fetch_skipped_updates_total",
-			Help:      "The number of Kuma MADS fetch calls that result in no updates to the targets.",
-		})
-	kumaFetchDuration = prometheus.NewSummary(
-		prometheus.SummaryOpts{
-			Namespace:  namespace,
-			Name:       "sd_kuma_fetch_duration_seconds",
-			Help:       "The duration of a Kuma MADS fetch call.",
-			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
-		},
-	)
-)
+// DefaultKumaSDConfig is the default Kuma MADS SD configuration.
+var DefaultKumaSDConfig = KumaSDConfig{
+	HTTPClientConfig: config.DefaultHTTPClientConfig,
+	RefreshInterval:  model.Duration(15 * time.Second),
+	FetchTimeout:     model.Duration(2 * time.Minute),
+}
 
 const (
 	// kumaMetaLabelPrefix is the meta prefix used for all kuma meta labels.
@@ -120,7 +97,7 @@ func (c *KumaSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discover
 		logger = log.NewNopLogger()
 	}
 
-	return NewKumaHTTPDiscovery(c, logger)
+	return NewKumaHTTPDiscovery(c, logger, opts.Registerer)
 }
 
 func convertKumaV1MonitoringAssignment(assignment *MonitoringAssignment) []model.LabelSet {
@@ -176,12 +153,16 @@ func kumaMadsV1ResourceParser(resources []*anypb.Any, typeURL string) ([]model.L
 	return targets, nil
 }
 
-func NewKumaHTTPDiscovery(conf *KumaSDConfig, logger log.Logger) (discovery.Discoverer, error) {
+func NewKumaHTTPDiscovery(conf *KumaSDConfig, logger log.Logger, reg prometheus.Registerer) (discovery.Discoverer, error) {
 	// Default to "prometheus" if hostname is unavailable.
-	clientID, err := osutil.GetFQDN()
-	if err != nil {
-		level.Debug(logger).Log("msg", "error getting FQDN", "err", err)
-		clientID = "prometheus"
+	clientID := conf.ClientID
+	if clientID == "" {
+		var err error
+		clientID, err = osutil.GetFQDN()
+		if err != nil {
+			level.Debug(logger).Log("msg", "error getting FQDN", "err", err)
+			clientID = "prometheus"
+		}
 	}
 
 	clientConfig := &HTTPResourceClientConfig{
@@ -203,15 +184,41 @@ func NewKumaHTTPDiscovery(conf *KumaSDConfig, logger log.Logger) (discovery.Disc
 	}
 
 	d := &fetchDiscovery{
-		client:               client,
-		logger:               logger,
-		refreshInterval:      time.Duration(conf.RefreshInterval),
-		source:               "kuma",
-		parseResources:       kumaMadsV1ResourceParser,
-		fetchFailuresCount:   kumaFetchFailuresCount,
-		fetchSkipUpdateCount: kumaFetchSkipUpdateCount,
-		fetchDuration:        kumaFetchDuration,
+		client:          client,
+		logger:          logger,
+		refreshInterval: time.Duration(conf.RefreshInterval),
+		source:          "kuma",
+		parseResources:  kumaMadsV1ResourceParser,
+		fetchFailuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      "sd_kuma_fetch_failures_total",
+				Help:      "The number of Kuma MADS fetch call failures.",
+			}),
+		fetchSkipUpdateCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      "sd_kuma_fetch_skipped_updates_total",
+				Help:      "The number of Kuma MADS fetch calls that result in no updates to the targets.",
+			}),
+		fetchDuration: prometheus.NewSummary(
+			prometheus.SummaryOpts{
+				Namespace:  namespace,
+				Name:       "sd_kuma_fetch_duration_seconds",
+				Help:       "The duration of a Kuma MADS fetch call.",
+				Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+			},
+		),
 	}
 
+	d.metricRegisterer = discovery.NewMetricRegisterer(
+		reg,
+		[]prometheus.Collector{
+			d.fetchFailuresCount,
+			d.fetchSkipUpdateCount,
+			d.fetchDuration,
+		},
+	)
+
 	return d, nil
 }

discovery/xds/kuma_test.go

@@ -21,6 +21,7 @@ import (
 	"time"
 
 	v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/protobuf/proto"
@@ -107,7 +108,7 @@ func getKumaMadsV1DiscoveryResponse(resources ...*MonitoringAssignment) (*v3.Dis
 }
 
 func newKumaTestHTTPDiscovery(c KumaSDConfig) (*fetchDiscovery, error) {
-	kd, err := NewKumaHTTPDiscovery(&c, nopLogger)
+	kd, err := NewKumaHTTPDiscovery(&c, nopLogger, prometheus.NewRegistry())
 	if err != nil {
 		return nil, err
 	}
@@ -204,7 +205,7 @@ func TestNewKumaHTTPDiscovery(t *testing.T) {
 	require.True(t, ok)
 	require.Equal(t, kumaConf.Server, resClient.Server())
 	require.Equal(t, KumaMadsV1ResourceTypeURL, resClient.ResourceTypeURL())
-	require.NotEmpty(t, resClient.ID())
+	require.Equal(t, kumaConf.ClientID, resClient.ID())
 	require.Equal(t, KumaMadsV1ResourceType, resClient.config.ResourceType)
 }
 

discovery/xds/xds.go

@@ -55,6 +55,7 @@ type SDConfig struct {
 	RefreshInterval  model.Duration          `yaml:"refresh_interval,omitempty"`
 	FetchTimeout     model.Duration          `yaml:"fetch_timeout,omitempty"`
 	Server           string                  `yaml:"server,omitempty"`
+	ClientID         string                  `yaml:"client_id,omitempty"`
 }
 
 // mustRegisterMessage registers the provided message type in the typeRegistry, and panics
@@ -69,9 +70,6 @@ func init() {
 	// Register top-level SD Configs.
 	discovery.RegisterConfig(&KumaSDConfig{})
 
-	// Register metrics.
-	prometheus.MustRegister(kumaFetchDuration, kumaFetchSkipUpdateCount, kumaFetchFailuresCount)
-
 	// Register protobuf types that need to be marshalled/ unmarshalled.
 	mustRegisterMessage(protoTypes, (&v3.DiscoveryRequest{}).ProtoReflect().Type())
 	mustRegisterMessage(protoTypes, (&v3.DiscoveryResponse{}).ProtoReflect().Type())
@@ -109,12 +107,20 @@ type fetchDiscovery struct {
 	parseResources resourceParser
 	logger         log.Logger
 
-	fetchDuration        prometheus.Observer
+	fetchDuration        prometheus.Summary
 	fetchSkipUpdateCount prometheus.Counter
 	fetchFailuresCount   prometheus.Counter
+
+	metricRegisterer discovery.MetricRegisterer
 }
 
 func (d *fetchDiscovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
+	err := d.metricRegisterer.RegisterMetrics()
+	if err != nil {
+		level.Error(d.logger).Log("msg", "Unable to register metrics", "err", err.Error())
+		return
+	}
+	defer d.metricRegisterer.UnregisterMetrics()
 	defer d.client.Close()
 
 	ticker := time.NewTicker(d.refreshInterval)

discovery/xds/xds_test.go

@@ -36,6 +36,7 @@ var (
 	sdConf = SDConfig{
 		Server:          "http://127.0.0.1",
 		RefreshInterval: model.Duration(10 * time.Second),
+		ClientID:        "test-id",
 	}
 
 	testFetchFailuresCount = prometheus.NewCounter(

docs/command-line/promtool.md

@@ -324,6 +324,25 @@ Run labels query.
 
 
 
+##### `promtool query analyze`
+
+Run queries against your Prometheus to analyze the usage pattern of certain metrics.
+
+
+
+###### Flags
+
+| Flag | Description | Default |
+| --- | --- | --- |
+| <code class="text-nowrap">--server</code> | Prometheus server to query. |  |
+| <code class="text-nowrap">--type</code> | Type of metric: histogram. |  |
+| <code class="text-nowrap">--duration</code> | Time frame to analyze. | `1h` |
+| <code class="text-nowrap">--time</code> | Query time (RFC3339 or Unix timestamp), defaults to now. |  |
+| <code class="text-nowrap">--match</code> | Series selector. Can be specified multiple times. |  |
+
+
+
+
 ### `promtool debug`
 
 Fetch debug information.
@@ -548,7 +567,7 @@ Dump samples from a TSDB.
 | --- | --- | --- |
 | <code class="text-nowrap">--min-time</code> | Minimum timestamp to dump. | `-9223372036854775808` |
 | <code class="text-nowrap">--max-time</code> | Maximum timestamp to dump. | `9223372036854775807` |
-| <code class="text-nowrap">--match</code> | Series selector. | `{__name__=~'(?s:.*)'}` |
+| <code class="text-nowrap">--match</code> | Series selector. Can be specified multiple times. | `{__name__=~'(?s:.*)'}` |
 
 
 

docs/configuration/configuration.md

@@ -2230,6 +2230,11 @@ See below for the configuration options for Kuma MonitoringAssignment discovery:
 # Address of the Kuma Control Plane's MADS xDS server.
 server: <string>
 
+# Client id is used by Kuma Control Plane to compute Monitoring Assignment for specific Prometheus backend. 
+# This is useful when migrating between multiple Prometheus backends, or having separate backend for each Mesh.
+# When not specified, system hostname/fqdn will be used if available, if not `prometheus` will be used.
+[ client_id: <string> ]
+
 # The time to wait between polling update requests.
 [ refresh_interval: <duration> | default = 30s ]
 
@@ -3614,6 +3619,10 @@ queue_config:
   # Retry upon receiving a 429 status code from the remote-write storage.
   # This is experimental and might change in the future.
   [ retry_on_http_429: <boolean> | default = false ]
+  # If set, any sample that is older than sample_age_limit
+  # will not be sent to the remote storage. The default value is 0s,
+  # which means that all samples are sent.
+  [ sample_age_limit: <duration> | default = 0s ]
 
 # Configures the sending of series metadata to remote storage.
 # Metadata configuration is subject to change at any point

docs/feature_flags.md

@@ -194,3 +194,13 @@ won't work when you push OTLP metrics.
 
 Enables PromQL functions that are considered experimental and whose name or
 semantics could change.
+
+## Created Timestamps Zero Injection
+
+`--enable-feature=created-timestamp-zero-ingestion`
+
+Enables ingestion of created timestamp. Created timestamps are injected as 0 valued samples when appropriate. See [PromCon talk](https://youtu.be/nWf0BfQ5EEA) for details.
+
+Currently Prometheus supports created timestamps only on the traditional Prometheus Protobuf protocol (WIP for other protocols). As a result, when enabling this feature, the Prometheus protobuf scrape protocol will be prioritized (See `scrape_config.scrape_protocols` settings for more details).
+
+Besides enabling this feature in Prometheus, created timestamps need to be exposed by the application being scraped.

docs/storage.md

@@ -17,9 +17,9 @@ Ingested samples are grouped into blocks of two hours. Each two-hour block consi
 of a directory containing a chunks subdirectory containing all the time series samples
 for that window of time, a metadata file, and an index file (which indexes metric names
 and labels to time series in the chunks directory). The samples in the chunks directory
-are grouped together into one or more segment files of up to 512MB each by default. When series are
-deleted via the API, deletion records are stored in separate tombstone files (instead
-of deleting the data immediately from the chunk segments).
+are grouped together into one or more segment files of up to 512MB each by default. When
+series are deleted via the API, deletion records are stored in separate tombstone files
+(instead of deleting the data immediately from the chunk segments).
 
 The current block for incoming samples is kept in memory and is not fully
 persisted. It is secured against crashes by a write-ahead log (WAL) that can be
@@ -58,15 +58,17 @@ A Prometheus server's data directory looks something like this:
         └── 00000000
 ```
 
-
 Note that a limitation of local storage is that it is not clustered or
 replicated. Thus, it is not arbitrarily scalable or durable in the face of
 drive or node outages and should be managed like any other single node
-database. The use of RAID is suggested for storage availability, and [snapshots](querying/api.md#snapshot)
-are recommended for backups. With proper
+database. The use of RAID is suggested for storage availability, and
+[snapshots](querying/api.md#snapshot) are recommended for backups. With proper
 architecture, it is possible to retain years of data in local storage.
 
-Alternatively, external storage may be used via the [remote read/write APIs](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage). Careful evaluation is required for these systems as they vary greatly in durability, performance, and efficiency.
+Alternatively, external storage may be used via the
+[remote read/write APIs](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage).
+Careful evaluation is required for these systems as they vary greatly in durability,
+performance, and efficiency.
 
 For further details on file format, see [TSDB format](/tsdb/docs/format/README.md).
 
@@ -74,40 +76,61 @@ For further details on file format, see [TSDB format](/tsdb/docs/format/README.m
 
 The initial two-hour blocks are eventually compacted into longer blocks in the background.
 
-Compaction will create larger blocks containing data spanning up to 10% of the retention time, or 31 days, whichever is smaller.
+Compaction will create larger blocks containing data spanning up to 10% of the retention time,
+or 31 days, whichever is smaller.
 
 ## Operational aspects
 
 Prometheus has several flags that configure local storage. The most important are:
 
-* `--storage.tsdb.path`: Where Prometheus writes its database. Defaults to `data/`.
-* `--storage.tsdb.retention.time`: When to remove old data. Defaults to `15d`. Overrides `storage.tsdb.retention` if this flag is set to anything other than default.
-* `--storage.tsdb.retention.size`: The maximum number of bytes of storage blocks to retain. The oldest data will be removed first. Defaults to `0` or disabled. Units supported: B, KB, MB, GB, TB, PB, EB. Ex: "512MB". Based on powers-of-2, so 1KB is 1024B. Only the persistent blocks are deleted to honor this retention although WAL and m-mapped chunks are counted in the total size. So the minimum requirement for the disk is the peak space taken by the `wal` (the WAL and Checkpoint) and `chunks_head` (m-mapped Head chunks) directory combined (peaks every 2 hours).
-* `--storage.tsdb.retention`: Deprecated in favor of `storage.tsdb.retention.time`.
-* `--storage.tsdb.wal-compression`: Enables compression of the write-ahead log (WAL). Depending on your data, you can expect the WAL size to be halved with little extra cpu load. This flag was introduced in 2.11.0 and enabled by default in 2.20.0. Note that once enabled, downgrading Prometheus to a version below 2.11.0 will require deleting the WAL.
+- `--storage.tsdb.path`: Where Prometheus writes its database. Defaults to `data/`.
+- `--storage.tsdb.retention.time`: When to remove old data. Defaults to `15d`.
+  Overrides `storage.tsdb.retention` if this flag is set to anything other than default.
+- `--storage.tsdb.retention.size`: The maximum number of bytes of storage blocks to retain.
+  The oldest data will be removed first. Defaults to `0` or disabled. Units supported:
+  B, KB, MB, GB, TB, PB, EB. Ex: "512MB". Based on powers-of-2, so 1KB is 1024B. Only
+  the persistent blocks are deleted to honor this retention although WAL and m-mapped
+  chunks are counted in the total size. So the minimum requirement for the disk is the
+  peak space taken by the `wal` (the WAL and Checkpoint) and `chunks_head`
+  (m-mapped Head chunks) directory combined (peaks every 2 hours).
+- `--storage.tsdb.retention`: Deprecated in favor of `storage.tsdb.retention.time`.
+- `--storage.tsdb.wal-compression`: Enables compression of the write-ahead log (WAL).
+  Depending on your data, you can expect the WAL size to be halved with little extra
+  cpu load. This flag was introduced in 2.11.0 and enabled by default in 2.20.0.
+  Note that once enabled, downgrading Prometheus to a version below 2.11.0 will
+  require deleting the WAL.
 
-Prometheus stores an average of only 1-2 bytes per sample. Thus, to plan the capacity of a Prometheus server, you can use the rough formula:
+Prometheus stores an average of only 1-2 bytes per sample. Thus, to plan the
+capacity of a Prometheus server, you can use the rough formula:
 
 ```
 needed_disk_space = retention_time_seconds * ingested_samples_per_second * bytes_per_sample
 ```
 
-To lower the rate of ingested samples, you can either reduce the number of time series you scrape (fewer targets or fewer series per target), or you can increase the scrape interval. However, reducing the number of series is likely more effective, due to compression of samples within a series.
+To lower the rate of ingested samples, you can either reduce the number of
+time series you scrape (fewer targets or fewer series per target), or you
+can increase the scrape interval. However, reducing the number of series is
+likely more effective, due to compression of samples within a series.
 
 If your local storage becomes corrupted for whatever reason, the best
 strategy to address the problem is to shut down Prometheus then remove the
 entire storage directory. You can also try removing individual block directories,
-or the WAL directory to resolve the problem.  Note that this means losing
+or the WAL directory to resolve the problem. Note that this means losing
 approximately two hours data per block directory. Again, Prometheus's local
 storage is not intended to be durable long-term storage; external solutions
 offer extended retention and data durability.
 
-CAUTION: Non-POSIX compliant filesystems are not supported for Prometheus' local storage as unrecoverable corruptions may happen. NFS filesystems (including AWS's EFS) are not supported. NFS could be POSIX-compliant, but most implementations are not. It is strongly recommended to use a local filesystem for reliability.
+CAUTION: Non-POSIX compliant filesystems are not supported for Prometheus'
+local storage as unrecoverable corruptions may happen. NFS filesystems
+(including AWS's EFS) are not supported. NFS could be POSIX-compliant,
+but most implementations are not. It is strongly recommended to use a
+local filesystem for reliability.
 
 If both time and size retention policies are specified, whichever triggers first
 will be used.
 
-Expired block cleanup happens in the background. It may take up to two hours to remove expired blocks. Blocks must be fully expired before they are removed.
+Expired block cleanup happens in the background. It may take up to two hours
+to remove expired blocks. Blocks must be fully expired before they are removed.
 
 ## Remote storage integrations
 
@@ -119,59 +142,101 @@ a set of interfaces that allow integrating with remote storage systems.
 
 Prometheus integrates with remote storage systems in three ways:
 
-* Prometheus can write samples that it ingests to a remote URL in a standardized format.
-* Prometheus can receive samples from other Prometheus servers in a standardized format.
-* Prometheus can read (back) sample data from a remote URL in a standardized format.
+- Prometheus can write samples that it ingests to a remote URL in a standardized format.
+- Prometheus can receive samples from other Prometheus servers in a standardized format.
+- Prometheus can read (back) sample data from a remote URL in a standardized format.
 
 ![Remote read and write architecture](images/remote_integrations.png)
 
-The read and write protocols both use a snappy-compressed protocol buffer encoding over HTTP. The protocols are not considered as stable APIs yet and may change to use gRPC over HTTP/2 in the future, when all hops between Prometheus and the remote storage can safely be assumed to support HTTP/2.
+The read and write protocols both use a snappy-compressed protocol buffer encoding over
+HTTP. The protocols are not considered as stable APIs yet and may change to use gRPC
+over HTTP/2 in the future, when all hops between Prometheus and the remote storage can
+safely be assumed to support HTTP/2.
 
-For details on configuring remote storage integrations in Prometheus, see the [remote write](configuration/configuration.md#remote_write) and [remote read](configuration/configuration.md#remote_read) sections of the Prometheus configuration documentation.
+For details on configuring remote storage integrations in Prometheus, see the
+[remote write](configuration/configuration.md#remote_write) and
+[remote read](configuration/configuration.md#remote_read) sections of the Prometheus
+configuration documentation.
 
-The built-in remote write receiver can be enabled by setting the `--web.enable-remote-write-receiver` command line flag. When enabled, the remote write receiver endpoint is `/api/v1/write`.
+The built-in remote write receiver can be enabled by setting the
+`--web.enable-remote-write-receiver` command line flag. When enabled,
+the remote write receiver endpoint is `/api/v1/write`.
 
-For details on the request and response messages, see the [remote storage protocol buffer definitions](https://github.com/prometheus/prometheus/blob/main/prompb/remote.proto).
+For details on the request and response messages, see the
+[remote storage protocol buffer definitions](https://github.com/prometheus/prometheus/blob/main/prompb/remote.proto).
 
-Note that on the read path, Prometheus only fetches raw series data for a set of label selectors and time ranges from the remote end. All PromQL evaluation on the raw data still happens in Prometheus itself. This means that remote read queries have some scalability limit, since all necessary data needs to be loaded into the querying Prometheus server first and then processed there. However, supporting fully distributed evaluation of PromQL was deemed infeasible for the time being.
+Note that on the read path, Prometheus only fetches raw series data for a set of
+label selectors and time ranges from the remote end. All PromQL evaluation on the
+raw data still happens in Prometheus itself. This means that remote read queries
+have some scalability limit, since all necessary data needs to be loaded into the
+querying Prometheus server first and then processed there. However, supporting
+fully distributed evaluation of PromQL was deemed infeasible for the time being.
 
 ### Existing integrations
 
-To learn more about existing integrations with remote storage systems, see the [Integrations documentation](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage).
+To learn more about existing integrations with remote storage systems, see the
+[Integrations documentation](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage).
 
 ## Backfilling from OpenMetrics format
 
 ### Overview
 
-If a user wants to create blocks into the TSDB from data that is in [OpenMetrics](https://openmetrics.io/) format, they can do so using backfilling. However, they should be careful and note that it is not safe to backfill data from the last 3 hours (the current head block) as this time range may overlap with the current head block Prometheus is still mutating. Backfilling will create new TSDB blocks, each containing two hours of metrics data. This limits the memory requirements of block creation. Compacting the two hour blocks into larger blocks is later done by the Prometheus server itself.
+If a user wants to create blocks into the TSDB from data that is in
+[OpenMetrics](https://openmetrics.io/) format, they can do so using backfilling.
+However, they should be careful and note that it is not safe to backfill data
+from the last 3 hours (the current head block) as this time range may overlap
+with the current head block Prometheus is still mutating. Backfilling will
+create new TSDB blocks, each containing two hours of metrics data. This limits
+the memory requirements of block creation. Compacting the two hour blocks into
+larger blocks is later done by the Prometheus server itself.
 
-A typical use case is to migrate metrics data from a different monitoring system or time-series database to Prometheus. To do so, the user must first convert the source data into [OpenMetrics](https://openmetrics.io/)  format, which is the input format for the backfilling as described below.
+A typical use case is to migrate metrics data from a different monitoring system
+or time-series database to Prometheus. To do so, the user must first convert the
+source data into [OpenMetrics](https://openmetrics.io/) format, which is the
+input format for the backfilling as described below.
 
 ### Usage
 
-Backfilling can be used via the Promtool command line. Promtool will write the blocks to a directory. By default this output directory is ./data/, you can change it by using the name of the desired output directory as an optional argument in the sub-command.
+Backfilling can be used via the Promtool command line. Promtool will write the blocks
+to a directory. By default this output directory is ./data/, you can change it by
+using the name of the desired output directory as an optional argument in the sub-command.
 
 ```
 promtool tsdb create-blocks-from openmetrics <input file> [<output directory>]
 ```
 
-After the creation of the blocks, move it to the data directory of Prometheus. If there is an overlap with the existing blocks in Prometheus, the flag `--storage.tsdb.allow-overlapping-blocks` needs to be set for Prometheus versions v2.38 and below. Note that any backfilled data is subject to the retention configured for your Prometheus server (by time or size).
+After the creation of the blocks, move it to the data directory of Prometheus.
+If there is an overlap with the existing blocks in Prometheus, the flag
+`--storage.tsdb.allow-overlapping-blocks` needs to be set for Prometheus versions
+v2.38 and below. Note that any backfilled data is subject to the retention
+configured for your Prometheus server (by time or size).
 
 #### Longer Block Durations
 
-By default, the promtool will use the default block duration (2h) for the blocks; this behavior is the most generally applicable and correct. However, when backfilling data over a long range of times, it may be advantageous to use a larger value for the block duration to backfill faster and prevent additional compactions by TSDB later.
+By default, the promtool will use the default block duration (2h) for the blocks;
+this behavior is the most generally applicable and correct. However, when backfilling
+data over a long range of times, it may be advantageous to use a larger value for
+the block duration to backfill faster and prevent additional compactions by TSDB later.
 
-The `--max-block-duration` flag allows the user to configure a maximum duration of blocks. The backfilling tool will pick a suitable block duration no larger than this.
+The `--max-block-duration` flag allows the user to configure a maximum duration of blocks.
+The backfilling tool will pick a suitable block duration no larger than this.
 
-While larger blocks may improve the performance of backfilling large datasets, drawbacks exist as well. Time-based retention policies must keep the entire block around if even one sample of the (potentially large) block is still within the retention policy. Conversely, size-based retention policies will remove the entire block even if the TSDB only goes over the size limit in a minor way.
+While larger blocks may improve the performance of backfilling large datasets,
+drawbacks exist as well. Time-based retention policies must keep the entire block
+around if even one sample of the (potentially large) block is still within the
+retention policy. Conversely, size-based retention policies will remove the entire
+block even if the TSDB only goes over the size limit in a minor way.
 
-Therefore, backfilling with few blocks, thereby choosing a larger block duration, must be done with care and is not recommended for any production instances.
+Therefore, backfilling with few blocks, thereby choosing a larger block duration,
+must be done with care and is not recommended for any production instances.
 
 ## Backfilling for Recording Rules
 
 ### Overview
 
-When a new recording rule is created, there is no historical data for it. Recording rule data only exists from the creation time on. `promtool` makes it possible to create historical recording rule data.
+When a new recording rule is created, there is no historical data for it.
+Recording rule data only exists from the creation time on.
+`promtool` makes it possible to create historical recording rule data.
 
 ### Usage
 
@@ -187,14 +252,26 @@ $ promtool tsdb create-blocks-from rules \
     rules.yaml rules2.yaml
 ```
 
-The recording rule files provided should be a normal [Prometheus rules file](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/).
+The recording rule files provided should be a normal
+[Prometheus rules file](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/).
 
-The output of `promtool tsdb create-blocks-from rules` command is a directory that contains blocks with the historical rule data for all rules in the recording rule files. By default, the output directory is `data/`. In order to make use of this new block data, the blocks must be moved to a running Prometheus instance data dir `storage.tsdb.path` (for Prometheus versions v2.38 and below, the flag `--storage.tsdb.allow-overlapping-blocks` must be enabled). Once moved, the new blocks will merge with existing blocks when the next compaction runs.
+The output of `promtool tsdb create-blocks-from rules` command is a directory that
+contains blocks with the historical rule data for all rules in the recording rule
+files. By default, the output directory is `data/`. In order to make use of this
+new block data, the blocks must be moved to a running Prometheus instance data dir
+`storage.tsdb.path` (for Prometheus versions v2.38 and below, the flag
+`--storage.tsdb.allow-overlapping-blocks` must be enabled). Once moved, the new
+blocks will merge with existing blocks when the next compaction runs.
 
 ### Limitations
 
-- If you run the rule backfiller multiple times with the overlapping start/end times, blocks containing the same data will be created each time the rule backfiller is run.
+- If you run the rule backfiller multiple times with the overlapping start/end times,
+  blocks containing the same data will be created each time the rule backfiller is run.
 - All rules in the recording rule files will be evaluated.
-- If the `interval` is set in the recording rule file that will take priority over the `eval-interval` flag in the rule backfill command.
+- If the `interval` is set in the recording rule file that will take priority over
+  the `eval-interval` flag in the rule backfill command.
 - Alerts are currently ignored if they are in the recording rule file.
-- Rules in the same group cannot see the results of previous rules. Meaning that rules that refer to other rules being backfilled is not supported. A workaround is to backfill multiple times and create the dependent data first (and move dependent data to the Prometheus server data dir so that it is accessible from the Prometheus API).
+- Rules in the same group cannot see the results of previous rules. Meaning that rules
+  that refer to other rules being backfilled is not supported. A workaround is to
+  backfill multiple times and create the dependent data first (and move dependent
+  data to the Prometheus server data dir so that it is accessible from the Prometheus API).

documentation/examples/custom-sd/adapter-usage/main.go

@@ -268,6 +268,11 @@ func main() {
 	if err != nil {
 		fmt.Println("err: ", err)
 	}
+
+	if err != nil {
+		level.Error(logger).Log("msg", "failed to create discovery metrics", "err", err)
+		os.Exit(1)
+	}
 	sdAdapter := adapter.NewAdapter(ctx, *outputFile, "exampleSD", disc, logger)
 	sdAdapter.Run()
 

documentation/examples/custom-sd/adapter/adapter.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 
 	"github.com/prometheus/prometheus/discovery"
@@ -167,7 +168,7 @@ func NewAdapter(ctx context.Context, file, name string, d discovery.Discoverer,
 		ctx:     ctx,
 		disc:    d,
 		groups:  make(map[string]*customSD),
-		manager: discovery.NewManager(ctx, logger),
+		manager: discovery.NewManager(ctx, logger, prometheus.NewRegistry()),
 		output:  file,
 		name:    name,
 		logger:  logger,

documentation/examples/custom-sd/adapter/adapter_test.go

@@ -226,6 +226,7 @@ func TestWriteOutput(t *testing.T) {
 	require.NoError(t, err)
 	defer os.Remove(tmpfile.Name())
 	tmpfile.Close()
+	require.NoError(t, err)
 	adapter := NewAdapter(ctx, tmpfile.Name(), "test_sd", nil, nil)
 	require.NoError(t, adapter.writeOutput())
 }

documentation/examples/remote_storage/go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/gogo/protobuf v1.3.2
 	github.com/golang/snappy v0.0.4
 	github.com/influxdata/influxdb v1.11.2
-	github.com/prometheus/client_golang v1.17.0
+	github.com/prometheus/client_golang v1.18.0
 	github.com/prometheus/common v0.45.0
 	github.com/prometheus/prometheus v0.48.0
 	github.com/stretchr/testify v1.8.4
@@ -48,7 +48,7 @@ require (
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/common/sigv4 v0.1.0 // indirect
-	github.com/prometheus/procfs v0.11.1 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
 	go.opentelemetry.io/collector/pdata v1.0.0-rcv0016 // indirect
 	go.opentelemetry.io/collector/semconv v0.87.0 // indirect
@@ -62,7 +62,7 @@ require (
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/oauth2 v0.13.0 // indirect
-	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect

documentation/examples/remote_storage/go.sum

@@ -194,8 +194,8 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
-github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
+github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=
+github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -213,8 +213,8 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
-github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
 github.com/prometheus/prometheus v0.48.0 h1:yrBloImGQ7je4h8M10ujGh4R6oxYQJQKlMuETwNskGk=
 github.com/prometheus/prometheus v0.48.0/go.mod h1:SRw624aMAxTfryAcP8rOjg4S/sHHaetx2lyJJ2nM83g=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
@@ -317,8 +317,8 @@ golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
-golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

go.mod

@@ -5,11 +5,13 @@ go 1.20
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0
-	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4 v4.2.1
-	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.3.0
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0
+	github.com/Code-Hex/go-generics-cache v1.3.1
 	github.com/alecthomas/kingpin/v2 v2.4.0
 	github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9
 	github.com/aws/aws-sdk-go v1.48.14
+	github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3
 	github.com/cespare/xxhash/v2 v2.2.0
 	github.com/dennwc/varint v1.0.0
 	github.com/digitalocean/godo v1.106.0
@@ -43,11 +45,10 @@ require (
 	github.com/oklog/run v1.1.0
 	github.com/oklog/ulid v1.3.1
 	github.com/ovh/go-ovh v1.4.3
-	github.com/pkg/errors v0.9.1
 	github.com/prometheus/alertmanager v0.26.0
-	github.com/prometheus/client_golang v1.17.0
+	github.com/prometheus/client_golang v1.18.0
 	github.com/prometheus/client_model v0.5.0
-	github.com/prometheus/common v0.45.0
+	github.com/prometheus/common v0.45.1-0.20231122191551-832cd6e99f99
 	github.com/prometheus/common/assets v0.2.0
 	github.com/prometheus/common/sigv4 v0.1.0
 	github.com/prometheus/exporter-toolkit v0.10.0
@@ -93,9 +94,7 @@ require (
 	cloud.google.com/go/compute v1.23.3 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect
-	github.com/Code-Hex/go-generics-cache v1.3.1
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
@@ -167,8 +166,9 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.2 // indirect
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/procfs v0.11.1 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/objx v0.5.0 // indirect
 	github.com/xhit/go-str2duration/v2 v2.1.0 // indirect

go.sum

@@ -40,14 +40,12 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZM
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG+S3q8UoJcmyU6nUeunJcMDHcRYHhs=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 h1:d81/ng9rET2YqdVkVwkb6EXeRrLJIwyGnJcAlAWKwhs=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4 v4.2.1 h1:UPeCRD+XY7QlaGQte2EVI2iOcWvUYA2XY8w5T/8v0NQ=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4 v4.2.1/go.mod h1:oGV6NlB0cvi1ZbYRR2UN44QHxWFyGk+iylgD0qaMXjA=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.1.2 h1:mLY+pNLjCUeKhgnAJWAKhEUQM+RJQo2H1fuGSw1Ky1E=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0 h1:QM6sE5k2ZT/vI5BEe0r7mqjsUSnhVBFbOsVkEuaEfiA=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0/go.mod h1:243D9iHbcQXoFUtgHJwL7gl2zx1aDuDMjvBZVGr2uW0=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1 h1:bWh0Z2rOEDfB/ywv/l0iHN1JgyazE6kW/aIA89+CEK0=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1/go.mod h1:Bzf34hhAE9NSxailk8xVeLEZbUjOXcC+GnU1mMKdhLw=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.0.0 h1:ECsQtyERDVz3NP3kvDOTLvbQhqWp/x9EsGKtb4ogUr8=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.3.0 h1:qgs/VAMSR+9qFhwTw4OwF2NbVuw+2m83pVZJjqkKQMw=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.3.0/go.mod h1:uYt4CfhkJA9o0FN7jfE5minm/i4nUE4MjGUJkzB6Zs8=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0 h1:bXwSugBiSbgtz7rOtbfGf+woewp4f06orW9OP5BjHLA=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0/go.mod h1:Y/HgrePTmGy9HjdSGTqZNa+apUpTVIEVKXJyARP2lrk=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 h1:7CBQ+Ei8SP2c6ydQTGCCrS35bDxgTMfoP2miAwK++OU=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 h1:WpB/QDNLpMw72xHJc34BNNykqSOeEJDAWkhf0u12/Jk=
@@ -94,6 +92,8 @@ github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2z
 github.com/aws/aws-sdk-go v1.48.14 h1:nVLrp+F84SG+xGiFMfe1TE6ZV6smF+42tuuNgYGV30s=
 github.com/aws/aws-sdk-go v1.48.14/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
+github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 h1:6df1vn4bBlDDo4tARvBm7l6KA9iVMnE3NWizDeWSrps=
+github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3/go.mod h1:CIWtjkly68+yqLPbvwwR/fjNJA/idrtULjZWh2v1ys0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -636,8 +636,8 @@ github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeD
 github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
-github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
+github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=
+github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -653,8 +653,8 @@ github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8b
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
 github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
-github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM=
-github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY=
+github.com/prometheus/common v0.45.1-0.20231122191551-832cd6e99f99 h1:V5ajRiLiCQGO+ggTr+07gMUcTqlIMMkDBfrJe5zKLmc=
+github.com/prometheus/common v0.45.1-0.20231122191551-832cd6e99f99/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY=
 github.com/prometheus/common/assets v0.2.0 h1:0P5OrzoHrYBOSM1OigWL3mY8ZvV2N4zIE/5AahrSrfM=
 github.com/prometheus/common/assets v0.2.0/go.mod h1:D17UVUE12bHbim7HzwUvtqm6gwBEaDQ0F+hIGbFbccI=
 github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4=
@@ -667,8 +667,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
-github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=

model/histogram/float_histogram.go

@@ -318,7 +318,6 @@ func (h *FloatHistogram) Sub(other *FloatHistogram) *FloatHistogram {
 		hPositiveSpans, hPositiveBuckets = reduceResolution(hPositiveSpans, hPositiveBuckets, h.Schema, other.Schema, false, true)
 		hNegativeSpans, hNegativeBuckets = reduceResolution(hNegativeSpans, hNegativeBuckets, h.Schema, other.Schema, false, true)
 		h.Schema = other.Schema
-
 	case other.Schema > h.Schema:
 		otherPositiveSpans, otherPositiveBuckets = reduceResolution(otherPositiveSpans, otherPositiveBuckets, other.Schema, h.Schema, false, false)
 		otherNegativeSpans, otherNegativeBuckets = reduceResolution(otherNegativeSpans, otherNegativeBuckets, other.Schema, h.Schema, false, false)

model/histogram/histogram.go

@@ -282,50 +282,49 @@ func (h *Histogram) Compact(maxEmptyBuckets int) *Histogram {
 	return h
 }
 
-// ToFloat returns a FloatHistogram representation of the Histogram. It is a
-// deep copy (e.g. spans are not shared).
-func (h *Histogram) ToFloat() *FloatHistogram {
-	var (
-		positiveSpans, negativeSpans     []Span
-		positiveBuckets, negativeBuckets []float64
-	)
-	if len(h.PositiveSpans) != 0 {
-		positiveSpans = make([]Span, len(h.PositiveSpans))
-		copy(positiveSpans, h.PositiveSpans)
+// ToFloat returns a FloatHistogram representation of the Histogram. It is a deep
+// copy (e.g. spans are not shared). The function accepts a FloatHistogram as an
+// argument whose memory will be reused and overwritten if provided. If this
+// argument is nil, a new FloatHistogram will be allocated.
+func (h *Histogram) ToFloat(fh *FloatHistogram) *FloatHistogram {
+	if fh == nil {
+		fh = &FloatHistogram{}
 	}
-	if len(h.NegativeSpans) != 0 {
-		negativeSpans = make([]Span, len(h.NegativeSpans))
-		copy(negativeSpans, h.NegativeSpans)
-	}
-	if len(h.PositiveBuckets) != 0 {
-		positiveBuckets = make([]float64, len(h.PositiveBuckets))
-		var current float64
-		for i, b := range h.PositiveBuckets {
-			current += float64(b)
-			positiveBuckets[i] = current
-		}
-	}
-	if len(h.NegativeBuckets) != 0 {
-		negativeBuckets = make([]float64, len(h.NegativeBuckets))
-		var current float64
-		for i, b := range h.NegativeBuckets {
-			current += float64(b)
-			negativeBuckets[i] = current
-		}
+	fh.CounterResetHint = h.CounterResetHint
+	fh.Schema = h.Schema
+	fh.ZeroThreshold = h.ZeroThreshold
+	fh.ZeroCount = float64(h.ZeroCount)
+	fh.Count = float64(h.Count)
+	fh.Sum = h.Sum
+
+	fh.PositiveSpans = resize(fh.PositiveSpans, len(h.PositiveSpans))
+	copy(fh.PositiveSpans, h.PositiveSpans)
+
+	fh.NegativeSpans = resize(fh.NegativeSpans, len(h.NegativeSpans))
+	copy(fh.NegativeSpans, h.NegativeSpans)
+
+	fh.PositiveBuckets = resize(fh.PositiveBuckets, len(h.PositiveBuckets))
+	var currentPositive float64
+	for i, b := range h.PositiveBuckets {
+		currentPositive += float64(b)
+		fh.PositiveBuckets[i] = currentPositive
 	}
 
-	return &FloatHistogram{
-		CounterResetHint: h.CounterResetHint,
-		Schema:           h.Schema,
-		ZeroThreshold:    h.ZeroThreshold,
-		ZeroCount:        float64(h.ZeroCount),
-		Count:            float64(h.Count),
-		Sum:              h.Sum,
-		PositiveSpans:    positiveSpans,
-		NegativeSpans:    negativeSpans,
-		PositiveBuckets:  positiveBuckets,
-		NegativeBuckets:  negativeBuckets,
+	fh.NegativeBuckets = resize(fh.NegativeBuckets, len(h.NegativeBuckets))
+	var currentNegative float64
+	for i, b := range h.NegativeBuckets {
+		currentNegative += float64(b)
+		fh.NegativeBuckets[i] = currentNegative
 	}
+
+	return fh
+}
+
+func resize[T any](items []T, n int) []T {
+	if cap(items) < n {
+		return make([]T, n)
+	}
+	return items[:n]
 }
 
 // Validate validates consistency between span and bucket slices. Also, buckets are checked

model/histogram/histogram_test.go

@@ -408,9 +408,57 @@ func TestHistogramToFloat(t *testing.T) {
 		},
 		NegativeBuckets: []int64{1, 2, -2, 1, -1, 0},
 	}
-	fh := h.ToFloat()
+	cases := []struct {
+		name string
+		fh   *FloatHistogram
+	}{
+		{name: "without prior float histogram"},
+		{name: "prior float histogram with more buckets", fh: &FloatHistogram{
+			Schema:        2,
+			Count:         3,
+			Sum:           5,
+			ZeroThreshold: 4,
+			ZeroCount:     1,
+			PositiveSpans: []Span{
+				{Offset: 1, Length: 2},
+				{Offset: 1, Length: 2},
+				{Offset: 1, Length: 2},
+			},
+			PositiveBuckets: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9},
+			NegativeSpans: []Span{
+				{Offset: 20, Length: 6},
+				{Offset: 12, Length: 7},
+				{Offset: 33, Length: 10},
+			},
+			NegativeBuckets: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9},
+		}},
+		{name: "prior float histogram with fewer buckets", fh: &FloatHistogram{
+			Schema:        2,
+			Count:         3,
+			Sum:           5,
+			ZeroThreshold: 4,
+			ZeroCount:     1,
+			PositiveSpans: []Span{
+				{Offset: 1, Length: 2},
+				{Offset: 1, Length: 2},
+				{Offset: 1, Length: 2},
+			},
+			PositiveBuckets: []float64{1, 2},
+			NegativeSpans: []Span{
+				{Offset: 20, Length: 6},
+				{Offset: 12, Length: 7},
+				{Offset: 33, Length: 10},
+			},
+			NegativeBuckets: []float64{1, 2},
+		}},
+	}
 
-	require.Equal(t, h.String(), fh.String())
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			fh := h.ToFloat(c.fh)
+			require.Equal(t, h.String(), fh.String())
+		})
+	}
 }
 
 // TestHistogramEquals tests both Histogram and FloatHistogram.
@@ -436,14 +484,14 @@ func TestHistogramEquals(t *testing.T) {
 	equals := func(h1, h2 Histogram) {
 		require.True(t, h1.Equals(&h2))
 		require.True(t, h2.Equals(&h1))
-		h1f, h2f := h1.ToFloat(), h2.ToFloat()
+		h1f, h2f := h1.ToFloat(nil), h2.ToFloat(nil)
 		require.True(t, h1f.Equals(h2f))
 		require.True(t, h2f.Equals(h1f))
 	}
 	notEquals := func(h1, h2 Histogram) {
 		require.False(t, h1.Equals(&h2))
 		require.False(t, h2.Equals(&h1))
-		h1f, h2f := h1.ToFloat(), h2.ToFloat()
+		h1f, h2f := h1.ToFloat(nil), h2.ToFloat(nil)
 		require.False(t, h1f.Equals(h2f))
 		require.False(t, h2f.Equals(h1f))
 	}
@@ -950,7 +998,7 @@ func TestHistogramValidation(t *testing.T) {
 				return
 			}
 
-			fh := tc.h.ToFloat()
+			fh := tc.h.ToFloat(nil)
 			if err := fh.Validate(); tc.errMsg != "" {
 				require.EqualError(t, err, tc.errMsg)
 			} else {