From 690d5d0b5122a041706cecaf3e7c56651b5cc398 Mon Sep 17 00:00:00 2001
From: Julian Wiedmann <jwi@linux.ibm.com>
Date: Thu, 2 Sep 2021 13:55:40 +0300
Subject: [PATCH 001/115] consoles: exclude iowait and steal from CPU
 Utilisation

'iowait' and 'steal' indicate specific idle/wait states, which shouldn't
be counted into CPU Utilisation. Also see
https://github.com/prometheus-operator/kube-prometheus/pull/796 and
https://github.com/kubernetes-monitoring/kubernetes-mixin/pull/667.

Per the iostat man page:

%idle
    Show the percentage of time that the CPU or CPUs were idle and the
    system did not have an outstanding disk I/O request.

%iowait
     Show the percentage of time that the CPU or CPUs were idle during
     which the system had an outstanding disk I/O request.

%steal
     Show the percentage of time spent in involuntary wait by the
     virtual CPU or CPUs while the hypervisor was servicing another
     virtual processor.

Signed-off-by: Julian Wiedmann <jwi@linux.ibm.com>
---
 consoles/node-cpu.html      | 2 +-
 consoles/node-overview.html | 2 +-
 consoles/node.html          | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/consoles/node-cpu.html b/consoles/node-cpu.html
index d6c515d2d..284ad738f 100644
--- a/consoles/node-cpu.html
+++ b/consoles/node-cpu.html
@@ -47,7 +47,7 @@
   <script>
   new PromConsole.Graph({
     node: document.querySelector("#cpuGraph"),
-    expr: "sum by (mode)(irate(node_cpu_seconds_total{job='node',instance='{{ .Params.instance }}',mode!='idle'}[5m]))",
+    expr: "sum by (mode)(irate(node_cpu_seconds_total{job='node',instance='{{ .Params.instance }}',mode!='idle',mode!='iowait',mode!='steal'}[5m]))",
     renderer: 'area',
     max: {{ with printf "count(count by (cpu)(node_cpu_seconds_total{job='node',instance='%s'}))" .Params.instance | query }}{{ . | first | value }}{{ else}}undefined{{end}},
     yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
diff --git a/consoles/node-overview.html b/consoles/node-overview.html
index 92f53ba87..4ae8984b9 100644
--- a/consoles/node-overview.html
+++ b/consoles/node-overview.html
@@ -69,7 +69,7 @@
   <script>
   new PromConsole.Graph({
     node: document.querySelector("#cpuGraph"),
-    expr: "sum by (mode)(irate(node_cpu_seconds_total{job='node',instance='{{ .Params.instance }}',mode!='idle'}[5m]))",
+    expr: "sum by (mode)(irate(node_cpu_seconds_total{job='node',instance='{{ .Params.instance }}',mode!='idle',mode!='iowait',mode!='steal'}[5m]))",
     renderer: 'area',
     max: {{ with printf "count(count by (cpu)(node_cpu_seconds_total{job='node',instance='%s'}))" .Params.instance | query }}{{ . | first | value }}{{ else}}undefined{{end}},
     yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
diff --git a/consoles/node.html b/consoles/node.html
index 9a37544ee..c1dfc1a89 100644
--- a/consoles/node.html
+++ b/consoles/node.html
@@ -21,7 +21,7 @@
 <tr>
   <td><a href="node-overview.html?instance={{ .Labels.instance }}">{{ reReplaceAll "(.*?://)([^:/]+?)(:\\d+)?/.*" "$2" .Labels.instance }}</a></td>
   <td{{ if eq (. | value) 1.0 }}>Yes{{ else }} class="alert-danger">No{{ end }}</td>
-  <td>{{ template "prom_query_drilldown" (args (printf "100 * (1 - avg by(instance)(irate(node_cpu_seconds_total{job='node',mode='idle',instance='%s'}[5m])))" .Labels.instance) "%" "printf.1f") }}</td>
+  <td>{{ template "prom_query_drilldown" (args (printf "100 * (1 - avg by(instance) (sum without(mode) (irate(node_cpu_seconds_total{job='node',mode=~'idle|iowait|steal',instance='%s'}[5m]))))" .Labels.instance) "%" "printf.1f") }}</td>
   <td>{{ template "prom_query_drilldown" (args (printf "node_memory_MemFree_bytes{job='node',instance='%s'} + node_memory_Cached_bytes{job='node',instance='%s'} + node_memory_Buffers_bytes{job='node',instance='%s'}" .Labels.instance .Labels.instance .Labels.instance) "B" "humanize1024") }}</td>
 </tr>
 {{ else }}

From 67294b092539163bb8c947a00cae4a983f42524a Mon Sep 17 00:00:00 2001
From: Michael <cloudyonspring@126.com>
Date: Tue, 15 Nov 2022 21:00:15 +0800
Subject: [PATCH 002/115] Tweak line wrappings in docs/storage.md

Signed-off-by: Michael <cloudyonspring@126.com>
---
 docs/storage.md | 159 +++++++++++++++++++++++++++++++++++-------------
 1 file changed, 118 insertions(+), 41 deletions(-)

diff --git a/docs/storage.md b/docs/storage.md
index bcb8f7853..b4c5b6ada 100644
--- a/docs/storage.md
+++ b/docs/storage.md
@@ -17,9 +17,9 @@ Ingested samples are grouped into blocks of two hours. Each two-hour block consi
 of a directory containing a chunks subdirectory containing all the time series samples
 for that window of time, a metadata file, and an index file (which indexes metric names
 and labels to time series in the chunks directory). The samples in the chunks directory
-are grouped together into one or more segment files of up to 512MB each by default. When series are
-deleted via the API, deletion records are stored in separate tombstone files (instead
-of deleting the data immediately from the chunk segments).
+are grouped together into one or more segment files of up to 512MB each by default. When
+series are deleted via the API, deletion records are stored in separate tombstone files
+(instead of deleting the data immediately from the chunk segments).
 
 The current block for incoming samples is kept in memory and is not fully
 persisted. It is secured against crashes by a write-ahead log (WAL) that can be
@@ -58,15 +58,17 @@ A Prometheus server's data directory looks something like this:
         └── 00000000
 ```
 
-
 Note that a limitation of local storage is that it is not clustered or
 replicated. Thus, it is not arbitrarily scalable or durable in the face of
 drive or node outages and should be managed like any other single node
-database. The use of RAID is suggested for storage availability, and [snapshots](querying/api.md#snapshot)
-are recommended for backups. With proper
+database. The use of RAID is suggested for storage availability, and
+[snapshots](querying/api.md#snapshot) are recommended for backups. With proper
 architecture, it is possible to retain years of data in local storage.
 
-Alternatively, external storage may be used via the [remote read/write APIs](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage). Careful evaluation is required for these systems as they vary greatly in durability, performance, and efficiency.
+Alternatively, external storage may be used via the
+[remote read/write APIs](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage).
+Careful evaluation is required for these systems as they vary greatly in durability,
+performance, and efficiency.
 
 For further details on file format, see [TSDB format](/tsdb/docs/format/README.md).
 
@@ -74,40 +76,61 @@ For further details on file format, see [TSDB format](/tsdb/docs/format/README.m
 
 The initial two-hour blocks are eventually compacted into longer blocks in the background.
 
-Compaction will create larger blocks containing data spanning up to 10% of the retention time, or 31 days, whichever is smaller.
+Compaction will create larger blocks containing data spanning up to 10% of the retention time,
+or 31 days, whichever is smaller.
 
 ## Operational aspects
 
 Prometheus has several flags that configure local storage. The most important are:
 
-* `--storage.tsdb.path`: Where Prometheus writes its database. Defaults to `data/`.
-* `--storage.tsdb.retention.time`: When to remove old data. Defaults to `15d`. Overrides `storage.tsdb.retention` if this flag is set to anything other than default.
-* `--storage.tsdb.retention.size`: The maximum number of bytes of storage blocks to retain. The oldest data will be removed first. Defaults to `0` or disabled. Units supported: B, KB, MB, GB, TB, PB, EB. Ex: "512MB". Based on powers-of-2, so 1KB is 1024B. Only the persistent blocks are deleted to honor this retention although WAL and m-mapped chunks are counted in the total size. So the minimum requirement for the disk is the peak space taken by the `wal` (the WAL and Checkpoint) and `chunks_head` (m-mapped Head chunks) directory combined (peaks every 2 hours).
-* `--storage.tsdb.retention`: Deprecated in favor of `storage.tsdb.retention.time`.
-* `--storage.tsdb.wal-compression`: Enables compression of the write-ahead log (WAL). Depending on your data, you can expect the WAL size to be halved with little extra cpu load. This flag was introduced in 2.11.0 and enabled by default in 2.20.0. Note that once enabled, downgrading Prometheus to a version below 2.11.0 will require deleting the WAL.
+- `--storage.tsdb.path`: Where Prometheus writes its database. Defaults to `data/`.
+- `--storage.tsdb.retention.time`: When to remove old data. Defaults to `15d`.
+  Overrides `storage.tsdb.retention` if this flag is set to anything other than default.
+- `--storage.tsdb.retention.size`: The maximum number of bytes of storage blocks to retain.
+  The oldest data will be removed first. Defaults to `0` or disabled. Units supported:
+  B, KB, MB, GB, TB, PB, EB. Ex: "512MB". Based on powers-of-2, so 1KB is 1024B. Only
+  the persistent blocks are deleted to honor this retention although WAL and m-mapped
+  chunks are counted in the total size. So the minimum requirement for the disk is the
+  peak space taken by the `wal` (the WAL and Checkpoint) and `chunks_head`
+  (m-mapped Head chunks) directory combined (peaks every 2 hours).
+- `--storage.tsdb.retention`: Deprecated in favor of `storage.tsdb.retention.time`.
+- `--storage.tsdb.wal-compression`: Enables compression of the write-ahead log (WAL).
+  Depending on your data, you can expect the WAL size to be halved with little extra
+  cpu load. This flag was introduced in 2.11.0 and enabled by default in 2.20.0.
+  Note that once enabled, downgrading Prometheus to a version below 2.11.0 will
+  require deleting the WAL.
 
-Prometheus stores an average of only 1-2 bytes per sample. Thus, to plan the capacity of a Prometheus server, you can use the rough formula:
+Prometheus stores an average of only 1-2 bytes per sample. Thus, to plan the
+capacity of a Prometheus server, you can use the rough formula:
 
 ```
 needed_disk_space = retention_time_seconds * ingested_samples_per_second * bytes_per_sample
 ```
 
-To lower the rate of ingested samples, you can either reduce the number of time series you scrape (fewer targets or fewer series per target), or you can increase the scrape interval. However, reducing the number of series is likely more effective, due to compression of samples within a series.
+To lower the rate of ingested samples, you can either reduce the number of
+time series you scrape (fewer targets or fewer series per target), or you
+can increase the scrape interval. However, reducing the number of series is
+likely more effective, due to compression of samples within a series.
 
 If your local storage becomes corrupted for whatever reason, the best
 strategy to address the problem is to shut down Prometheus then remove the
 entire storage directory. You can also try removing individual block directories,
-or the WAL directory to resolve the problem.  Note that this means losing
+or the WAL directory to resolve the problem. Note that this means losing
 approximately two hours data per block directory. Again, Prometheus's local
 storage is not intended to be durable long-term storage; external solutions
 offer extended retention and data durability.
 
-CAUTION: Non-POSIX compliant filesystems are not supported for Prometheus' local storage as unrecoverable corruptions may happen. NFS filesystems (including AWS's EFS) are not supported. NFS could be POSIX-compliant, but most implementations are not. It is strongly recommended to use a local filesystem for reliability.
+CAUTION: Non-POSIX compliant filesystems are not supported for Prometheus'
+local storage as unrecoverable corruptions may happen. NFS filesystems
+(including AWS's EFS) are not supported. NFS could be POSIX-compliant,
+but most implementations are not. It is strongly recommended to use a
+local filesystem for reliability.
 
 If both time and size retention policies are specified, whichever triggers first
 will be used.
 
-Expired block cleanup happens in the background. It may take up to two hours to remove expired blocks. Blocks must be fully expired before they are removed.
+Expired block cleanup happens in the background. It may take up to two hours
+to remove expired blocks. Blocks must be fully expired before they are removed.
 
 ## Remote storage integrations
 
@@ -119,59 +142,101 @@ a set of interfaces that allow integrating with remote storage systems.
 
 Prometheus integrates with remote storage systems in three ways:
 
-* Prometheus can write samples that it ingests to a remote URL in a standardized format.
-* Prometheus can receive samples from other Prometheus servers in a standardized format.
-* Prometheus can read (back) sample data from a remote URL in a standardized format.
+- Prometheus can write samples that it ingests to a remote URL in a standardized format.
+- Prometheus can receive samples from other Prometheus servers in a standardized format.
+- Prometheus can read (back) sample data from a remote URL in a standardized format.
 
 ![Remote read and write architecture](images/remote_integrations.png)
 
-The read and write protocols both use a snappy-compressed protocol buffer encoding over HTTP. The protocols are not considered as stable APIs yet and may change to use gRPC over HTTP/2 in the future, when all hops between Prometheus and the remote storage can safely be assumed to support HTTP/2.
+The read and write protocols both use a snappy-compressed protocol buffer encoding over
+HTTP. The protocols are not considered as stable APIs yet and may change to use gRPC
+over HTTP/2 in the future, when all hops between Prometheus and the remote storage can
+safely be assumed to support HTTP/2.
 
-For details on configuring remote storage integrations in Prometheus, see the [remote write](configuration/configuration.md#remote_write) and [remote read](configuration/configuration.md#remote_read) sections of the Prometheus configuration documentation.
+For details on configuring remote storage integrations in Prometheus, see the
+[remote write](configuration/configuration.md#remote_write) and
+[remote read](configuration/configuration.md#remote_read) sections of the Prometheus
+configuration documentation.
 
-The built-in remote write receiver can be enabled by setting the `--web.enable-remote-write-receiver` command line flag. When enabled, the remote write receiver endpoint is `/api/v1/write`.
+The built-in remote write receiver can be enabled by setting the
+`--web.enable-remote-write-receiver` command line flag. When enabled,
+the remote write receiver endpoint is `/api/v1/write`.
 
-For details on the request and response messages, see the [remote storage protocol buffer definitions](https://github.com/prometheus/prometheus/blob/main/prompb/remote.proto).
+For details on the request and response messages, see the
+[remote storage protocol buffer definitions](https://github.com/prometheus/prometheus/blob/main/prompb/remote.proto).
 
-Note that on the read path, Prometheus only fetches raw series data for a set of label selectors and time ranges from the remote end. All PromQL evaluation on the raw data still happens in Prometheus itself. This means that remote read queries have some scalability limit, since all necessary data needs to be loaded into the querying Prometheus server first and then processed there. However, supporting fully distributed evaluation of PromQL was deemed infeasible for the time being.
+Note that on the read path, Prometheus only fetches raw series data for a set of
+label selectors and time ranges from the remote end. All PromQL evaluation on the
+raw data still happens in Prometheus itself. This means that remote read queries
+have some scalability limit, since all necessary data needs to be loaded into the
+querying Prometheus server first and then processed there. However, supporting
+fully distributed evaluation of PromQL was deemed infeasible for the time being.
 
 ### Existing integrations
 
-To learn more about existing integrations with remote storage systems, see the [Integrations documentation](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage).
+To learn more about existing integrations with remote storage systems, see the
+[Integrations documentation](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage).
 
 ## Backfilling from OpenMetrics format
 
 ### Overview
 
-If a user wants to create blocks into the TSDB from data that is in [OpenMetrics](https://openmetrics.io/) format, they can do so using backfilling. However, they should be careful and note that it is not safe to backfill data from the last 3 hours (the current head block) as this time range may overlap with the current head block Prometheus is still mutating. Backfilling will create new TSDB blocks, each containing two hours of metrics data. This limits the memory requirements of block creation. Compacting the two hour blocks into larger blocks is later done by the Prometheus server itself.
+If a user wants to create blocks into the TSDB from data that is in
+[OpenMetrics](https://openmetrics.io/) format, they can do so using backfilling.
+However, they should be careful and note that it is not safe to backfill data
+from the last 3 hours (the current head block) as this time range may overlap
+with the current head block Prometheus is still mutating. Backfilling will
+create new TSDB blocks, each containing two hours of metrics data. This limits
+the memory requirements of block creation. Compacting the two hour blocks into
+larger blocks is later done by the Prometheus server itself.
 
-A typical use case is to migrate metrics data from a different monitoring system or time-series database to Prometheus. To do so, the user must first convert the source data into [OpenMetrics](https://openmetrics.io/)  format, which is the input format for the backfilling as described below.
+A typical use case is to migrate metrics data from a different monitoring system
+or time-series database to Prometheus. To do so, the user must first convert the
+source data into [OpenMetrics](https://openmetrics.io/) format, which is the
+input format for the backfilling as described below.
 
 ### Usage
 
-Backfilling can be used via the Promtool command line. Promtool will write the blocks to a directory. By default this output directory is ./data/, you can change it by using the name of the desired output directory as an optional argument in the sub-command.
+Backfilling can be used via the Promtool command line. Promtool will write the blocks
+to a directory. By default this output directory is ./data/, you can change it by
+using the name of the desired output directory as an optional argument in the sub-command.
 
 ```
 promtool tsdb create-blocks-from openmetrics <input file> [<output directory>]
 ```
 
-After the creation of the blocks, move it to the data directory of Prometheus. If there is an overlap with the existing blocks in Prometheus, the flag `--storage.tsdb.allow-overlapping-blocks` needs to be set for Prometheus versions v2.38 and below. Note that any backfilled data is subject to the retention configured for your Prometheus server (by time or size).
+After the creation of the blocks, move it to the data directory of Prometheus.
+If there is an overlap with the existing blocks in Prometheus, the flag
+`--storage.tsdb.allow-overlapping-blocks` needs to be set for Prometheus versions
+v2.38 and below. Note that any backfilled data is subject to the retention
+configured for your Prometheus server (by time or size).
 
 #### Longer Block Durations
 
-By default, the promtool will use the default block duration (2h) for the blocks; this behavior is the most generally applicable and correct. However, when backfilling data over a long range of times, it may be advantageous to use a larger value for the block duration to backfill faster and prevent additional compactions by TSDB later.
+By default, the promtool will use the default block duration (2h) for the blocks;
+this behavior is the most generally applicable and correct. However, when backfilling
+data over a long range of times, it may be advantageous to use a larger value for
+the block duration to backfill faster and prevent additional compactions by TSDB later.
 
-The `--max-block-duration` flag allows the user to configure a maximum duration of blocks. The backfilling tool will pick a suitable block duration no larger than this.
+The `--max-block-duration` flag allows the user to configure a maximum duration of blocks.
+The backfilling tool will pick a suitable block duration no larger than this.
 
-While larger blocks may improve the performance of backfilling large datasets, drawbacks exist as well. Time-based retention policies must keep the entire block around if even one sample of the (potentially large) block is still within the retention policy. Conversely, size-based retention policies will remove the entire block even if the TSDB only goes over the size limit in a minor way.
+While larger blocks may improve the performance of backfilling large datasets,
+drawbacks exist as well. Time-based retention policies must keep the entire block
+around if even one sample of the (potentially large) block is still within the
+retention policy. Conversely, size-based retention policies will remove the entire
+block even if the TSDB only goes over the size limit in a minor way.
 
-Therefore, backfilling with few blocks, thereby choosing a larger block duration, must be done with care and is not recommended for any production instances.
+Therefore, backfilling with few blocks, thereby choosing a larger block duration,
+must be done with care and is not recommended for any production instances.
 
 ## Backfilling for Recording Rules
 
 ### Overview
 
-When a new recording rule is created, there is no historical data for it. Recording rule data only exists from the creation time on. `promtool` makes it possible to create historical recording rule data.
+When a new recording rule is created, there is no historical data for it.
+Recording rule data only exists from the creation time on.
+`promtool` makes it possible to create historical recording rule data.
 
 ### Usage
 
@@ -187,14 +252,26 @@ $ promtool tsdb create-blocks-from rules \
     rules.yaml rules2.yaml
 ```
 
-The recording rule files provided should be a normal [Prometheus rules file](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/).
+The recording rule files provided should be a normal
+[Prometheus rules file](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/).
 
-The output of `promtool tsdb create-blocks-from rules` command is a directory that contains blocks with the historical rule data for all rules in the recording rule files. By default, the output directory is `data/`. In order to make use of this new block data, the blocks must be moved to a running Prometheus instance data dir `storage.tsdb.path` (for Prometheus versions v2.38 and below, the flag `--storage.tsdb.allow-overlapping-blocks` must be enabled). Once moved, the new blocks will merge with existing blocks when the next compaction runs.
+The output of `promtool tsdb create-blocks-from rules` command is a directory that
+contains blocks with the historical rule data for all rules in the recording rule
+files. By default, the output directory is `data/`. In order to make use of this
+new block data, the blocks must be moved to a running Prometheus instance data dir
+`storage.tsdb.path` (for Prometheus versions v2.38 and below, the flag
+`--storage.tsdb.allow-overlapping-blocks` must be enabled). Once moved, the new
+blocks will merge with existing blocks when the next compaction runs.
 
 ### Limitations
 
-- If you run the rule backfiller multiple times with the overlapping start/end times, blocks containing the same data will be created each time the rule backfiller is run.
+- If you run the rule backfiller multiple times with the overlapping start/end times,
+  blocks containing the same data will be created each time the rule backfiller is run.
 - All rules in the recording rule files will be evaluated.
-- If the `interval` is set in the recording rule file that will take priority over the `eval-interval` flag in the rule backfill command.
+- If the `interval` is set in the recording rule file that will take priority over
+  the `eval-interval` flag in the rule backfill command.
 - Alerts are currently ignored if they are in the recording rule file.
-- Rules in the same group cannot see the results of previous rules. Meaning that rules that refer to other rules being backfilled is not supported. A workaround is to backfill multiple times and create the dependent data first (and move dependent data to the Prometheus server data dir so that it is accessible from the Prometheus API).
+- Rules in the same group cannot see the results of previous rules. Meaning that rules
+  that refer to other rules being backfilled is not supported. A workaround is to
+  backfill multiple times and create the dependent data first (and move dependent
+  data to the Prometheus server data dir so that it is accessible from the Prometheus API).

From 6fe8217ce4ab9a161aeba0c375a26bc2102beed0 Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Sat, 21 Oct 2023 12:38:01 +0000
Subject: [PATCH 003/115] tsdb: shrink txRing with smaller integers

4 billion active transactions ought to be enough for anyone.

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
---
 tsdb/head_read.go |  2 +-
 tsdb/head_test.go |  2 +-
 tsdb/isolation.go | 16 ++++++++--------
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/tsdb/head_read.go b/tsdb/head_read.go
index 6964d5472..6e74b74a6 100644
--- a/tsdb/head_read.go
+++ b/tsdb/head_read.go
@@ -724,7 +724,7 @@ func (s *memSeries) iterator(id chunks.HeadChunkID, c chunkenc.Chunk, isoState *
 
 		// Removing the extra transactionIDs that are relevant for samples that
 		// come after this chunk, from the total transactionIDs.
-		appendIDsToConsider := s.txs.txIDCount - (totalSamples - (previousSamples + numSamples))
+		appendIDsToConsider := int(s.txs.txIDCount) - (totalSamples - (previousSamples + numSamples))
 
 		// Iterate over the appendIDs, find the first one that the isolation state says not
 		// to return.
diff --git a/tsdb/head_test.go b/tsdb/head_test.go
index 54fd469a3..616270100 100644
--- a/tsdb/head_test.go
+++ b/tsdb/head_test.go
@@ -2550,7 +2550,7 @@ func TestIsolationAppendIDZeroIsNoop(t *testing.T) {
 
 	ok, _ := s.append(0, 0, 0, cOpts)
 	require.True(t, ok, "Series append failed.")
-	require.Equal(t, 0, s.txs.txIDCount, "Series should not have an appendID after append with appendID=0.")
+	require.Equal(t, 0, int(s.txs.txIDCount), "Series should not have an appendID after append with appendID=0.")
 }
 
 func TestHeadSeriesChunkRace(t *testing.T) {
diff --git a/tsdb/isolation.go b/tsdb/isolation.go
index e436884a8..19eba9340 100644
--- a/tsdb/isolation.go
+++ b/tsdb/isolation.go
@@ -240,8 +240,8 @@ func (i *isolation) closeAppend(appendID uint64) {
 // The transactionID ring buffer.
 type txRing struct {
 	txIDs     []uint64
-	txIDFirst int // Position of the first id in the ring.
-	txIDCount int // How many ids in the ring.
+	txIDFirst uint32 // Position of the first id in the ring.
+	txIDCount uint32 // How many ids in the ring.
 }
 
 func newTxRing(capacity int) *txRing {
@@ -251,7 +251,7 @@ func newTxRing(capacity int) *txRing {
 }
 
 func (txr *txRing) add(appendID uint64) {
-	if txr.txIDCount == len(txr.txIDs) {
+	if int(txr.txIDCount) == len(txr.txIDs) {
 		// Ring buffer is full, expand by doubling.
 		newRing := make([]uint64, txr.txIDCount*2)
 		idx := copy(newRing, txr.txIDs[txr.txIDFirst:])
@@ -260,12 +260,12 @@ func (txr *txRing) add(appendID uint64) {
 		txr.txIDFirst = 0
 	}
 
-	txr.txIDs[(txr.txIDFirst+txr.txIDCount)%len(txr.txIDs)] = appendID
+	txr.txIDs[int(txr.txIDFirst+txr.txIDCount)%len(txr.txIDs)] = appendID
 	txr.txIDCount++
 }
 
 func (txr *txRing) cleanupAppendIDsBelow(bound uint64) {
-	pos := txr.txIDFirst
+	pos := int(txr.txIDFirst)
 
 	for txr.txIDCount > 0 {
 		if txr.txIDs[pos] < bound {
@@ -281,7 +281,7 @@ func (txr *txRing) cleanupAppendIDsBelow(bound uint64) {
 		}
 	}
 
-	txr.txIDFirst %= len(txr.txIDs)
+	txr.txIDFirst %= uint32(len(txr.txIDs))
 }
 
 func (txr *txRing) iterator() *txRingIterator {
@@ -296,7 +296,7 @@ func (txr *txRing) iterator() *txRingIterator {
 type txRingIterator struct {
 	ids []uint64
 
-	pos int
+	pos uint32
 }
 
 func (it *txRingIterator) At() uint64 {
@@ -305,7 +305,7 @@ func (it *txRingIterator) At() uint64 {
 
 func (it *txRingIterator) Next() {
 	it.pos++
-	if it.pos == len(it.ids) {
+	if int(it.pos) == len(it.ids) {
 		it.pos = 0
 	}
 }

From 90e98e0235522451bf35d054173702c8837f97b1 Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Sat, 21 Oct 2023 13:45:47 +0000
Subject: [PATCH 004/115] tsdb: create isolation transaction slice on demand

When Prometheus restarts it creates every series read in from the WAL,
but many of those series will be finished, and never receive any more
samples. By defering allocation of the txRing slice to when it is first
needed, we save 32 bytes per stale series.

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
---
 tsdb/head.go      | 2 +-
 tsdb/isolation.go | 9 ++++++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/tsdb/head.go b/tsdb/head.go
index 0ea2b8385..4a6e6ac73 100644
--- a/tsdb/head.go
+++ b/tsdb/head.go
@@ -2011,7 +2011,7 @@ func newMemSeries(lset labels.Labels, id chunks.HeadSeriesRef, isolationDisabled
 		nextAt: math.MinInt64,
 	}
 	if !isolationDisabled {
-		s.txs = newTxRing(4)
+		s.txs = newTxRing(0)
 	}
 	return s
 }
diff --git a/tsdb/isolation.go b/tsdb/isolation.go
index 19eba9340..86330f36e 100644
--- a/tsdb/isolation.go
+++ b/tsdb/isolation.go
@@ -253,7 +253,11 @@ func newTxRing(capacity int) *txRing {
 func (txr *txRing) add(appendID uint64) {
 	if int(txr.txIDCount) == len(txr.txIDs) {
 		// Ring buffer is full, expand by doubling.
-		newRing := make([]uint64, txr.txIDCount*2)
+		newLen := txr.txIDCount * 2
+		if newLen == 0 {
+			newLen = 4
+		}
+		newRing := make([]uint64, newLen)
 		idx := copy(newRing, txr.txIDs[txr.txIDFirst:])
 		copy(newRing[idx:], txr.txIDs[:txr.txIDFirst])
 		txr.txIDs = newRing
@@ -265,6 +269,9 @@ func (txr *txRing) add(appendID uint64) {
 }
 
 func (txr *txRing) cleanupAppendIDsBelow(bound uint64) {
+	if len(txr.txIDs) == 0 {
+		return
+	}
 	pos := int(txr.txIDFirst)
 
 	for txr.txIDCount > 0 {

From f095c33da12958b5d92a7d857264fafb8d4c549a Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Mon, 20 Nov 2023 19:28:05 +0000
Subject: [PATCH 005/115] scrape: simplify TargetsActive function

Since everything was serialized on a single mutex, it's exactly the same
if we process targets in sequence without starting goroutines.

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
---
 scrape/manager.go | 16 +---------------
 1 file changed, 1 insertion(+), 15 deletions(-)

diff --git a/scrape/manager.go b/scrape/manager.go
index a0ac38f6b..96cad00c9 100644
--- a/scrape/manager.go
+++ b/scrape/manager.go
@@ -279,24 +279,10 @@ func (m *Manager) TargetsActive() map[string][]*Target {
 	m.mtxScrape.Lock()
 	defer m.mtxScrape.Unlock()
 
-	var (
-		wg  sync.WaitGroup
-		mtx sync.Mutex
-	)
-
 	targets := make(map[string][]*Target, len(m.scrapePools))
-	wg.Add(len(m.scrapePools))
 	for tset, sp := range m.scrapePools {
-		// Running in parallel limits the blocking time of scrapePool to scrape
-		// interval when there's an update from SD.
-		go func(tset string, sp *scrapePool) {
-			mtx.Lock()
-			targets[tset] = sp.ActiveTargets()
-			mtx.Unlock()
-			wg.Done()
-		}(tset, sp)
+		targets[tset] = sp.ActiveTargets()
 	}
-	wg.Wait()
 	return targets
 }
 

From 34676a240e21705e8832e7be0b40df785bc06eb8 Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Wed, 22 Nov 2023 18:50:57 +0000
Subject: [PATCH 006/115] scrape: consistent function names for metadata

Too confusing to have `MetadataList` and `ListMetadata`, etc.
I standardised on the ones which are in an interface.

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
---
 scrape/metrics.go                  |  4 ++--
 scrape/target.go                   | 10 +++++-----
 storage/remote/metadata_watcher.go |  2 +-
 web/api/v1/api.go                  |  8 ++++----
 4 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/scrape/metrics.go b/scrape/metrics.go
index d74143185..7082bc743 100644
--- a/scrape/metrics.go
+++ b/scrape/metrics.go
@@ -286,8 +286,8 @@ func (mc *MetadataMetricsCollector) Collect(ch chan<- prometheus.Metric) {
 	for tset, targets := range mc.TargetsGatherer.TargetsActive() {
 		var size, length int
 		for _, t := range targets {
-			size += t.MetadataSize()
-			length += t.MetadataLength()
+			size += t.SizeMetadata()
+			length += t.LengthMetadata()
 		}
 
 		ch <- prometheus.MustNewConstMetric(
diff --git a/scrape/target.go b/scrape/target.go
index 8cc8597a4..c100e1bee 100644
--- a/scrape/target.go
+++ b/scrape/target.go
@@ -92,7 +92,7 @@ type MetricMetadata struct {
 	Unit   string
 }
 
-func (t *Target) MetadataList() []MetricMetadata {
+func (t *Target) ListMetadata() []MetricMetadata {
 	t.mtx.RLock()
 	defer t.mtx.RUnlock()
 
@@ -102,7 +102,7 @@ func (t *Target) MetadataList() []MetricMetadata {
 	return t.metadata.ListMetadata()
 }
 
-func (t *Target) MetadataSize() int {
+func (t *Target) SizeMetadata() int {
 	t.mtx.RLock()
 	defer t.mtx.RUnlock()
 
@@ -113,7 +113,7 @@ func (t *Target) MetadataSize() int {
 	return t.metadata.SizeMetadata()
 }
 
-func (t *Target) MetadataLength() int {
+func (t *Target) LengthMetadata() int {
 	t.mtx.RLock()
 	defer t.mtx.RUnlock()
 
@@ -124,8 +124,8 @@ func (t *Target) MetadataLength() int {
 	return t.metadata.LengthMetadata()
 }
 
-// Metadata returns type and help metadata for the given metric.
-func (t *Target) Metadata(metric string) (MetricMetadata, bool) {
+// GetMetadata returns type and help metadata for the given metric.
+func (t *Target) GetMetadata(metric string) (MetricMetadata, bool) {
 	t.mtx.RLock()
 	defer t.mtx.RUnlock()
 
diff --git a/storage/remote/metadata_watcher.go b/storage/remote/metadata_watcher.go
index 21de565ed..abfea3c7b 100644
--- a/storage/remote/metadata_watcher.go
+++ b/storage/remote/metadata_watcher.go
@@ -136,7 +136,7 @@ func (mw *MetadataWatcher) collect() {
 	metadata := []scrape.MetricMetadata{}
 	for _, tset := range mw.manager.TargetsActive() {
 		for _, target := range tset {
-			for _, entry := range target.MetadataList() {
+			for _, entry := range target.ListMetadata() {
 				if _, ok := metadataSet[entry]; !ok {
 					metadata = append(metadata, entry)
 					metadataSet[entry] = struct{}{}
diff --git a/web/api/v1/api.go b/web/api/v1/api.go
index 671df7887..8fa7ce14a 100644
--- a/web/api/v1/api.go
+++ b/web/api/v1/api.go
@@ -1114,7 +1114,7 @@ func (api *API) targetMetadata(r *http.Request) apiFuncResult {
 			}
 			// If no metric is specified, get the full list for the target.
 			if metric == "" {
-				for _, md := range t.MetadataList() {
+				for _, md := range t.ListMetadata() {
 					res = append(res, metricMetadata{
 						Target: t.Labels(),
 						Metric: md.Metric,
@@ -1126,7 +1126,7 @@ func (api *API) targetMetadata(r *http.Request) apiFuncResult {
 				continue
 			}
 			// Get metadata for the specified metric.
-			if md, ok := t.Metadata(metric); ok {
+			if md, ok := t.GetMetadata(metric); ok {
 				res = append(res, metricMetadata{
 					Target: t.Labels(),
 					Type:   md.Type,
@@ -1249,7 +1249,7 @@ func (api *API) metricMetadata(r *http.Request) apiFuncResult {
 	for _, tt := range api.targetRetriever(r.Context()).TargetsActive() {
 		for _, t := range tt {
 			if metric == "" {
-				for _, mm := range t.MetadataList() {
+				for _, mm := range t.ListMetadata() {
 					m := metadata{Type: mm.Type, Help: mm.Help, Unit: mm.Unit}
 					ms, ok := metrics[mm.Metric]
 
@@ -1266,7 +1266,7 @@ func (api *API) metricMetadata(r *http.Request) apiFuncResult {
 				continue
 			}
 
-			if md, ok := t.Metadata(metric); ok {
+			if md, ok := t.GetMetadata(metric); ok {
 				m := metadata{Type: md.Type, Help: md.Help, Unit: md.Unit}
 				ms, ok := metrics[md.Metric]
 

From 0e585bf5c0561990867cbb424045d64de1001f85 Mon Sep 17 00:00:00 2001
From: Erik Sommer <ersotech@posteo.de>
Date: Thu, 23 Nov 2023 17:04:57 +0100
Subject: [PATCH 007/115] add cluster variable to Overview dashboard

Signed-off-by: Erik Sommer <ersotech@posteo.de>
---
 .../prometheus-mixin/dashboards.libsonnet     | 46 ++++++++++---------
 1 file changed, 24 insertions(+), 22 deletions(-)

diff --git a/documentation/prometheus-mixin/dashboards.libsonnet b/documentation/prometheus-mixin/dashboards.libsonnet
index 56db2af20..2c508e6e0 100644
--- a/documentation/prometheus-mixin/dashboards.libsonnet
+++ b/documentation/prometheus-mixin/dashboards.libsonnet
@@ -13,16 +13,18 @@ local template = grafana.template;
       g.dashboard(
         '%(prefix)sOverview' % $._config.grafanaPrometheus
       )
-      .addMultiTemplate('job', 'prometheus_build_info{%(prometheusSelector)s}' % $._config, 'job')
-      .addMultiTemplate('instance', 'prometheus_build_info{job=~"$job"}', 'instance')
+      .addMultiTemplate('cluster', 'prometheus_build_info{%(prometheusSelector)s}' % $._config, 'cluster')
+      .addMultiTemplate('job', 'prometheus_build_info{cluster=~"$cluster"}', 'job')
+      .addMultiTemplate('instance', 'prometheus_build_info{cluster=~"$cluster", job=~"$job"}', 'instance')
       .addRow(
         g.row('Prometheus Stats')
         .addPanel(
           g.panel('Prometheus Stats') +
           g.tablePanel([
-            'count by (job, instance, version) (prometheus_build_info{job=~"$job", instance=~"$instance"})',
-            'max by (job, instance) (time() - process_start_time_seconds{job=~"$job", instance=~"$instance"})',
+            'count by (cluster, job, instance, version) (prometheus_build_info{cluster=~"$cluster", job=~"$job", instance=~"$instance"})',
+            'max by (cluster, job, instance) (time() - process_start_time_seconds{cluster=~"$cluster", job=~"$job", instance=~"$instance"})',
           ], {
+            cluster: { alias: 'Cluster' },
             job: { alias: 'Job' },
             instance: { alias: 'Instance' },
             version: { alias: 'Version' },
@@ -35,12 +37,12 @@ local template = grafana.template;
         g.row('Discovery')
         .addPanel(
           g.panel('Target Sync') +
-          g.queryPanel('sum(rate(prometheus_target_sync_length_seconds_sum{job=~"$job",instance=~"$instance"}[5m])) by (scrape_job) * 1e3', '{{scrape_job}}') +
+          g.queryPanel('sum(rate(prometheus_target_sync_length_seconds_sum{cluster=~"$cluster",job=~"$job",instance=~"$instance"}[5m])) by (cluster, job, scrape_job, instance) * 1e3', '{{cluster}}:{{job}}:{{scrape_job}}') +
           { yaxes: g.yaxes('ms') }
         )
         .addPanel(
           g.panel('Targets') +
-          g.queryPanel('sum(prometheus_sd_discovered_targets{job=~"$job",instance=~"$instance"})', 'Targets') +
+          g.queryPanel('sum by (cluster, job, instance) (prometheus_sd_discovered_targets{cluster=~"$cluster", job=~"$job",instance=~"$instance"})', '{{cluster}}:{{job}}') +
           g.stack
         )
       )
@@ -48,29 +50,29 @@ local template = grafana.template;
         g.row('Retrieval')
         .addPanel(
           g.panel('Average Scrape Interval Duration') +
-          g.queryPanel('rate(prometheus_target_interval_length_seconds_sum{job=~"$job",instance=~"$instance"}[5m]) / rate(prometheus_target_interval_length_seconds_count{job=~"$job",instance=~"$instance"}[5m]) * 1e3', '{{interval}} configured') +
+          g.queryPanel('rate(prometheus_target_interval_length_seconds_sum{cluster=~"$cluster", job=~"$job",instance=~"$instance"}[5m]) / rate(prometheus_target_interval_length_seconds_count{cluster=~"$cluster", job=~"$job",instance=~"$instance"}[5m]) * 1e3', '{{cluster}}:{{job}} {{interval}} configured') +
           { yaxes: g.yaxes('ms') }
         )
         .addPanel(
           g.panel('Scrape failures') +
           g.queryPanel([
-            'sum by (job) (rate(prometheus_target_scrapes_exceeded_body_size_limit_total[1m]))',
-            'sum by (job) (rate(prometheus_target_scrapes_exceeded_sample_limit_total[1m]))',
-            'sum by (job) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total[1m]))',
-            'sum by (job) (rate(prometheus_target_scrapes_sample_out_of_bounds_total[1m]))',
-            'sum by (job) (rate(prometheus_target_scrapes_sample_out_of_order_total[1m]))',
+            'sum by (cluster, job, instance) (rate(prometheus_target_scrapes_exceeded_body_size_limit_total{cluster=~"$cluster",job=~"$job",instance=~"$instance"}[1m]))',
+            'sum by (cluster, job, instance) (rate(prometheus_target_scrapes_exceeded_sample_limit_total{cluster=~"$cluster",job=~"$job",instance=~"$instance"}[1m]))',
+            'sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{cluster=~"$cluster",job=~"$job",instance=~"$instance"}[1m]))',
+            'sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_out_of_bounds_total{cluster=~"$cluster",job=~"$job",instance=~"$instance"}[1m]))',
+            'sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_out_of_order_total{cluster=~"$cluster",job=~"$job",instance=~"$instance"}[1m]))',
           ], [
-            'exceeded body size limit: {{job}}',
-            'exceeded sample limit: {{job}}',
-            'duplicate timestamp: {{job}}',
-            'out of bounds: {{job}}',
-            'out of order: {{job}}',
+            'exceeded body size limit: {{cluster}} {{job}}',
+            'exceeded sample limit: {{cluster}} {{job}}',
+            'duplicate timestamp: {{cluster}} {{job}}',
+            'out of bounds: {{cluster}} {{job}}',
+            'out of order: {{cluster}} {{job}}',
           ]) +
           g.stack
         )
         .addPanel(
           g.panel('Appended Samples') +
-          g.queryPanel('rate(prometheus_tsdb_head_samples_appended_total{job=~"$job",instance=~"$instance"}[5m])', '{{job}} {{instance}}') +
+          g.queryPanel('rate(prometheus_tsdb_head_samples_appended_total{cluster=~"$cluster", job=~"$job",instance=~"$instance"}[5m])', '{{cluster}} {{job}} {{instance}}') +
           g.stack
         )
       )
@@ -78,12 +80,12 @@ local template = grafana.template;
         g.row('Storage')
         .addPanel(
           g.panel('Head Series') +
-          g.queryPanel('prometheus_tsdb_head_series{job=~"$job",instance=~"$instance"}', '{{job}} {{instance}} head series') +
+          g.queryPanel('prometheus_tsdb_head_series{cluster=~"$cluster",job=~"$job",instance=~"$instance"}', '{{cluster}} {{job}} {{instance}} head series') +
           g.stack
         )
         .addPanel(
           g.panel('Head Chunks') +
-          g.queryPanel('prometheus_tsdb_head_chunks{job=~"$job",instance=~"$instance"}', '{{job}} {{instance}} head chunks') +
+          g.queryPanel('prometheus_tsdb_head_chunks{cluster=~"$cluster",job=~"$job",instance=~"$instance"}', '{{cluster}} {{job}} {{instance}} head chunks') +
           g.stack
         )
       )
@@ -91,12 +93,12 @@ local template = grafana.template;
         g.row('Query')
         .addPanel(
           g.panel('Query Rate') +
-          g.queryPanel('rate(prometheus_engine_query_duration_seconds_count{job=~"$job",instance=~"$instance",slice="inner_eval"}[5m])', '{{job}} {{instance}}') +
+          g.queryPanel('rate(prometheus_engine_query_duration_seconds_count{cluster=~"$cluster",job=~"$job",instance=~"$instance",slice="inner_eval"}[5m])', '{{cluster}} {{job}} {{instance}}') +
           g.stack,
         )
         .addPanel(
           g.panel('Stage Duration') +
-          g.queryPanel('max by (slice) (prometheus_engine_query_duration_seconds{quantile="0.9",job=~"$job",instance=~"$instance"}) * 1e3', '{{slice}}') +
+          g.queryPanel('max by (slice) (prometheus_engine_query_duration_seconds{quantile="0.9",cluster=~"$cluster", job=~"$job",instance=~"$instance"}) * 1e3', '{{slice}}') +
           { yaxes: g.yaxes('ms') } +
           g.stack,
         )

From 237bfea46b59600cca77520de033d8896f6d3aeb Mon Sep 17 00:00:00 2001
From: Arve Knudsen <arve.knudsen@gmail.com>
Date: Tue, 28 Nov 2023 16:33:28 +0100
Subject: [PATCH 008/115] `chunks.Reader`: Fix typo in ChunkOrIterable doc
 string.

Also fix comment typo in `FloatHistogram.Sub`.

Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
---
 model/histogram/float_histogram.go | 2 +-
 tsdb/chunks/chunks.go              | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/model/histogram/float_histogram.go b/model/histogram/float_histogram.go
index 1c2f3ebec..cf20b5080 100644
--- a/model/histogram/float_histogram.go
+++ b/model/histogram/float_histogram.go
@@ -303,7 +303,7 @@ func (h *FloatHistogram) Sub(other *FloatHistogram) *FloatHistogram {
 	)
 
 	if other.Schema < h.Schema {
-		panic(fmt.Errorf("cannot subtract histigram with schema %d to %d", other.Schema, h.Schema))
+		panic(fmt.Errorf("cannot subtract histogram with schema %d to %d", other.Schema, h.Schema))
 	} else if other.Schema > h.Schema {
 		otherPositiveSpans, otherPositiveBuckets = reduceResolution(otherPositiveSpans, otherPositiveBuckets, other.Schema, h.Schema, false, false)
 		otherNegativeSpans, otherNegativeBuckets = reduceResolution(otherNegativeSpans, otherNegativeBuckets, other.Schema, h.Schema, false, false)
diff --git a/tsdb/chunks/chunks.go b/tsdb/chunks/chunks.go
index f22285a0c..2c6db3637 100644
--- a/tsdb/chunks/chunks.go
+++ b/tsdb/chunks/chunks.go
@@ -671,7 +671,7 @@ func (s *Reader) Size() int64 {
 	return s.size
 }
 
-// Chunk returns a chunk from a given reference.
+// ChunkOrIterable returns a chunk from a given reference.
 func (s *Reader) ChunkOrIterable(meta Meta) (chunkenc.Chunk, chunkenc.Iterable, error) {
 	sgmIndex, chkStart := BlockChunkRef(meta.Ref).Unpack()
 

From 7319ad6a0ba2f8a6450d777530f9aa6248699515 Mon Sep 17 00:00:00 2001
From: Linas Medziunas <linas.medziunas@gmail.com>
Date: Fri, 8 Dec 2023 10:59:00 +0200
Subject: [PATCH 009/115] promql: simplify Native Histogram arithmetics

Signed-off-by: Linas Medziunas <linas.medziunas@gmail.com>
---
 promql/engine.go    | 33 +++++----------------------------
 promql/functions.go | 19 +++----------------
 2 files changed, 8 insertions(+), 44 deletions(-)

diff --git a/promql/engine.go b/promql/engine.go
index 4fa2a513b..4e0900686 100644
--- a/promql/engine.go
+++ b/promql/engine.go
@@ -2478,22 +2478,12 @@ func vectorElemBinop(op parser.ItemType, lhs, rhs float64, hlhs, hrhs *histogram
 	switch op {
 	case parser.ADD:
 		if hlhs != nil && hrhs != nil {
-			// The histogram being added must have the larger schema
-			// code (i.e. the higher resolution).
-			if hrhs.Schema >= hlhs.Schema {
-				return 0, hlhs.Copy().Add(hrhs).Compact(0), true
-			}
-			return 0, hrhs.Copy().Add(hlhs).Compact(0), true
+			return 0, hlhs.Copy().Add(hrhs).Compact(0), true
 		}
 		return lhs + rhs, nil, true
 	case parser.SUB:
 		if hlhs != nil && hrhs != nil {
-			// The histogram being subtracted must have the larger schema
-			// code (i.e. the higher resolution).
-			if hrhs.Schema >= hlhs.Schema {
-				return 0, hlhs.Copy().Sub(hrhs).Compact(0), true
-			}
-			return 0, hrhs.Copy().Mul(-1).Add(hlhs).Compact(0), true
+			return 0, hlhs.Copy().Sub(hrhs).Compact(0), true
 		}
 		return lhs - rhs, nil, true
 	case parser.MUL:
@@ -2678,13 +2668,7 @@ func (ev *evaluator) aggregation(e *parser.AggregateExpr, grouping []string, par
 			if s.H != nil {
 				group.hasHistogram = true
 				if group.histogramValue != nil {
-					// The histogram being added must have
-					// an equal or larger schema.
-					if s.H.Schema >= group.histogramValue.Schema {
-						group.histogramValue.Add(s.H)
-					} else {
-						group.histogramValue = s.H.Copy().Add(group.histogramValue)
-					}
+					group.histogramValue.Add(s.H)
 				}
 				// Otherwise the aggregation contained floats
 				// previously and will be invalid anyway. No
@@ -2701,15 +2685,8 @@ func (ev *evaluator) aggregation(e *parser.AggregateExpr, grouping []string, par
 				if group.histogramMean != nil {
 					left := s.H.Copy().Div(float64(group.groupCount))
 					right := group.histogramMean.Copy().Div(float64(group.groupCount))
-					// The histogram being added/subtracted must have
-					// an equal or larger schema.
-					if s.H.Schema >= group.histogramMean.Schema {
-						toAdd := right.Mul(-1).Add(left)
-						group.histogramMean.Add(toAdd)
-					} else {
-						toAdd := left.Sub(right)
-						group.histogramMean = toAdd.Add(group.histogramMean)
-					}
+					toAdd := left.Sub(right)
+					group.histogramMean.Add(toAdd)
 				}
 				// Otherwise the aggregation contained floats
 				// previously and will be invalid anyway. No
diff --git a/promql/functions.go b/promql/functions.go
index 06f6f8c71..64066edbb 100644
--- a/promql/functions.go
+++ b/promql/functions.go
@@ -532,15 +532,8 @@ func funcAvgOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNode
 				count++
 				left := h.H.Copy().Div(float64(count))
 				right := mean.Copy().Div(float64(count))
-				// The histogram being added/subtracted must have
-				// an equal or larger schema.
-				if h.H.Schema >= mean.Schema {
-					toAdd := right.Mul(-1).Add(left)
-					mean.Add(toAdd)
-				} else {
-					toAdd := left.Sub(right)
-					mean = toAdd.Add(mean)
-				}
+				toAdd := left.Sub(right)
+				mean.Add(toAdd)
 			}
 			return mean
 		}), nil
@@ -661,13 +654,7 @@ func funcSumOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNode
 		return aggrHistOverTime(vals, enh, func(s Series) *histogram.FloatHistogram {
 			sum := s.Histograms[0].H.Copy()
 			for _, h := range s.Histograms[1:] {
-				// The histogram being added must have
-				// an equal or larger schema.
-				if h.H.Schema >= sum.Schema {
-					sum.Add(h.H)
-				} else {
-					sum = h.H.Copy().Add(sum)
-				}
+				sum.Add(h.H)
 			}
 			return sum
 		}), nil

From 10a82f87fd07b89a5badc935b5535120c1122eb2 Mon Sep 17 00:00:00 2001
From: Filip Petkovski <filip.petkovsky@gmail.com>
Date: Wed, 29 Nov 2023 15:15:57 +0100
Subject: [PATCH 010/115] Enable reusing memory when converting between
 histogram types

The 'ToFloat' method on integer histograms currently allocates new memory
each time it is called.

This commit adds an optional *FloatHistogram parameter that can be used
to reuse span and bucket slices. It is up to the caller to make sure the
input float histogram is not used anymore after the call.

Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
---
 model/histogram/histogram.go          | 79 +++++++++++++--------------
 model/histogram/histogram_test.go     | 58 ++++++++++++++++++--
 promql/engine_test.go                 | 20 +++----
 rules/manager_test.go                 |  4 +-
 scrape/target_test.go                 |  2 +-
 storage/buffer.go                     |  4 +-
 storage/buffer_test.go                |  6 +-
 storage/remote/codec_test.go          |  4 +-
 storage/remote/queue_manager_test.go  |  2 +-
 storage/remote/write_handler_test.go  |  2 +-
 tsdb/block_test.go                    |  4 +-
 tsdb/chunkenc/float_histogram_test.go | 22 ++++----
 tsdb/chunkenc/histogram_test.go       | 10 ++--
 tsdb/compact_test.go                  |  4 +-
 tsdb/db_test.go                       | 14 ++---
 tsdb/head_test.go                     | 10 ++--
 tsdb/record/record_test.go            |  2 +-
 tsdb/wlog/watcher_test.go             |  2 +-
 web/federate.go                       |  2 +-
 web/federate_test.go                  |  4 +-
 20 files changed, 151 insertions(+), 104 deletions(-)

diff --git a/model/histogram/histogram.go b/model/histogram/histogram.go
index fb0185a63..1fe342482 100644
--- a/model/histogram/histogram.go
+++ b/model/histogram/histogram.go
@@ -282,50 +282,49 @@ func (h *Histogram) Compact(maxEmptyBuckets int) *Histogram {
 	return h
 }
 
-// ToFloat returns a FloatHistogram representation of the Histogram. It is a
-// deep copy (e.g. spans are not shared).
-func (h *Histogram) ToFloat() *FloatHistogram {
-	var (
-		positiveSpans, negativeSpans     []Span
-		positiveBuckets, negativeBuckets []float64
-	)
-	if len(h.PositiveSpans) != 0 {
-		positiveSpans = make([]Span, len(h.PositiveSpans))
-		copy(positiveSpans, h.PositiveSpans)
+// ToFloat returns a FloatHistogram representation of the Histogram. It is a deep
+// copy (e.g. spans are not shared). The function accepts a FloatHistogram as an
+// argument whose memory will be reused and overwritten if provided. If this
+// argument is nil, a new FloatHistogram will be allocated.
+func (h *Histogram) ToFloat(fh *FloatHistogram) *FloatHistogram {
+	if fh == nil {
+		fh = &FloatHistogram{}
 	}
-	if len(h.NegativeSpans) != 0 {
-		negativeSpans = make([]Span, len(h.NegativeSpans))
-		copy(negativeSpans, h.NegativeSpans)
-	}
-	if len(h.PositiveBuckets) != 0 {
-		positiveBuckets = make([]float64, len(h.PositiveBuckets))
-		var current float64
-		for i, b := range h.PositiveBuckets {
-			current += float64(b)
-			positiveBuckets[i] = current
-		}
-	}
-	if len(h.NegativeBuckets) != 0 {
-		negativeBuckets = make([]float64, len(h.NegativeBuckets))
-		var current float64
-		for i, b := range h.NegativeBuckets {
-			current += float64(b)
-			negativeBuckets[i] = current
-		}
+	fh.CounterResetHint = h.CounterResetHint
+	fh.Schema = h.Schema
+	fh.ZeroThreshold = h.ZeroThreshold
+	fh.ZeroCount = float64(h.ZeroCount)
+	fh.Count = float64(h.Count)
+	fh.Sum = h.Sum
+
+	fh.PositiveSpans = resize(fh.PositiveSpans, len(h.PositiveSpans))
+	copy(fh.PositiveSpans, h.PositiveSpans)
+
+	fh.NegativeSpans = resize(fh.NegativeSpans, len(h.NegativeSpans))
+	copy(fh.NegativeSpans, h.NegativeSpans)
+
+	fh.PositiveBuckets = resize(fh.PositiveBuckets, len(h.PositiveBuckets))
+	var currentPositive float64
+	for i, b := range h.PositiveBuckets {
+		currentPositive += float64(b)
+		fh.PositiveBuckets[i] = currentPositive
 	}
 
-	return &FloatHistogram{
-		CounterResetHint: h.CounterResetHint,
-		Schema:           h.Schema,
-		ZeroThreshold:    h.ZeroThreshold,
-		ZeroCount:        float64(h.ZeroCount),
-		Count:            float64(h.Count),
-		Sum:              h.Sum,
-		PositiveSpans:    positiveSpans,
-		NegativeSpans:    negativeSpans,
-		PositiveBuckets:  positiveBuckets,
-		NegativeBuckets:  negativeBuckets,
+	fh.NegativeBuckets = resize(fh.NegativeBuckets, len(h.NegativeBuckets))
+	var currentNegative float64
+	for i, b := range h.NegativeBuckets {
+		currentNegative += float64(b)
+		fh.NegativeBuckets[i] = currentNegative
 	}
+
+	return fh
+}
+
+func resize[T any](items []T, n int) []T {
+	if len(items) < n {
+		return make([]T, n)
+	}
+	return items[:n]
 }
 
 // Validate validates consistency between span and bucket slices. Also, buckets are checked
diff --git a/model/histogram/histogram_test.go b/model/histogram/histogram_test.go
index d5aed112a..9a64faaaa 100644
--- a/model/histogram/histogram_test.go
+++ b/model/histogram/histogram_test.go
@@ -408,9 +408,57 @@ func TestHistogramToFloat(t *testing.T) {
 		},
 		NegativeBuckets: []int64{1, 2, -2, 1, -1, 0},
 	}
-	fh := h.ToFloat()
+	cases := []struct {
+		name string
+		fh   *FloatHistogram
+	}{
+		{name: "without prior float histogram"},
+		{name: "prior float histogram with more buckets", fh: &FloatHistogram{
+			Schema:        2,
+			Count:         3,
+			Sum:           5,
+			ZeroThreshold: 4,
+			ZeroCount:     1,
+			PositiveSpans: []Span{
+				{Offset: 1, Length: 2},
+				{Offset: 1, Length: 2},
+				{Offset: 1, Length: 2},
+			},
+			PositiveBuckets: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9},
+			NegativeSpans: []Span{
+				{Offset: 20, Length: 6},
+				{Offset: 12, Length: 7},
+				{Offset: 33, Length: 10},
+			},
+			NegativeBuckets: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9},
+		}},
+		{name: "prior float histogram with fewer buckets", fh: &FloatHistogram{
+			Schema:        2,
+			Count:         3,
+			Sum:           5,
+			ZeroThreshold: 4,
+			ZeroCount:     1,
+			PositiveSpans: []Span{
+				{Offset: 1, Length: 2},
+				{Offset: 1, Length: 2},
+				{Offset: 1, Length: 2},
+			},
+			PositiveBuckets: []float64{1, 2},
+			NegativeSpans: []Span{
+				{Offset: 20, Length: 6},
+				{Offset: 12, Length: 7},
+				{Offset: 33, Length: 10},
+			},
+			NegativeBuckets: []float64{1, 2},
+		}},
+	}
 
-	require.Equal(t, h.String(), fh.String())
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			fh := h.ToFloat(c.fh)
+			require.Equal(t, h.String(), fh.String())
+		})
+	}
 }
 
 // TestHistogramEquals tests both Histogram and FloatHistogram.
@@ -436,14 +484,14 @@ func TestHistogramEquals(t *testing.T) {
 	equals := func(h1, h2 Histogram) {
 		require.True(t, h1.Equals(&h2))
 		require.True(t, h2.Equals(&h1))
-		h1f, h2f := h1.ToFloat(), h2.ToFloat()
+		h1f, h2f := h1.ToFloat(nil), h2.ToFloat(nil)
 		require.True(t, h1f.Equals(h2f))
 		require.True(t, h2f.Equals(h1f))
 	}
 	notEquals := func(h1, h2 Histogram) {
 		require.False(t, h1.Equals(&h2))
 		require.False(t, h2.Equals(&h1))
-		h1f, h2f := h1.ToFloat(), h2.ToFloat()
+		h1f, h2f := h1.ToFloat(nil), h2.ToFloat(nil)
 		require.False(t, h1f.Equals(h2f))
 		require.False(t, h2f.Equals(h1f))
 	}
@@ -950,7 +998,7 @@ func TestHistogramValidation(t *testing.T) {
 				return
 			}
 
-			fh := tc.h.ToFloat()
+			fh := tc.h.ToFloat(nil)
 			if err := fh.Validate(); tc.errMsg != "" {
 				require.EqualError(t, err, tc.errMsg)
 			} else {
diff --git a/promql/engine_test.go b/promql/engine_test.go
index b8fd19299..9ab54dd16 100644
--- a/promql/engine_test.go
+++ b/promql/engine_test.go
@@ -3267,7 +3267,7 @@ func TestNativeHistogram_HistogramCountAndSum(t *testing.T) {
 			app := storage.Appender(context.Background())
 			var err error
 			if floatHisto {
-				_, err = app.AppendHistogram(0, lbls, ts, nil, h.ToFloat())
+				_, err = app.AppendHistogram(0, lbls, ts, nil, h.ToFloat(nil))
 			} else {
 				_, err = app.AppendHistogram(0, lbls, ts, h, nil)
 			}
@@ -3287,7 +3287,7 @@ func TestNativeHistogram_HistogramCountAndSum(t *testing.T) {
 			require.Len(t, vector, 1)
 			require.Nil(t, vector[0].H)
 			if floatHisto {
-				require.Equal(t, h.ToFloat().Count, vector[0].F)
+				require.Equal(t, h.ToFloat(nil).Count, vector[0].F)
 			} else {
 				require.Equal(t, float64(h.Count), vector[0].F)
 			}
@@ -3305,7 +3305,7 @@ func TestNativeHistogram_HistogramCountAndSum(t *testing.T) {
 			require.Len(t, vector, 1)
 			require.Nil(t, vector[0].H)
 			if floatHisto {
-				require.Equal(t, h.ToFloat().Sum, vector[0].F)
+				require.Equal(t, h.ToFloat(nil).Sum, vector[0].F)
 			} else {
 				require.Equal(t, h.Sum, vector[0].F)
 			}
@@ -3433,7 +3433,7 @@ func TestNativeHistogram_HistogramStdDevVar(t *testing.T) {
 				app := storage.Appender(context.Background())
 				var err error
 				if floatHisto {
-					_, err = app.AppendHistogram(0, lbls, ts, nil, tc.h.ToFloat())
+					_, err = app.AppendHistogram(0, lbls, ts, nil, tc.h.ToFloat(nil))
 				} else {
 					_, err = app.AppendHistogram(0, lbls, ts, tc.h, nil)
 				}
@@ -3678,7 +3678,7 @@ func TestNativeHistogram_HistogramQuantile(t *testing.T) {
 				app := storage.Appender(context.Background())
 				var err error
 				if floatHisto {
-					_, err = app.AppendHistogram(0, lbls, ts, nil, c.h.ToFloat())
+					_, err = app.AppendHistogram(0, lbls, ts, nil, c.h.ToFloat(nil))
 				} else {
 					_, err = app.AppendHistogram(0, lbls, ts, c.h, nil)
 				}
@@ -4109,7 +4109,7 @@ func TestNativeHistogram_HistogramFraction(t *testing.T) {
 				app := storage.Appender(context.Background())
 				var err error
 				if floatHisto {
-					_, err = app.AppendHistogram(0, lbls, ts, nil, c.h.ToFloat())
+					_, err = app.AppendHistogram(0, lbls, ts, nil, c.h.ToFloat(nil))
 				} else {
 					_, err = app.AppendHistogram(0, lbls, ts, c.h, nil)
 				}
@@ -4272,7 +4272,7 @@ func TestNativeHistogram_Sum_Count_Add_AvgOperator(t *testing.T) {
 					// Since we mutate h later, we need to create a copy here.
 					var err error
 					if floatHisto {
-						_, err = app.AppendHistogram(0, lbls, ts, nil, h.Copy().ToFloat())
+						_, err = app.AppendHistogram(0, lbls, ts, nil, h.Copy().ToFloat(nil))
 					} else {
 						_, err = app.AppendHistogram(0, lbls, ts, h.Copy(), nil)
 					}
@@ -4282,7 +4282,7 @@ func TestNativeHistogram_Sum_Count_Add_AvgOperator(t *testing.T) {
 					newTs := ts + int64(idx1)*int64(time.Minute/time.Millisecond)
 					// Since we mutate h later, we need to create a copy here.
 					if floatHisto {
-						_, err = app.AppendHistogram(0, lbls, newTs, nil, h.Copy().ToFloat())
+						_, err = app.AppendHistogram(0, lbls, newTs, nil, h.Copy().ToFloat(nil))
 					} else {
 						_, err = app.AppendHistogram(0, lbls, newTs, h.Copy(), nil)
 					}
@@ -4530,7 +4530,7 @@ func TestNativeHistogram_SubOperator(t *testing.T) {
 					// Since we mutate h later, we need to create a copy here.
 					var err error
 					if floatHisto {
-						_, err = app.AppendHistogram(0, lbls, ts, nil, h.Copy().ToFloat())
+						_, err = app.AppendHistogram(0, lbls, ts, nil, h.Copy().ToFloat(nil))
 					} else {
 						_, err = app.AppendHistogram(0, lbls, ts, h.Copy(), nil)
 					}
@@ -4687,7 +4687,7 @@ func TestNativeHistogram_MulDivOperator(t *testing.T) {
 				// Since we mutate h later, we need to create a copy here.
 				var err error
 				if floatHisto {
-					_, err = app.AppendHistogram(0, lbls, ts, nil, h.Copy().ToFloat())
+					_, err = app.AppendHistogram(0, lbls, ts, nil, h.Copy().ToFloat(nil))
 				} else {
 					_, err = app.AppendHistogram(0, lbls, ts, h.Copy(), nil)
 				}
diff --git a/rules/manager_test.go b/rules/manager_test.go
index 0c9a49e46..6418c5a37 100644
--- a/rules/manager_test.go
+++ b/rules/manager_test.go
@@ -1390,9 +1390,9 @@ func TestNativeHistogramsInRecordingRules(t *testing.T) {
 
 	require.Equal(t, labels.FromStrings("__name__", "sum:histogram_metric"), s.Labels())
 
-	expHist := hists[0].ToFloat()
+	expHist := hists[0].ToFloat(nil)
 	for _, h := range hists[1:] {
-		expHist = expHist.Add(h.ToFloat())
+		expHist = expHist.Add(h.ToFloat(nil))
 	}
 
 	it := s.Iterator(nil)
diff --git a/scrape/target_test.go b/scrape/target_test.go
index 6631f328c..f37c75a76 100644
--- a/scrape/target_test.go
+++ b/scrape/target_test.go
@@ -565,7 +565,7 @@ func TestBucketLimitAppender(t *testing.T) {
 				lbls := labels.FromStrings("__name__", "sparse_histogram_series")
 				var err error
 				if floatHisto {
-					fh := c.h.Copy().ToFloat()
+					fh := c.h.Copy().ToFloat(nil)
 					_, err = app.AppendHistogram(0, lbls, ts, nil, fh)
 					if c.expectError {
 						require.Error(t, err)
diff --git a/storage/buffer.go b/storage/buffer.go
index d2d89e042..f686796ca 100644
--- a/storage/buffer.go
+++ b/storage/buffer.go
@@ -202,7 +202,7 @@ func (s hSample) H() *histogram.Histogram {
 }
 
 func (s hSample) FH() *histogram.FloatHistogram {
-	return s.h.ToFloat()
+	return s.h.ToFloat(nil)
 }
 
 func (s hSample) Type() chunkenc.ValueType {
@@ -376,7 +376,7 @@ func (it *sampleRingIterator) AtHistogram() (int64, *histogram.Histogram) {
 
 func (it *sampleRingIterator) AtFloatHistogram() (int64, *histogram.FloatHistogram) {
 	if it.fh == nil {
-		return it.t, it.h.ToFloat()
+		return it.t, it.h.ToFloat(nil)
 	}
 	return it.t, it.fh
 }
diff --git a/storage/buffer_test.go b/storage/buffer_test.go
index c2542f3d9..84f94a008 100644
--- a/storage/buffer_test.go
+++ b/storage/buffer_test.go
@@ -233,7 +233,7 @@ func TestBufferedSeriesIteratorMixedHistograms(t *testing.T) {
 	histograms := tsdbutil.GenerateTestHistograms(2)
 
 	it := NewBufferIterator(NewListSeriesIterator(samples{
-		fhSample{t: 1, fh: histograms[0].ToFloat()},
+		fhSample{t: 1, fh: histograms[0].ToFloat(nil)},
 		hSample{t: 2, h: histograms[1]},
 	}), 2)
 
@@ -244,11 +244,11 @@ func TestBufferedSeriesIteratorMixedHistograms(t *testing.T) {
 
 	require.Equal(t, chunkenc.ValFloatHistogram, buf.Next())
 	_, fh := buf.AtFloatHistogram()
-	require.Equal(t, histograms[0].ToFloat(), fh)
+	require.Equal(t, histograms[0].ToFloat(nil), fh)
 
 	require.Equal(t, chunkenc.ValHistogram, buf.Next())
 	_, fh = buf.AtFloatHistogram()
-	require.Equal(t, histograms[1].ToFloat(), fh)
+	require.Equal(t, histograms[1].ToFloat(nil), fh)
 }
 
 func BenchmarkBufferedSeriesIterator(b *testing.B) {
diff --git a/storage/remote/codec_test.go b/storage/remote/codec_test.go
index 4df3c972e..d2a7d45be 100644
--- a/storage/remote/codec_test.go
+++ b/storage/remote/codec_test.go
@@ -57,7 +57,7 @@ var writeRequestFixture = &prompb.WriteRequest{
 			},
 			Samples:    []prompb.Sample{{Value: 1, Timestamp: 0}},
 			Exemplars:  []prompb.Exemplar{{Labels: []prompb.Label{{Name: "f", Value: "g"}}, Value: 1, Timestamp: 0}},
-			Histograms: []prompb.Histogram{HistogramToHistogramProto(0, &testHistogram), FloatHistogramToHistogramProto(1, testHistogram.ToFloat())},
+			Histograms: []prompb.Histogram{HistogramToHistogramProto(0, &testHistogram), FloatHistogramToHistogramProto(1, testHistogram.ToFloat(nil))},
 		},
 		{
 			Labels: []prompb.Label{
@@ -69,7 +69,7 @@ var writeRequestFixture = &prompb.WriteRequest{
 			},
 			Samples:    []prompb.Sample{{Value: 2, Timestamp: 1}},
 			Exemplars:  []prompb.Exemplar{{Labels: []prompb.Label{{Name: "h", Value: "i"}}, Value: 2, Timestamp: 1}},
-			Histograms: []prompb.Histogram{HistogramToHistogramProto(2, &testHistogram), FloatHistogramToHistogramProto(3, testHistogram.ToFloat())},
+			Histograms: []prompb.Histogram{HistogramToHistogramProto(2, &testHistogram), FloatHistogramToHistogramProto(3, testHistogram.ToFloat(nil))},
 		},
 	},
 }
diff --git a/storage/remote/queue_manager_test.go b/storage/remote/queue_manager_test.go
index 16691a174..c878c750b 100644
--- a/storage/remote/queue_manager_test.go
+++ b/storage/remote/queue_manager_test.go
@@ -619,7 +619,7 @@ func createHistograms(numSamples, numSeries int, floatHistogram bool) ([]record.
 				fh := record.RefFloatHistogramSample{
 					Ref: chunks.HeadSeriesRef(i),
 					T:   int64(j),
-					FH:  hist.ToFloat(),
+					FH:  hist.ToFloat(nil),
 				}
 				floatHistograms = append(floatHistograms, fh)
 			} else {
diff --git a/storage/remote/write_handler_test.go b/storage/remote/write_handler_test.go
index b00fe891a..839009b2a 100644
--- a/storage/remote/write_handler_test.go
+++ b/storage/remote/write_handler_test.go
@@ -134,7 +134,7 @@ func TestOutOfOrderExemplar(t *testing.T) {
 func TestOutOfOrderHistogram(t *testing.T) {
 	buf, _, err := buildWriteRequest([]prompb.TimeSeries{{
 		Labels:     []prompb.Label{{Name: "__name__", Value: "test_metric"}},
-		Histograms: []prompb.Histogram{HistogramToHistogramProto(0, &testHistogram), FloatHistogramToHistogramProto(1, testHistogram.ToFloat())},
+		Histograms: []prompb.Histogram{HistogramToHistogramProto(0, &testHistogram), FloatHistogramToHistogramProto(1, testHistogram.ToFloat(nil))},
 	}}, nil, nil, nil)
 	require.NoError(t, err)
 
diff --git a/tsdb/block_test.go b/tsdb/block_test.go
index 778fcf708..7858e6b0c 100644
--- a/tsdb/block_test.go
+++ b/tsdb/block_test.go
@@ -656,7 +656,7 @@ func genHistogramSeries(totalSeries, labelCount int, mint, maxt, step int64, flo
 			h.CounterResetHint = histogram.NotCounterReset
 		}
 		if floatHistogram {
-			return sample{t: ts, fh: h.ToFloat()}
+			return sample{t: ts, fh: h.ToFloat(nil)}
 		}
 		return sample{t: ts, h: h}
 	})
@@ -692,7 +692,7 @@ func genHistogramAndFloatSeries(totalSeries, labelCount int, mint, maxt, step in
 				h.CounterResetHint = histogram.NotCounterReset
 			}
 			if floatHistogram {
-				s = sample{t: ts, fh: h.ToFloat()}
+				s = sample{t: ts, fh: h.ToFloat(nil)}
 			} else {
 				s = sample{t: ts, h: h}
 			}
diff --git a/tsdb/chunkenc/float_histogram_test.go b/tsdb/chunkenc/float_histogram_test.go
index 05b1fa0a1..6f5a95fb1 100644
--- a/tsdb/chunkenc/float_histogram_test.go
+++ b/tsdb/chunkenc/float_histogram_test.go
@@ -94,10 +94,10 @@ func TestFloatHistogramChunkSameBuckets(t *testing.T) {
 		},
 		NegativeBuckets: []int64{2, 1, -1, -1}, // counts: 2, 3, 2, 1 (total 8)
 	}
-	chk, _, app, err := app.AppendFloatHistogram(nil, ts, h.ToFloat(), false)
+	chk, _, app, err := app.AppendFloatHistogram(nil, ts, h.ToFloat(nil), false)
 	require.NoError(t, err)
 	require.Nil(t, chk)
-	exp = append(exp, floatResult{t: ts, h: h.ToFloat()})
+	exp = append(exp, floatResult{t: ts, h: h.ToFloat(nil)})
 	require.Equal(t, 1, c.NumSamples())
 
 	// Add an updated histogram.
@@ -108,10 +108,10 @@ func TestFloatHistogramChunkSameBuckets(t *testing.T) {
 	h.Sum = 24.4
 	h.PositiveBuckets = []int64{5, -2, 1, -2} // counts: 5, 3, 4, 2 (total 14)
 	h.NegativeBuckets = []int64{4, -1, 1, -1} // counts: 4, 3, 4, 4 (total 15)
-	chk, _, _, err = app.AppendFloatHistogram(nil, ts, h.ToFloat(), false)
+	chk, _, _, err = app.AppendFloatHistogram(nil, ts, h.ToFloat(nil), false)
 	require.NoError(t, err)
 	require.Nil(t, chk)
-	expH := h.ToFloat()
+	expH := h.ToFloat(nil)
 	expH.CounterResetHint = histogram.NotCounterReset
 	exp = append(exp, floatResult{t: ts, h: expH})
 	require.Equal(t, 2, c.NumSamples())
@@ -127,10 +127,10 @@ func TestFloatHistogramChunkSameBuckets(t *testing.T) {
 	h.Sum = 24.4
 	h.PositiveBuckets = []int64{6, 1, -3, 6} // counts: 6, 7, 4, 10 (total 27)
 	h.NegativeBuckets = []int64{5, 1, -2, 3} // counts: 5, 6, 4, 7 (total 22)
-	chk, _, _, err = app.AppendFloatHistogram(nil, ts, h.ToFloat(), false)
+	chk, _, _, err = app.AppendFloatHistogram(nil, ts, h.ToFloat(nil), false)
 	require.NoError(t, err)
 	require.Nil(t, chk)
-	expH = h.ToFloat()
+	expH = h.ToFloat(nil)
 	expH.CounterResetHint = histogram.NotCounterReset
 	exp = append(exp, floatResult{t: ts, h: expH})
 	require.Equal(t, 3, c.NumSamples())
@@ -217,7 +217,7 @@ func TestFloatHistogramChunkBucketChanges(t *testing.T) {
 		NegativeBuckets: []int64{1},
 	}
 
-	chk, _, app, err := app.AppendFloatHistogram(nil, ts1, h1.ToFloat(), false)
+	chk, _, app, err := app.AppendFloatHistogram(nil, ts1, h1.ToFloat(nil), false)
 	require.NoError(t, err)
 	require.Nil(t, chk)
 	require.Equal(t, 1, c.NumSamples())
@@ -245,13 +245,13 @@ func TestFloatHistogramChunkBucketChanges(t *testing.T) {
 	h2.NegativeBuckets = []int64{2, -1} // 2 1 (total 3)
 	// This is how span changes will be handled.
 	hApp, _ := app.(*FloatHistogramAppender)
-	posInterjections, negInterjections, ok, cr := hApp.appendable(h2.ToFloat())
+	posInterjections, negInterjections, ok, cr := hApp.appendable(h2.ToFloat(nil))
 	require.NotEmpty(t, posInterjections)
 	require.NotEmpty(t, negInterjections)
 	require.True(t, ok) // Only new buckets came in.
 	require.False(t, cr)
 	c, app = hApp.recode(posInterjections, negInterjections, h2.PositiveSpans, h2.NegativeSpans)
-	chk, _, _, err = app.AppendFloatHistogram(nil, ts2, h2.ToFloat(), false)
+	chk, _, _, err = app.AppendFloatHistogram(nil, ts2, h2.ToFloat(nil), false)
 	require.NoError(t, err)
 	require.Nil(t, chk)
 	require.Equal(t, 2, c.NumSamples())
@@ -263,10 +263,10 @@ func TestFloatHistogramChunkBucketChanges(t *testing.T) {
 	h1.PositiveBuckets = []int64{6, -3, -3, 3, -3, 0, 2, 2, 1, -5, 1}
 	h1.NegativeSpans = h2.NegativeSpans
 	h1.NegativeBuckets = []int64{0, 1}
-	expH2 := h2.ToFloat()
+	expH2 := h2.ToFloat(nil)
 	expH2.CounterResetHint = histogram.NotCounterReset
 	exp := []floatResult{
-		{t: ts1, h: h1.ToFloat()},
+		{t: ts1, h: h1.ToFloat(nil)},
 		{t: ts2, h: expH2},
 	}
 	it := c.Iterator(nil)
diff --git a/tsdb/chunkenc/histogram_test.go b/tsdb/chunkenc/histogram_test.go
index 768ef2e6e..53aee89db 100644
--- a/tsdb/chunkenc/histogram_test.go
+++ b/tsdb/chunkenc/histogram_test.go
@@ -98,7 +98,7 @@ func TestHistogramChunkSameBuckets(t *testing.T) {
 	chk, _, app, err := app.AppendHistogram(nil, ts, h, false)
 	require.NoError(t, err)
 	require.Nil(t, chk)
-	exp = append(exp, result{t: ts, h: h, fh: h.ToFloat()})
+	exp = append(exp, result{t: ts, h: h, fh: h.ToFloat(nil)})
 	require.Equal(t, 1, c.NumSamples())
 
 	// Add an updated histogram.
@@ -114,7 +114,7 @@ func TestHistogramChunkSameBuckets(t *testing.T) {
 	require.Nil(t, chk)
 	hExp := h.Copy()
 	hExp.CounterResetHint = histogram.NotCounterReset
-	exp = append(exp, result{t: ts, h: hExp, fh: hExp.ToFloat()})
+	exp = append(exp, result{t: ts, h: hExp, fh: hExp.ToFloat(nil)})
 	require.Equal(t, 2, c.NumSamples())
 
 	// Add update with new appender.
@@ -133,7 +133,7 @@ func TestHistogramChunkSameBuckets(t *testing.T) {
 	require.Nil(t, chk)
 	hExp = h.Copy()
 	hExp.CounterResetHint = histogram.NotCounterReset
-	exp = append(exp, result{t: ts, h: hExp, fh: hExp.ToFloat()})
+	exp = append(exp, result{t: ts, h: hExp, fh: hExp.ToFloat(nil)})
 	require.Equal(t, 3, c.NumSamples())
 
 	// 1. Expand iterator in simple case.
@@ -278,8 +278,8 @@ func TestHistogramChunkBucketChanges(t *testing.T) {
 	hExp := h2.Copy()
 	hExp.CounterResetHint = histogram.NotCounterReset
 	exp := []result{
-		{t: ts1, h: h1, fh: h1.ToFloat()},
-		{t: ts2, h: hExp, fh: hExp.ToFloat()},
+		{t: ts1, h: h1, fh: h1.ToFloat(nil)},
+		{t: ts2, h: hExp, fh: hExp.ToFloat(nil)},
 	}
 	it := c.Iterator(nil)
 	var act []result
diff --git a/tsdb/compact_test.go b/tsdb/compact_test.go
index 3eb997712..75d564139 100644
--- a/tsdb/compact_test.go
+++ b/tsdb/compact_test.go
@@ -1402,8 +1402,8 @@ func TestHeadCompactionWithHistograms(t *testing.T) {
 				for tsMinute := from; tsMinute <= to; tsMinute++ {
 					var err error
 					if floatTest {
-						_, err = app.AppendHistogram(0, lbls, minute(tsMinute), nil, h.ToFloat())
-						efh := h.ToFloat()
+						_, err = app.AppendHistogram(0, lbls, minute(tsMinute), nil, h.ToFloat(nil))
+						efh := h.ToFloat(nil)
 						if tsMinute == from {
 							efh.CounterResetHint = histogram.UnknownCounterReset
 						} else {
diff --git a/tsdb/db_test.go b/tsdb/db_test.go
index cc219a20b..4dcdef858 100644
--- a/tsdb/db_test.go
+++ b/tsdb/db_test.go
@@ -519,7 +519,7 @@ func TestAmendHistogramDatapointCausesError(t *testing.T) {
 		},
 		PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 0},
 	}
-	fh := h.ToFloat()
+	fh := h.ToFloat(nil)
 
 	app = db.Appender(ctx)
 	_, err = app.AppendHistogram(0, labels.FromStrings("a", "c"), 0, h.Copy(), nil)
@@ -6392,8 +6392,8 @@ func testHistogramAppendAndQueryHelper(t *testing.T, floatHistogram bool) {
 		var err error
 		app := db.Appender(ctx)
 		if floatHistogram {
-			_, err = app.AppendHistogram(0, lbls, minute(tsMinute), nil, h.ToFloat())
-			efh := h.ToFloat()
+			_, err = app.AppendHistogram(0, lbls, minute(tsMinute), nil, h.ToFloat(nil))
+			efh := h.ToFloat(nil)
 			efh.CounterResetHint = expCRH
 			*exp = append(*exp, sample{t: minute(tsMinute), fh: efh})
 		} else {
@@ -6814,20 +6814,20 @@ func TestNativeHistogramFlag(t *testing.T) {
 	// Disabled by default.
 	_, err = app.AppendHistogram(0, l, 100, h, nil)
 	require.Equal(t, storage.ErrNativeHistogramsDisabled, err)
-	_, err = app.AppendHistogram(0, l, 105, nil, h.ToFloat())
+	_, err = app.AppendHistogram(0, l, 105, nil, h.ToFloat(nil))
 	require.Equal(t, storage.ErrNativeHistogramsDisabled, err)
 
 	// Enable and append.
 	db.EnableNativeHistograms()
 	_, err = app.AppendHistogram(0, l, 200, h, nil)
 	require.NoError(t, err)
-	_, err = app.AppendHistogram(0, l, 205, nil, h.ToFloat())
+	_, err = app.AppendHistogram(0, l, 205, nil, h.ToFloat(nil))
 	require.NoError(t, err)
 
 	db.DisableNativeHistograms()
 	_, err = app.AppendHistogram(0, l, 300, h, nil)
 	require.Equal(t, storage.ErrNativeHistogramsDisabled, err)
-	_, err = app.AppendHistogram(0, l, 305, nil, h.ToFloat())
+	_, err = app.AppendHistogram(0, l, 305, nil, h.ToFloat(nil))
 	require.Equal(t, storage.ErrNativeHistogramsDisabled, err)
 
 	require.NoError(t, app.Commit())
@@ -6836,7 +6836,7 @@ func TestNativeHistogramFlag(t *testing.T) {
 	require.NoError(t, err)
 	act := query(t, q, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
 	require.Equal(t, map[string][]chunks.Sample{
-		l.String(): {sample{t: 200, h: h}, sample{t: 205, fh: h.ToFloat()}},
+		l.String(): {sample{t: 200, h: h}, sample{t: 205, fh: h.ToFloat(nil)}},
 	}, act)
 }
 
diff --git a/tsdb/head_test.go b/tsdb/head_test.go
index 65ed72834..d444e1496 100644
--- a/tsdb/head_test.go
+++ b/tsdb/head_test.go
@@ -4082,8 +4082,8 @@ func testHistogramStaleSampleHelper(t *testing.T, floatHistogram bool) {
 	for _, h := range tsdbutil.GenerateTestHistograms(numHistograms) {
 		var err error
 		if floatHistogram {
-			_, err = app.AppendHistogram(0, l, 100*int64(len(expHistograms)), nil, h.ToFloat())
-			expHistograms = append(expHistograms, timedHistogram{t: 100 * int64(len(expHistograms)), fh: h.ToFloat()})
+			_, err = app.AppendHistogram(0, l, 100*int64(len(expHistograms)), nil, h.ToFloat(nil))
+			expHistograms = append(expHistograms, timedHistogram{t: 100 * int64(len(expHistograms)), fh: h.ToFloat(nil)})
 		} else {
 			_, err = app.AppendHistogram(0, l, 100*int64(len(expHistograms)), h, nil)
 			expHistograms = append(expHistograms, timedHistogram{t: 100 * int64(len(expHistograms)), h: h})
@@ -4113,8 +4113,8 @@ func testHistogramStaleSampleHelper(t *testing.T, floatHistogram bool) {
 	for _, h := range tsdbutil.GenerateTestHistograms(2 * numHistograms)[numHistograms:] {
 		var err error
 		if floatHistogram {
-			_, err = app.AppendHistogram(0, l, 100*int64(len(expHistograms)), nil, h.ToFloat())
-			expHistograms = append(expHistograms, timedHistogram{t: 100 * int64(len(expHistograms)), fh: h.ToFloat()})
+			_, err = app.AppendHistogram(0, l, 100*int64(len(expHistograms)), nil, h.ToFloat(nil))
+			expHistograms = append(expHistograms, timedHistogram{t: 100 * int64(len(expHistograms)), fh: h.ToFloat(nil)})
 		} else {
 			_, err = app.AppendHistogram(0, l, 100*int64(len(expHistograms)), h, nil)
 			expHistograms = append(expHistograms, timedHistogram{t: 100 * int64(len(expHistograms)), h: h})
@@ -4160,7 +4160,7 @@ func TestHistogramCounterResetHeader(t *testing.T) {
 				app := head.Appender(context.Background())
 				var err error
 				if floatHisto {
-					_, err = app.AppendHistogram(0, l, ts, nil, h.ToFloat())
+					_, err = app.AppendHistogram(0, l, ts, nil, h.ToFloat(nil))
 				} else {
 					_, err = app.AppendHistogram(0, l, ts, h.Copy(), nil)
 				}
diff --git a/tsdb/record/record_test.go b/tsdb/record/record_test.go
index 544fa5af2..57599ef6d 100644
--- a/tsdb/record/record_test.go
+++ b/tsdb/record/record_test.go
@@ -158,7 +158,7 @@ func TestRecord_EncodeDecode(t *testing.T) {
 		floatHistograms[i] = RefFloatHistogramSample{
 			Ref: h.Ref,
 			T:   h.T,
-			FH:  h.H.ToFloat(),
+			FH:  h.H.ToFloat(nil),
 		}
 	}
 	decFloatHistograms, err := dec.FloatHistogramSamples(enc.FloatHistogramSamples(floatHistograms, nil), nil)
diff --git a/tsdb/wlog/watcher_test.go b/tsdb/wlog/watcher_test.go
index fc665b57d..b30dce91a 100644
--- a/tsdb/wlog/watcher_test.go
+++ b/tsdb/wlog/watcher_test.go
@@ -199,7 +199,7 @@ func TestTailSamples(t *testing.T) {
 					floatHistogram := enc.FloatHistogramSamples([]record.RefFloatHistogramSample{{
 						Ref: chunks.HeadSeriesRef(inner),
 						T:   now.UnixNano() + 1,
-						FH:  hist.ToFloat(),
+						FH:  hist.ToFloat(nil),
 					}}, nil)
 					require.NoError(t, w.Log(floatHistogram))
 				}
diff --git a/web/federate.go b/web/federate.go
index 2b79d0053..2e7bac21d 100644
--- a/web/federate.go
+++ b/web/federate.go
@@ -138,7 +138,7 @@ Loop:
 			case chunkenc.ValFloat:
 				f = sample.F()
 			case chunkenc.ValHistogram:
-				fh = sample.H().ToFloat()
+				fh = sample.H().ToFloat(nil)
 			case chunkenc.ValFloatHistogram:
 				fh = sample.FH()
 			default:
diff --git a/web/federate_test.go b/web/federate_test.go
index 80539861d..94783a739 100644
--- a/web/federate_test.go
+++ b/web/federate_test.go
@@ -354,7 +354,7 @@ func TestFederationWithNativeHistograms(t *testing.T) {
 			_, err = app.AppendHistogram(0, l, 100*60*1000, histWithoutZeroBucket.Copy(), nil)
 			expVec = append(expVec, promql.Sample{
 				T:      100 * 60 * 1000,
-				H:      histWithoutZeroBucket.ToFloat(),
+				H:      histWithoutZeroBucket.ToFloat(nil),
 				Metric: expL,
 			})
 		default:
@@ -363,7 +363,7 @@ func TestFederationWithNativeHistograms(t *testing.T) {
 			_, err = app.AppendHistogram(0, l, 100*60*1000, hist.Copy(), nil)
 			expVec = append(expVec, promql.Sample{
 				T:      100 * 60 * 1000,
-				H:      hist.ToFloat(),
+				H:      hist.ToFloat(nil),
 				Metric: expL,
 			})
 		}

From 9008271df57d4392d3e424feba80b394267428ef Mon Sep 17 00:00:00 2001
From: Filip Petkovski <filip.petkovsky@gmail.com>
Date: Fri, 8 Dec 2023 10:12:50 +0100
Subject: [PATCH 011/115] Use cap to determine slice capacity

Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
---
 model/histogram/histogram.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/model/histogram/histogram.go b/model/histogram/histogram.go
index 1fe342482..f4d292b34 100644
--- a/model/histogram/histogram.go
+++ b/model/histogram/histogram.go
@@ -321,7 +321,7 @@ func (h *Histogram) ToFloat(fh *FloatHistogram) *FloatHistogram {
 }
 
 func resize[T any](items []T, n int) []T {
-	if len(items) < n {
+	if cap(items) < n {
 		return make([]T, n)
 	}
 	return items[:n]

From acbaee8c9d7d780b6bf67128e435d049d0ea685c Mon Sep 17 00:00:00 2001
From: SuperQ <superq@gmail.com>
Date: Fri, 8 Dec 2023 11:01:38 +0100
Subject: [PATCH 012/115] Sync golangci-lint version

Update `scripts/golangci-lint.yml` golangci-lint version to match
main workflow.
* Add note to keep things in sync.

Signed-off-by: SuperQ <superq@gmail.com>
---
 .github/workflows/ci.yml  | 1 +
 scripts/golangci-lint.yml | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 22d3f8ad5..8ba154e25 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -151,6 +151,7 @@ jobs:
         uses: golangci/golangci-lint-action@3a919529898de77ec3da873e3063ca4b10e7f5cc # v3.7.0
         with:
           args: --verbose
+          # Make sure to sync this with Makefile.common and scripts/golangci-lint.yml.
           version: v1.55.2
   fuzzing:
     uses: ./.github/workflows/fuzzing.yml
diff --git a/scripts/golangci-lint.yml b/scripts/golangci-lint.yml
index ffa6b3090..805c59fb7 100644
--- a/scripts/golangci-lint.yml
+++ b/scripts/golangci-lint.yml
@@ -29,4 +29,4 @@ jobs:
       - name: Lint
         uses: golangci/golangci-lint-action@3a919529898de77ec3da873e3063ca4b10e7f5cc # v3.7.0
         with:
-          version: v1.54.2
+          version: v1.55.2

From ee700151a3bc901bdf52b184dee60f477f1d6ea8 Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Tue, 3 Jan 2023 19:11:00 +0000
Subject: [PATCH 013/115] tsdb/index: add benchmark for Postings.Merge

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
---
 tsdb/index/postings_test.go | 34 +++++++++++++++++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)

diff --git a/tsdb/index/postings_test.go b/tsdb/index/postings_test.go
index c20b4506f..e8df6dbd2 100644
--- a/tsdb/index/postings_test.go
+++ b/tsdb/index/postings_test.go
@@ -380,6 +380,38 @@ func BenchmarkIntersect(t *testing.B) {
 	})
 }
 
+func BenchmarkMerge(t *testing.B) {
+	var lps []*ListPostings
+	var refs [][]storage.SeriesRef
+
+	// Create 100000 matchers(k=100000), making sure all memory allocation is done before starting the loop.
+	for i := 0; i < 100000; i++ {
+		var temp []storage.SeriesRef
+		for j := 1; j < 100; j++ {
+			temp = append(temp, storage.SeriesRef(i+j*100000))
+		}
+		lps = append(lps, newListPostings(temp...))
+		refs = append(refs, temp)
+	}
+
+	its := make([]Postings, len(refs))
+	for _, nSeries := range []int{1, 10, 100, 1000, 10000, 100000} {
+		t.Run(fmt.Sprint(nSeries), func(bench *testing.B) {
+			ctx := context.Background()
+			for i := 0; i < bench.N; i++ {
+				// Reset the ListPostings to their original values each time round the loop.
+				for j := range refs[:nSeries] {
+					lps[j].list = refs[j]
+					its[j] = lps[j]
+				}
+				if err := consumePostings(Merge(ctx, its[:nSeries]...)); err != nil {
+					bench.Fatal(err)
+				}
+			}
+		})
+	}
+}
+
 func TestMultiMerge(t *testing.T) {
 	i1 := newListPostings(1, 2, 3, 4, 5, 6, 1000, 1001)
 	i2 := newListPostings(2, 4, 5, 6, 7, 8, 999, 1001)
@@ -481,7 +513,7 @@ func TestMergedPostings(t *testing.T) {
 			m := Merge(ctx, c.in...)
 
 			if c.res == EmptyPostings() {
-				require.Equal(t, EmptyPostings(), m)
+				require.False(t, m.Next())
 				return
 			}
 

From ab3a47b48905fb2dd91ea011480521b5f3671f95 Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Mon, 4 Sep 2023 15:23:30 +0100
Subject: [PATCH 014/115] postings: use Loser Tree for merge

It's faster.

Note change to test - instead of requiring that the data structure is
identical to `EmptyPostings()`, check that calling `Next()` returns
false, which implies it was empty.

Also the check for context cancellation during initialization was
removed. Initialization should be a small portion of the work done
during merge, so it's not worth plumbing a context argument through.

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
---
 go.mod                 |   1 +
 go.sum                 |   2 +
 tsdb/index/postings.go | 125 ++++++++++-------------------------------
 3 files changed, 32 insertions(+), 96 deletions(-)

diff --git a/go.mod b/go.mod
index 95cc7b8af..f022f09e9 100644
--- a/go.mod
+++ b/go.mod
@@ -10,6 +10,7 @@ require (
 	github.com/alecthomas/kingpin/v2 v2.4.0
 	github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9
 	github.com/aws/aws-sdk-go v1.48.14
+	github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3
 	github.com/cespare/xxhash/v2 v2.2.0
 	github.com/dennwc/varint v1.0.0
 	github.com/digitalocean/godo v1.106.0
diff --git a/go.sum b/go.sum
index d29694fb9..ae367bb90 100644
--- a/go.sum
+++ b/go.sum
@@ -94,6 +94,8 @@ github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2z
 github.com/aws/aws-sdk-go v1.48.14 h1:nVLrp+F84SG+xGiFMfe1TE6ZV6smF+42tuuNgYGV30s=
 github.com/aws/aws-sdk-go v1.48.14/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
+github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 h1:6df1vn4bBlDDo4tARvBm7l6KA9iVMnE3NWizDeWSrps=
+github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3/go.mod h1:CIWtjkly68+yqLPbvwwR/fjNJA/idrtULjZWh2v1ys0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
diff --git a/tsdb/index/postings.go b/tsdb/index/postings.go
index c83957427..222a8b0d6 100644
--- a/tsdb/index/postings.go
+++ b/tsdb/index/postings.go
@@ -18,11 +18,13 @@ import (
 	"context"
 	"encoding/binary"
 	"fmt"
+	"math"
 	"runtime"
 	"sort"
 	"strings"
 	"sync"
 
+	"github.com/bboreham/go-loser"
 	"golang.org/x/exp/slices"
 
 	"github.com/prometheus/prometheus/model/labels"
@@ -525,7 +527,7 @@ func (it *intersectPostings) Err() error {
 }
 
 // Merge returns a new iterator over the union of the input iterators.
-func Merge(ctx context.Context, its ...Postings) Postings {
+func Merge(_ context.Context, its ...Postings) Postings {
 	if len(its) == 0 {
 		return EmptyPostings()
 	}
@@ -533,122 +535,48 @@ func Merge(ctx context.Context, its ...Postings) Postings {
 		return its[0]
 	}
 
-	p, ok := newMergedPostings(ctx, its)
+	p, ok := newMergedPostings(its)
 	if !ok {
 		return EmptyPostings()
 	}
 	return p
 }
 
-type postingsHeap []Postings
-
-func (h postingsHeap) Len() int           { return len(h) }
-func (h postingsHeap) Less(i, j int) bool { return h[i].At() < h[j].At() }
-func (h *postingsHeap) Swap(i, j int)     { (*h)[i], (*h)[j] = (*h)[j], (*h)[i] }
-
-func (h *postingsHeap) Push(x interface{}) {
-	*h = append(*h, x.(Postings))
-}
-
-func (h *postingsHeap) Pop() interface{} {
-	old := *h
-	n := len(old)
-	x := old[n-1]
-	*h = old[0 : n-1]
-	return x
-}
-
 type mergedPostings struct {
-	h           postingsHeap
-	initialized bool
-	cur         storage.SeriesRef
-	err         error
+	p   []Postings
+	h   *loser.Tree[storage.SeriesRef, Postings]
+	cur storage.SeriesRef
 }
 
-func newMergedPostings(ctx context.Context, p []Postings) (m *mergedPostings, nonEmpty bool) {
-	ph := make(postingsHeap, 0, len(p))
-
-	for _, it := range p {
-		// NOTE: mergedPostings struct requires the user to issue an initial Next.
-		switch {
-		case ctx.Err() != nil:
-			return &mergedPostings{err: ctx.Err()}, true
-		case it.Next():
-			ph = append(ph, it)
-		case it.Err() != nil:
-			return &mergedPostings{err: it.Err()}, true
-		}
-	}
-
-	if len(ph) == 0 {
-		return nil, false
-	}
-	return &mergedPostings{h: ph}, true
+func newMergedPostings(p []Postings) (m *mergedPostings, nonEmpty bool) {
+	const maxVal = storage.SeriesRef(math.MaxUint64) // This value must be higher than all real values used in the tree.
+	lt := loser.New(p, maxVal)
+	return &mergedPostings{p: p, h: lt}, true
 }
 
 func (it *mergedPostings) Next() bool {
-	if it.h.Len() == 0 || it.err != nil {
-		return false
-	}
-
-	// The user must issue an initial Next.
-	if !it.initialized {
-		heap.Init(&it.h)
-		it.cur = it.h[0].At()
-		it.initialized = true
-		return true
-	}
-
 	for {
-		cur := it.h[0]
-		if !cur.Next() {
-			heap.Pop(&it.h)
-			if cur.Err() != nil {
-				it.err = cur.Err()
-				return false
-			}
-			if it.h.Len() == 0 {
-				return false
-			}
-		} else {
-			// Value of top of heap has changed, re-heapify.
-			heap.Fix(&it.h, 0)
+		if !it.h.Next() {
+			return false
 		}
-
-		if it.h[0].At() != it.cur {
-			it.cur = it.h[0].At()
+		// Remove duplicate entries.
+		newItem := it.h.At()
+		if newItem != it.cur {
+			it.cur = newItem
 			return true
 		}
 	}
 }
 
 func (it *mergedPostings) Seek(id storage.SeriesRef) bool {
-	if it.h.Len() == 0 || it.err != nil {
+	for !it.h.IsEmpty() && it.h.At() < id {
+		finished := !it.h.Winner().Seek(id)
+		it.h.Fix(finished)
+	}
+	if it.h.IsEmpty() {
 		return false
 	}
-	if !it.initialized {
-		if !it.Next() {
-			return false
-		}
-	}
-	for it.cur < id {
-		cur := it.h[0]
-		if !cur.Seek(id) {
-			heap.Pop(&it.h)
-			if cur.Err() != nil {
-				it.err = cur.Err()
-				return false
-			}
-			if it.h.Len() == 0 {
-				return false
-			}
-		} else {
-			// Value of top of heap has changed, re-heapify.
-			heap.Fix(&it.h, 0)
-		}
-
-		it.cur = it.h[0].At()
-	}
+	it.cur = it.h.At()
 	return true
 }
 
@@ -657,7 +585,12 @@ func (it mergedPostings) At() storage.SeriesRef {
 }
 
 func (it mergedPostings) Err() error {
-	return it.err
+	for _, p := range it.p {
+		if err := p.Err(); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 // Without returns a new postings list that contains all elements from the full list that

From e2a9f8ac0f2d2b0c1565241ca46b4d3b6dc22007 Mon Sep 17 00:00:00 2001
From: Filip Petkovski <filip.petkovsky@gmail.com>
Date: Sun, 10 Dec 2023 21:07:19 +0100
Subject: [PATCH 015/115] Reuse float histogram objects

This commit reduces the memory needed to query native histogram objects
by reusing existing HPoint instances.

Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
---
 promql/engine.go       | 33 ++++++++++++++++++++++-----------
 storage/buffer.go      |  8 ++++----
 storage/buffer_test.go |  4 ++--
 3 files changed, 28 insertions(+), 17 deletions(-)

diff --git a/promql/engine.go b/promql/engine.go
index 4fa2a513b..8b4987921 100644
--- a/promql/engine.go
+++ b/promql/engine.go
@@ -2078,21 +2078,27 @@ loop:
 		case chunkenc.ValNone:
 			break loop
 		case chunkenc.ValFloatHistogram, chunkenc.ValHistogram:
-			t, h := buf.AtFloatHistogram()
-			if value.IsStaleNaN(h.Sum) {
-				continue loop
-			}
+			t := buf.AtT()
 			// Values in the buffer are guaranteed to be smaller than maxt.
 			if t >= mintHistograms {
-				if ev.currentSamples >= ev.maxSamples {
-					ev.error(ErrTooManySamples(env))
-				}
-				point := HPoint{T: t, H: h}
 				if histograms == nil {
 					histograms = getHPointSlice(16)
 				}
-				histograms = append(histograms, point)
-				ev.currentSamples += point.size()
+				n := len(histograms)
+				if n < cap(histograms) {
+					histograms = histograms[:n+1]
+				} else {
+					histograms = append(histograms, HPoint{})
+				}
+				histograms[n].T, histograms[n].H = buf.AtFloatHistogram(histograms[n].H)
+				if value.IsStaleNaN(histograms[n].H.Sum) {
+					histograms = histograms[:n]
+					continue loop
+				}
+				if ev.currentSamples >= ev.maxSamples {
+					ev.error(ErrTooManySamples(env))
+				}
+				ev.currentSamples += histograms[n].size()
 			}
 		case chunkenc.ValFloat:
 			t, f := buf.At()
@@ -2108,7 +2114,12 @@ loop:
 				if floats == nil {
 					floats = getFPointSlice(16)
 				}
-				floats = append(floats, FPoint{T: t, F: f})
+				if n := len(floats); n < cap(floats) {
+					floats = floats[:n+1]
+					floats[n].T, floats[n].F = t, f
+				} else {
+					floats = append(floats, FPoint{T: t, F: f})
+				}
 			}
 		}
 	}
diff --git a/storage/buffer.go b/storage/buffer.go
index f686796ca..49dfbd299 100644
--- a/storage/buffer.go
+++ b/storage/buffer.go
@@ -74,7 +74,7 @@ func (b *BufferedSeriesIterator) PeekBack(n int) (sample chunks.Sample, ok bool)
 
 // Buffer returns an iterator over the buffered data. Invalidates previously
 // returned iterators.
-func (b *BufferedSeriesIterator) Buffer() chunkenc.Iterator {
+func (b *BufferedSeriesIterator) Buffer() *sampleRingIterator {
 	return b.buf.iterator()
 }
 
@@ -304,7 +304,7 @@ func (r *sampleRing) reset() {
 }
 
 // Returns the current iterator. Invalidates previously returned iterators.
-func (r *sampleRing) iterator() chunkenc.Iterator {
+func (r *sampleRing) iterator() *sampleRingIterator {
 	r.it.r = r
 	r.it.i = -1
 	return &r.it
@@ -374,9 +374,9 @@ func (it *sampleRingIterator) AtHistogram() (int64, *histogram.Histogram) {
 	return it.t, it.h
 }
 
-func (it *sampleRingIterator) AtFloatHistogram() (int64, *histogram.FloatHistogram) {
+func (it *sampleRingIterator) AtFloatHistogram(fh *histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
 	if it.fh == nil {
-		return it.t, it.h.ToFloat(nil)
+		return it.t, it.h.ToFloat(fh)
 	}
 	return it.t, it.fh
 }
diff --git a/storage/buffer_test.go b/storage/buffer_test.go
index 84f94a008..12e6ff0f0 100644
--- a/storage/buffer_test.go
+++ b/storage/buffer_test.go
@@ -243,11 +243,11 @@ func TestBufferedSeriesIteratorMixedHistograms(t *testing.T) {
 	buf := it.Buffer()
 
 	require.Equal(t, chunkenc.ValFloatHistogram, buf.Next())
-	_, fh := buf.AtFloatHistogram()
+	_, fh := buf.AtFloatHistogram(nil)
 	require.Equal(t, histograms[0].ToFloat(nil), fh)
 
 	require.Equal(t, chunkenc.ValHistogram, buf.Next())
-	_, fh = buf.AtFloatHistogram()
+	_, fh = buf.AtFloatHistogram(nil)
 	require.Equal(t, histograms[1].ToFloat(nil), fh)
 }
 

From 5082655392fe2eb036f24da51b54ca050496368a Mon Sep 17 00:00:00 2001
From: Arthur Silva Sens <arthur.sens@coralogix.com>
Date: Mon, 11 Dec 2023 05:43:42 -0300
Subject: [PATCH 016/115] Append Created Timestamps (#12733)

* Append created timestamps.

Signed-off-by: Arthur Silva Sens <arthur.sens@coralogix.com>

* Log when created timestamps are ignored

Signed-off-by: Arthur Silva Sens <arthur.sens@coralogix.com>

* Proposed changes to Append CT PR.

Changes:

* Changed textparse Parser interface for consistency and robustness.
* Changed CT interface to be more explicit and handle validation.
* Simplified test, change scrapeManager to allow testability.
* Added TODOs.

Signed-off-by: bwplotka <bwplotka@gmail.com>

* Updates.

Signed-off-by: bwplotka <bwplotka@gmail.com>

* Addressed comments.

Signed-off-by: bwplotka <bwplotka@gmail.com>

* Refactor head_appender test

Signed-off-by: Arthur Silva Sens <arthur.sens@coralogix.com>

* Fix linter issues

Signed-off-by: Arthur Silva Sens <arthur.sens@coralogix.com>

* Use model.Sample in head appender test

Signed-off-by: Arthur Silva Sens <arthur.sens@coralogix.com>

---------

Signed-off-by: Arthur Silva Sens <arthur.sens@coralogix.com>
Signed-off-by: bwplotka <bwplotka@gmail.com>
Co-authored-by: bwplotka <bwplotka@gmail.com>
---
 cmd/prometheus/main.go                |  14 ++-
 config/config.go                      |   9 +-
 model/textparse/interface.go          |  10 +-
 model/textparse/openmetricsparse.go   |   9 +-
 model/textparse/promparse.go          |   9 +-
 model/textparse/protobufparse.go      |  22 ++--
 model/textparse/protobufparse_test.go |  34 +++---
 scrape/helpers_test.go                |  57 ++++++++++
 scrape/manager.go                     |   6 +
 scrape/manager_test.go                | 153 ++++++++++++++++++++++++++
 scrape/scrape.go                      |  39 +++++--
 scrape/scrape_test.go                 |   8 +-
 storage/fanout.go                     |  14 +++
 storage/interface.go                  |  26 +++++
 storage/remote/write.go               |   5 +
 storage/remote/write_handler_test.go  |   5 +
 tsdb/agent/db.go                      |   5 +
 tsdb/head.go                          |   4 +
 tsdb/head_append.go                   |  95 +++++++++++++---
 tsdb/head_test.go                     |  91 +++++++++++++++
 util/runutil/runutil.go               |  37 +++++++
 21 files changed, 578 insertions(+), 74 deletions(-)
 create mode 100644 util/runutil/runutil.go

diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go
index dfafe66c6..106f9d05c 100644
--- a/cmd/prometheus/main.go
+++ b/cmd/prometheus/main.go
@@ -206,9 +206,15 @@ func (c *flagConfig) setFeatureListOptions(logger log.Logger) error {
 			case "native-histograms":
 				c.tsdb.EnableNativeHistograms = true
 				// Change relevant global variables. Hacky, but it's hard to pass a new option or default to unmarshallers.
-				config.DefaultConfig.GlobalConfig.ScrapeProtocols = config.DefaultNativeHistogramScrapeProtocols
-				config.DefaultGlobalConfig.ScrapeProtocols = config.DefaultNativeHistogramScrapeProtocols
+				config.DefaultConfig.GlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
+				config.DefaultGlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
 				level.Info(logger).Log("msg", "Experimental native histogram support enabled. Changed default scrape_protocols to prefer PrometheusProto format.", "global.scrape_protocols", fmt.Sprintf("%v", config.DefaultGlobalConfig.ScrapeProtocols))
+			case "created-timestamp-zero-ingestion":
+				c.scrape.EnableCreatedTimestampZeroIngestion = true
+				// Change relevant global variables. Hacky, but it's hard to pass a new option or default to unmarshallers.
+				config.DefaultConfig.GlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
+				config.DefaultGlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
+				level.Info(logger).Log("msg", "Experimental created timestamp zero ingestion enabled. Changed default scrape_protocols to prefer PrometheusProto format.", "global.scrape_protocols", fmt.Sprintf("%v", config.DefaultGlobalConfig.ScrapeProtocols))
 			case "":
 				continue
 			case "promql-at-modifier", "promql-negative-offset":
@@ -1449,6 +1455,10 @@ func (n notReadyAppender) UpdateMetadata(ref storage.SeriesRef, l labels.Labels,
 	return 0, tsdb.ErrNotReady
 }
 
+func (n notReadyAppender) AppendCTZeroSample(ref storage.SeriesRef, l labels.Labels, t, ct int64) (storage.SeriesRef, error) {
+	return 0, tsdb.ErrNotReady
+}
+
 func (n notReadyAppender) Commit() error { return tsdb.ErrNotReady }
 
 func (n notReadyAppender) Rollback() error { return tsdb.ErrNotReady }
diff --git a/config/config.go b/config/config.go
index b832ac9a1..ddcca84dc 100644
--- a/config/config.go
+++ b/config/config.go
@@ -454,12 +454,19 @@ var (
 		OpenMetricsText1_0_0: "application/openmetrics-text;version=1.0.0",
 	}
 
+	// DefaultScrapeProtocols is the set of scrape protocols that will be proposed
+	// to scrape target, ordered by priority.
 	DefaultScrapeProtocols = []ScrapeProtocol{
 		OpenMetricsText1_0_0,
 		OpenMetricsText0_0_1,
 		PrometheusText0_0_4,
 	}
-	DefaultNativeHistogramScrapeProtocols = []ScrapeProtocol{
+
+	// DefaultProtoFirstScrapeProtocols is like DefaultScrapeProtocols, but it
+	// favors protobuf Prometheus exposition format.
+	// Used by default for certain feature-flags like
+	// "native-histograms" and "created-timestamp-zero-ingestion".
+	DefaultProtoFirstScrapeProtocols = []ScrapeProtocol{
 		PrometheusProto,
 		OpenMetricsText1_0_0,
 		OpenMetricsText0_0_1,
diff --git a/model/textparse/interface.go b/model/textparse/interface.go
index 2f5fdbc3b..df4259c85 100644
--- a/model/textparse/interface.go
+++ b/model/textparse/interface.go
@@ -16,8 +16,6 @@ package textparse
 import (
 	"mime"
 
-	"github.com/gogo/protobuf/types"
-
 	"github.com/prometheus/prometheus/model/exemplar"
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
@@ -66,10 +64,10 @@ type Parser interface {
 	// retrieved (including the case where no exemplars exist at all).
 	Exemplar(l *exemplar.Exemplar) bool
 
-	// CreatedTimestamp writes the created timestamp of the current sample
-	// into the passed timestamp. It returns false if no created timestamp
-	// exists or if the metric type does not support created timestamps.
-	CreatedTimestamp(ct *types.Timestamp) bool
+	// CreatedTimestamp returns the created timestamp (in milliseconds) for the
+	// current sample. It returns nil if it is unknown e.g. if it wasn't set,
+	// if the scrape protocol or metric type does not support created timestamps.
+	CreatedTimestamp() *int64
 
 	// Next advances the parser to the next sample. It returns false if no
 	// more samples were read or an error occurred.
diff --git a/model/textparse/openmetricsparse.go b/model/textparse/openmetricsparse.go
index bb5075544..f0c383723 100644
--- a/model/textparse/openmetricsparse.go
+++ b/model/textparse/openmetricsparse.go
@@ -24,8 +24,6 @@ import (
 	"strings"
 	"unicode/utf8"
 
-	"github.com/gogo/protobuf/types"
-
 	"github.com/prometheus/prometheus/model/exemplar"
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
@@ -213,9 +211,10 @@ func (p *OpenMetricsParser) Exemplar(e *exemplar.Exemplar) bool {
 	return true
 }
 
-// CreatedTimestamp returns false because OpenMetricsParser does not support created timestamps (yet).
-func (p *OpenMetricsParser) CreatedTimestamp(_ *types.Timestamp) bool {
-	return false
+// CreatedTimestamp returns nil as it's not implemented yet.
+// TODO(bwplotka): https://github.com/prometheus/prometheus/issues/12980
+func (p *OpenMetricsParser) CreatedTimestamp() *int64 {
+	return nil
 }
 
 // nextToken returns the next token from the openMetricsLexer.
diff --git a/model/textparse/promparse.go b/model/textparse/promparse.go
index b3fa2d8a6..935801fb9 100644
--- a/model/textparse/promparse.go
+++ b/model/textparse/promparse.go
@@ -26,8 +26,6 @@ import (
 	"unicode/utf8"
 	"unsafe"
 
-	"github.com/gogo/protobuf/types"
-
 	"github.com/prometheus/prometheus/model/exemplar"
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
@@ -247,9 +245,10 @@ func (p *PromParser) Exemplar(*exemplar.Exemplar) bool {
 	return false
 }
 
-// CreatedTimestamp returns false because PromParser does not support created timestamps.
-func (p *PromParser) CreatedTimestamp(_ *types.Timestamp) bool {
-	return false
+// CreatedTimestamp returns nil as it's not implemented yet.
+// TODO(bwplotka): https://github.com/prometheus/prometheus/issues/12980
+func (p *PromParser) CreatedTimestamp() *int64 {
+	return nil
 }
 
 // nextToken returns the next token from the promlexer. It skips over tabs
diff --git a/model/textparse/protobufparse.go b/model/textparse/protobufparse.go
index 23afb5c59..baede7e1d 100644
--- a/model/textparse/protobufparse.go
+++ b/model/textparse/protobufparse.go
@@ -360,22 +360,26 @@ func (p *ProtobufParser) Exemplar(ex *exemplar.Exemplar) bool {
 	return true
 }
 
-func (p *ProtobufParser) CreatedTimestamp(ct *types.Timestamp) bool {
-	var foundCT *types.Timestamp
+// CreatedTimestamp returns CT or nil if CT is not present or
+// invalid (as timestamp e.g. negative value) on counters, summaries or histograms.
+func (p *ProtobufParser) CreatedTimestamp() *int64 {
+	var ct *types.Timestamp
 	switch p.mf.GetType() {
 	case dto.MetricType_COUNTER:
-		foundCT = p.mf.GetMetric()[p.metricPos].GetCounter().GetCreatedTimestamp()
+		ct = p.mf.GetMetric()[p.metricPos].GetCounter().GetCreatedTimestamp()
 	case dto.MetricType_SUMMARY:
-		foundCT = p.mf.GetMetric()[p.metricPos].GetSummary().GetCreatedTimestamp()
+		ct = p.mf.GetMetric()[p.metricPos].GetSummary().GetCreatedTimestamp()
 	case dto.MetricType_HISTOGRAM, dto.MetricType_GAUGE_HISTOGRAM:
-		foundCT = p.mf.GetMetric()[p.metricPos].GetHistogram().GetCreatedTimestamp()
+		ct = p.mf.GetMetric()[p.metricPos].GetHistogram().GetCreatedTimestamp()
 	default:
 	}
-	if foundCT == nil {
-		return false
+	ctAsTime, err := types.TimestampFromProto(ct)
+	if err != nil {
+		// Errors means ct == nil or invalid timestamp, which we silently ignore.
+		return nil
 	}
-	*ct = *foundCT
-	return true
+	ctMilis := ctAsTime.UnixMilli()
+	return &ctMilis
 }
 
 // Next advances the parser to the next "sample" (emulating the behavior of a
diff --git a/model/textparse/protobufparse_test.go b/model/textparse/protobufparse_test.go
index e062e64dd..c5b672dbc 100644
--- a/model/textparse/protobufparse_test.go
+++ b/model/textparse/protobufparse_test.go
@@ -21,7 +21,6 @@ import (
 	"testing"
 
 	"github.com/gogo/protobuf/proto"
-	"github.com/gogo/protobuf/types"
 	"github.com/stretchr/testify/require"
 
 	"github.com/prometheus/prometheus/model/exemplar"
@@ -630,7 +629,7 @@ func TestProtobufParse(t *testing.T) {
 		shs     *histogram.Histogram
 		fhs     *histogram.FloatHistogram
 		e       []exemplar.Exemplar
-		ct      *types.Timestamp
+		ct      int64
 	}
 
 	inputBuf := createTestProtoBuf(t)
@@ -1069,7 +1068,7 @@ func TestProtobufParse(t *testing.T) {
 				{
 					m:  "test_counter_with_createdtimestamp",
 					v:  42,
-					ct: &types.Timestamp{Seconds: 1, Nanos: 1},
+					ct: 1000,
 					lset: labels.FromStrings(
 						"__name__", "test_counter_with_createdtimestamp",
 					),
@@ -1085,7 +1084,7 @@ func TestProtobufParse(t *testing.T) {
 				{
 					m:  "test_summary_with_createdtimestamp_count",
 					v:  42,
-					ct: &types.Timestamp{Seconds: 1, Nanos: 1},
+					ct: 1000,
 					lset: labels.FromStrings(
 						"__name__", "test_summary_with_createdtimestamp_count",
 					),
@@ -1093,7 +1092,7 @@ func TestProtobufParse(t *testing.T) {
 				{
 					m:  "test_summary_with_createdtimestamp_sum",
 					v:  1.234,
-					ct: &types.Timestamp{Seconds: 1, Nanos: 1},
+					ct: 1000,
 					lset: labels.FromStrings(
 						"__name__", "test_summary_with_createdtimestamp_sum",
 					),
@@ -1108,7 +1107,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{
 					m:  "test_histogram_with_createdtimestamp",
-					ct: &types.Timestamp{Seconds: 1, Nanos: 1},
+					ct: 1000,
 					shs: &histogram.Histogram{
 						CounterResetHint: histogram.UnknownCounterReset,
 						PositiveSpans:    []histogram.Span{},
@@ -1128,7 +1127,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{
 					m:  "test_gaugehistogram_with_createdtimestamp",
-					ct: &types.Timestamp{Seconds: 1, Nanos: 1},
+					ct: 1000,
 					shs: &histogram.Histogram{
 						CounterResetHint: histogram.GaugeType,
 						PositiveSpans:    []histogram.Span{},
@@ -1887,7 +1886,7 @@ func TestProtobufParse(t *testing.T) {
 				{ // 83
 					m:  "test_counter_with_createdtimestamp",
 					v:  42,
-					ct: &types.Timestamp{Seconds: 1, Nanos: 1},
+					ct: 1000,
 					lset: labels.FromStrings(
 						"__name__", "test_counter_with_createdtimestamp",
 					),
@@ -1903,7 +1902,7 @@ func TestProtobufParse(t *testing.T) {
 				{ // 86
 					m:  "test_summary_with_createdtimestamp_count",
 					v:  42,
-					ct: &types.Timestamp{Seconds: 1, Nanos: 1},
+					ct: 1000,
 					lset: labels.FromStrings(
 						"__name__", "test_summary_with_createdtimestamp_count",
 					),
@@ -1911,7 +1910,7 @@ func TestProtobufParse(t *testing.T) {
 				{ // 87
 					m:  "test_summary_with_createdtimestamp_sum",
 					v:  1.234,
-					ct: &types.Timestamp{Seconds: 1, Nanos: 1},
+					ct: 1000,
 					lset: labels.FromStrings(
 						"__name__", "test_summary_with_createdtimestamp_sum",
 					),
@@ -1926,7 +1925,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{ // 90
 					m:  "test_histogram_with_createdtimestamp",
-					ct: &types.Timestamp{Seconds: 1, Nanos: 1},
+					ct: 1000,
 					shs: &histogram.Histogram{
 						CounterResetHint: histogram.UnknownCounterReset,
 						PositiveSpans:    []histogram.Span{},
@@ -1946,7 +1945,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{ // 93
 					m:  "test_gaugehistogram_with_createdtimestamp",
-					ct: &types.Timestamp{Seconds: 1, Nanos: 1},
+					ct: 1000,
 					shs: &histogram.Histogram{
 						CounterResetHint: histogram.GaugeType,
 						PositiveSpans:    []histogram.Span{},
@@ -1981,10 +1980,9 @@ func TestProtobufParse(t *testing.T) {
 					m, ts, v := p.Series()
 
 					var e exemplar.Exemplar
-					var ct types.Timestamp
 					p.Metric(&res)
 					eFound := p.Exemplar(&e)
-					ctFound := p.CreatedTimestamp(&ct)
+					ct := p.CreatedTimestamp()
 					require.Equal(t, exp[i].m, string(m), "i: %d", i)
 					if ts != nil {
 						require.Equal(t, exp[i].t, *ts, "i: %d", i)
@@ -2000,11 +1998,11 @@ func TestProtobufParse(t *testing.T) {
 						require.Equal(t, exp[i].e[0], e, "i: %d", i)
 						require.False(t, p.Exemplar(&e), "too many exemplars returned, i: %d", i)
 					}
-					if exp[i].ct != nil {
-						require.True(t, ctFound, "i: %d", i)
-						require.Equal(t, exp[i].ct.String(), ct.String(), "i: %d", i)
+					if exp[i].ct != 0 {
+						require.NotNilf(t, ct, "i: %d", i)
+						require.Equal(t, exp[i].ct, *ct, "i: %d", i)
 					} else {
-						require.False(t, ctFound, "i: %d", i)
+						require.Nilf(t, ct, "i: %d", i)
 					}
 
 				case EntryHistogram:
diff --git a/scrape/helpers_test.go b/scrape/helpers_test.go
index c580a5051..43ee0fcec 100644
--- a/scrape/helpers_test.go
+++ b/scrape/helpers_test.go
@@ -14,10 +14,18 @@
 package scrape
 
 import (
+	"bytes"
 	"context"
+	"encoding/binary"
 	"fmt"
 	"math/rand"
 	"strings"
+	"sync"
+	"testing"
+
+	"github.com/gogo/protobuf/proto"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/stretchr/testify/require"
 
 	"github.com/prometheus/prometheus/model/exemplar"
 	"github.com/prometheus/prometheus/model/histogram"
@@ -50,6 +58,10 @@ func (a nopAppender) UpdateMetadata(storage.SeriesRef, labels.Labels, metadata.M
 	return 0, nil
 }
 
+func (a nopAppender) AppendCTZeroSample(storage.SeriesRef, labels.Labels, int64, int64) (storage.SeriesRef, error) {
+	return 0, nil
+}
+
 func (a nopAppender) Commit() error   { return nil }
 func (a nopAppender) Rollback() error { return nil }
 
@@ -65,9 +77,19 @@ type histogramSample struct {
 	fh *histogram.FloatHistogram
 }
 
+type collectResultAppendable struct {
+	*collectResultAppender
+}
+
+func (a *collectResultAppendable) Appender(_ context.Context) storage.Appender {
+	return a
+}
+
 // collectResultAppender records all samples that were added through the appender.
 // It can be used as its zero value or be backed by another appender it writes samples through.
 type collectResultAppender struct {
+	mtx sync.Mutex
+
 	next                 storage.Appender
 	resultFloats         []floatSample
 	pendingFloats        []floatSample
@@ -82,6 +104,8 @@ type collectResultAppender struct {
 }
 
 func (a *collectResultAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64) (storage.SeriesRef, error) {
+	a.mtx.Lock()
+	defer a.mtx.Unlock()
 	a.pendingFloats = append(a.pendingFloats, floatSample{
 		metric: lset,
 		t:      t,
@@ -103,6 +127,8 @@ func (a *collectResultAppender) Append(ref storage.SeriesRef, lset labels.Labels
 }
 
 func (a *collectResultAppender) AppendExemplar(ref storage.SeriesRef, l labels.Labels, e exemplar.Exemplar) (storage.SeriesRef, error) {
+	a.mtx.Lock()
+	defer a.mtx.Unlock()
 	a.pendingExemplars = append(a.pendingExemplars, e)
 	if a.next == nil {
 		return 0, nil
@@ -112,6 +138,8 @@ func (a *collectResultAppender) AppendExemplar(ref storage.SeriesRef, l labels.L
 }
 
 func (a *collectResultAppender) AppendHistogram(ref storage.SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) {
+	a.mtx.Lock()
+	defer a.mtx.Unlock()
 	a.pendingHistograms = append(a.pendingHistograms, histogramSample{h: h, fh: fh, t: t})
 	if a.next == nil {
 		return 0, nil
@@ -121,6 +149,8 @@ func (a *collectResultAppender) AppendHistogram(ref storage.SeriesRef, l labels.
 }
 
 func (a *collectResultAppender) UpdateMetadata(ref storage.SeriesRef, l labels.Labels, m metadata.Metadata) (storage.SeriesRef, error) {
+	a.mtx.Lock()
+	defer a.mtx.Unlock()
 	a.pendingMetadata = append(a.pendingMetadata, m)
 	if ref == 0 {
 		ref = storage.SeriesRef(rand.Uint64())
@@ -132,7 +162,13 @@ func (a *collectResultAppender) UpdateMetadata(ref storage.SeriesRef, l labels.L
 	return a.next.UpdateMetadata(ref, l, m)
 }
 
+func (a *collectResultAppender) AppendCTZeroSample(ref storage.SeriesRef, l labels.Labels, t, ct int64) (storage.SeriesRef, error) {
+	return a.Append(ref, l, ct, 0.0)
+}
+
 func (a *collectResultAppender) Commit() error {
+	a.mtx.Lock()
+	defer a.mtx.Unlock()
 	a.resultFloats = append(a.resultFloats, a.pendingFloats...)
 	a.resultExemplars = append(a.resultExemplars, a.pendingExemplars...)
 	a.resultHistograms = append(a.resultHistograms, a.pendingHistograms...)
@@ -148,6 +184,8 @@ func (a *collectResultAppender) Commit() error {
 }
 
 func (a *collectResultAppender) Rollback() error {
+	a.mtx.Lock()
+	defer a.mtx.Unlock()
 	a.rolledbackFloats = a.pendingFloats
 	a.rolledbackHistograms = a.pendingHistograms
 	a.pendingFloats = nil
@@ -171,3 +209,22 @@ func (a *collectResultAppender) String() string {
 	}
 	return sb.String()
 }
+
+// protoMarshalDelimited marshals a MetricFamily into a delimited
+// Prometheus proto exposition format bytes (known as 'encoding=delimited`)
+//
+// See also https://eli.thegreenplace.net/2011/08/02/length-prefix-framing-for-protocol-buffers
+func protoMarshalDelimited(t *testing.T, mf *dto.MetricFamily) []byte {
+	t.Helper()
+
+	protoBuf, err := proto.Marshal(mf)
+	require.NoError(t, err)
+
+	varintBuf := make([]byte, binary.MaxVarintLen32)
+	varintLength := binary.PutUvarint(varintBuf, uint64(len(protoBuf)))
+
+	buf := &bytes.Buffer{}
+	buf.Write(varintBuf[:varintLength])
+	buf.Write(protoBuf)
+	return buf.Bytes()
+}
diff --git a/scrape/manager.go b/scrape/manager.go
index 3b70e48a1..faa46f54d 100644
--- a/scrape/manager.go
+++ b/scrape/manager.go
@@ -78,9 +78,15 @@ type Options struct {
 	EnableMetadataStorage bool
 	// Option to increase the interval used by scrape manager to throttle target groups updates.
 	DiscoveryReloadInterval model.Duration
+	// Option to enable the ingestion of the created timestamp as a synthetic zero sample.
+	// See: https://github.com/prometheus/proposals/blob/main/proposals/2023-06-13_created-timestamp.md
+	EnableCreatedTimestampZeroIngestion bool
 
 	// Optional HTTP client options to use when scraping.
 	HTTPClientOptions []config_util.HTTPClientOption
+
+	// private option for testability.
+	skipOffsetting bool
 }
 
 // Manager maintains a set of scrape pools and manages start/stop cycles
diff --git a/scrape/manager_test.go b/scrape/manager_test.go
index a689c469d..524424269 100644
--- a/scrape/manager_test.go
+++ b/scrape/manager_test.go
@@ -15,14 +15,23 @@ package scrape
 
 import (
 	"context"
+	"fmt"
 	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
 	"strconv"
+	"sync"
 	"testing"
 	"time"
 
+	"github.com/go-kit/log"
+	"github.com/gogo/protobuf/proto"
 	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/types/known/timestamppb"
 	"gopkg.in/yaml.v2"
 
 	"github.com/prometheus/prometheus/config"
@@ -30,6 +39,7 @@ import (
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/relabel"
+	"github.com/prometheus/prometheus/util/runutil"
 )
 
 func TestPopulateLabels(t *testing.T) {
@@ -714,3 +724,146 @@ scrape_configs:
 	reload(scrapeManager, cfg2)
 	require.ElementsMatch(t, []string{"job1", "job3"}, scrapeManager.ScrapePools())
 }
+
+// TestManagerCTZeroIngestion tests scrape manager for CT cases.
+func TestManagerCTZeroIngestion(t *testing.T) {
+	const mName = "expected_counter"
+
+	for _, tc := range []struct {
+		name                  string
+		counterSample         *dto.Counter
+		enableCTZeroIngestion bool
+
+		expectedValues []float64
+	}{
+		{
+			name: "disabled with CT on counter",
+			counterSample: &dto.Counter{
+				Value: proto.Float64(1.0),
+				// Timestamp does not matter as long as it exists in this test.
+				CreatedTimestamp: timestamppb.Now(),
+			},
+			expectedValues: []float64{1.0},
+		},
+		{
+			name: "enabled with CT on counter",
+			counterSample: &dto.Counter{
+				Value: proto.Float64(1.0),
+				// Timestamp does not matter as long as it exists in this test.
+				CreatedTimestamp: timestamppb.Now(),
+			},
+			enableCTZeroIngestion: true,
+			expectedValues:        []float64{0.0, 1.0},
+		},
+		{
+			name: "enabled without CT on counter",
+			counterSample: &dto.Counter{
+				Value: proto.Float64(1.0),
+			},
+			enableCTZeroIngestion: true,
+			expectedValues:        []float64{1.0},
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			app := &collectResultAppender{}
+			scrapeManager, err := NewManager(
+				&Options{
+					EnableCreatedTimestampZeroIngestion: tc.enableCTZeroIngestion,
+					skipOffsetting:                      true,
+				},
+				log.NewLogfmtLogger(os.Stderr),
+				&collectResultAppendable{app},
+				prometheus.NewRegistry(),
+			)
+			require.NoError(t, err)
+
+			require.NoError(t, scrapeManager.ApplyConfig(&config.Config{
+				GlobalConfig: config.GlobalConfig{
+					// Disable regular scrapes.
+					ScrapeInterval: model.Duration(9999 * time.Minute),
+					ScrapeTimeout:  model.Duration(5 * time.Second),
+					// Ensure the proto is chosen. We need proto as it's the only protocol
+					// with the CT parsing support.
+					ScrapeProtocols: []config.ScrapeProtocol{config.PrometheusProto},
+				},
+				ScrapeConfigs: []*config.ScrapeConfig{{JobName: "test"}},
+			}))
+
+			once := sync.Once{}
+			// Start fake HTTP target to that allow one scrape only.
+			server := httptest.NewServer(
+				http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+					fail := true
+					once.Do(func() {
+						fail = false
+						w.Header().Set("Content-Type", `application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; encoding=delimited`)
+
+						ctrType := dto.MetricType_COUNTER
+						w.Write(protoMarshalDelimited(t, &dto.MetricFamily{
+							Name:   proto.String(mName),
+							Type:   &ctrType,
+							Metric: []*dto.Metric{{Counter: tc.counterSample}},
+						}))
+					})
+
+					if fail {
+						w.WriteHeader(http.StatusInternalServerError)
+					}
+				}),
+			)
+			defer server.Close()
+
+			serverURL, err := url.Parse(server.URL)
+			require.NoError(t, err)
+
+			// Add fake target directly into tsets + reload. Normally users would use
+			// Manager.Run and wait for minimum 5s refresh interval.
+			scrapeManager.updateTsets(map[string][]*targetgroup.Group{
+				"test": {{
+					Targets: []model.LabelSet{{
+						model.SchemeLabel:  model.LabelValue(serverURL.Scheme),
+						model.AddressLabel: model.LabelValue(serverURL.Host),
+					}},
+				}},
+			})
+			scrapeManager.reload()
+
+			// Wait for one scrape.
+			ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
+			defer cancel()
+			require.NoError(t, runutil.Retry(100*time.Millisecond, ctx.Done(), func() error {
+				if countFloatSamples(app, mName) != len(tc.expectedValues) {
+					return fmt.Errorf("expected %v samples", tc.expectedValues)
+				}
+				return nil
+			}), "after 1 minute")
+			scrapeManager.Stop()
+
+			require.Equal(t, tc.expectedValues, getResultFloats(app, mName))
+		})
+	}
+}
+
+func countFloatSamples(a *collectResultAppender, expectedMetricName string) (count int) {
+	a.mtx.Lock()
+	defer a.mtx.Unlock()
+
+	for _, f := range a.resultFloats {
+		if f.metric.Get(model.MetricNameLabel) == expectedMetricName {
+			count++
+		}
+	}
+	return count
+}
+
+func getResultFloats(app *collectResultAppender, expectedMetricName string) (result []float64) {
+	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
+	for _, f := range app.resultFloats {
+		if f.metric.Get(model.MetricNameLabel) == expectedMetricName {
+			result = append(result, f.f)
+		}
+	}
+	return result
+}
diff --git a/scrape/scrape.go b/scrape/scrape.go
index 9a0ba1d00..be27a5d48 100644
--- a/scrape/scrape.go
+++ b/scrape/scrape.go
@@ -106,9 +106,10 @@ type scrapeLoopOptions struct {
 	interval                 time.Duration
 	timeout                  time.Duration
 	scrapeClassicHistograms  bool
-	mrc                      []*relabel.Config
-	cache                    *scrapeCache
-	enableCompression        bool
+
+	mrc               []*relabel.Config
+	cache             *scrapeCache
+	enableCompression bool
 }
 
 const maxAheadTime = 10 * time.Minute
@@ -168,11 +169,13 @@ func newScrapePool(cfg *config.ScrapeConfig, app storage.Appendable, offsetSeed
 			opts.interval,
 			opts.timeout,
 			opts.scrapeClassicHistograms,
+			options.EnableCreatedTimestampZeroIngestion,
 			options.ExtraMetrics,
 			options.EnableMetadataStorage,
 			opts.target,
 			options.PassMetadataInContext,
 			metrics,
+			options.skipOffsetting,
 		)
 	}
 	sp.metrics.targetScrapePoolTargetLimit.WithLabelValues(sp.config.JobName).Set(float64(sp.config.TargetLimit))
@@ -787,6 +790,7 @@ type scrapeLoop struct {
 	interval                 time.Duration
 	timeout                  time.Duration
 	scrapeClassicHistograms  bool
+	enableCTZeroIngestion    bool
 
 	appender            func(ctx context.Context) storage.Appender
 	sampleMutator       labelsMutator
@@ -804,6 +808,8 @@ type scrapeLoop struct {
 	appendMetadataToWAL bool
 
 	metrics *scrapeMetrics
+
+	skipOffsetting bool // For testability.
 }
 
 // scrapeCache tracks mappings of exposed metric strings to label sets and
@@ -1076,11 +1082,13 @@ func newScrapeLoop(ctx context.Context,
 	interval time.Duration,
 	timeout time.Duration,
 	scrapeClassicHistograms bool,
+	enableCTZeroIngestion bool,
 	reportExtraMetrics bool,
 	appendMetadataToWAL bool,
 	target *Target,
 	passMetadataInContext bool,
 	metrics *scrapeMetrics,
+	skipOffsetting bool,
 ) *scrapeLoop {
 	if l == nil {
 		l = log.NewNopLogger()
@@ -1124,9 +1132,11 @@ func newScrapeLoop(ctx context.Context,
 		interval:                 interval,
 		timeout:                  timeout,
 		scrapeClassicHistograms:  scrapeClassicHistograms,
+		enableCTZeroIngestion:    enableCTZeroIngestion,
 		reportExtraMetrics:       reportExtraMetrics,
 		appendMetadataToWAL:      appendMetadataToWAL,
 		metrics:                  metrics,
+		skipOffsetting:           skipOffsetting,
 	}
 	sl.ctx, sl.cancel = context.WithCancel(ctx)
 
@@ -1134,12 +1144,14 @@ func newScrapeLoop(ctx context.Context,
 }
 
 func (sl *scrapeLoop) run(errc chan<- error) {
-	select {
-	case <-time.After(sl.scraper.offset(sl.interval, sl.offsetSeed)):
-		// Continue after a scraping offset.
-	case <-sl.ctx.Done():
-		close(sl.stopped)
-		return
+	if !sl.skipOffsetting {
+		select {
+		case <-time.After(sl.scraper.offset(sl.interval, sl.offsetSeed)):
+			// Continue after a scraping offset.
+		case <-sl.ctx.Done():
+			close(sl.stopped)
+			return
+		}
 	}
 
 	var last time.Time
@@ -1557,6 +1569,15 @@ loop:
 			updateMetadata(lset, true)
 		}
 
+		if ctMs := p.CreatedTimestamp(); sl.enableCTZeroIngestion && ctMs != nil {
+			ref, err = app.AppendCTZeroSample(ref, lset, t, *ctMs)
+			if err != nil && !errors.Is(err, storage.ErrOutOfOrderCT) { // OOO is a common case, ignoring completely for now.
+				// CT is an experimental feature. For now, we don't need to fail the
+				// scrape on errors updating the created timestamp, log debug.
+				level.Debug(sl.l).Log("msg", "Error when appending CT in scrape loop", "series", string(met), "ct", *ctMs, "t", t, "err", err)
+			}
+		}
+
 		if isHistogram {
 			if h != nil {
 				ref, err = app.AppendHistogram(ref, lset, t, h, nil)
diff --git a/scrape/scrape_test.go b/scrape/scrape_test.go
index 8dee1f2c7..90578f2e9 100644
--- a/scrape/scrape_test.go
+++ b/scrape/scrape_test.go
@@ -660,9 +660,11 @@ func newBasicScrapeLoop(t testing.TB, ctx context.Context, scraper scraper, app
 		false,
 		false,
 		false,
+		false,
 		nil,
 		false,
 		newTestScrapeMetrics(t),
+		false,
 	)
 }
 
@@ -801,9 +803,11 @@ func TestScrapeLoopRun(t *testing.T) {
 		false,
 		false,
 		false,
+		false,
 		nil,
 		false,
 		scrapeMetrics,
+		false,
 	)
 
 	// The loop must terminate during the initial offset if the context
@@ -945,9 +949,11 @@ func TestScrapeLoopMetadata(t *testing.T) {
 		false,
 		false,
 		false,
+		false,
 		nil,
 		false,
 		scrapeMetrics,
+		false,
 	)
 	defer cancel()
 
@@ -2377,7 +2383,7 @@ func TestTargetScraperScrapeOK(t *testing.T) {
 
 	runTest(acceptHeader(config.DefaultScrapeProtocols))
 	protobufParsing = true
-	runTest(acceptHeader(config.DefaultNativeHistogramScrapeProtocols))
+	runTest(acceptHeader(config.DefaultProtoFirstScrapeProtocols))
 }
 
 func TestTargetScrapeScrapeCancel(t *testing.T) {
diff --git a/storage/fanout.go b/storage/fanout.go
index 33257046f..a9a3f904b 100644
--- a/storage/fanout.go
+++ b/storage/fanout.go
@@ -202,6 +202,20 @@ func (f *fanoutAppender) UpdateMetadata(ref SeriesRef, l labels.Labels, m metada
 	return ref, nil
 }
 
+func (f *fanoutAppender) AppendCTZeroSample(ref SeriesRef, l labels.Labels, t, ct int64) (SeriesRef, error) {
+	ref, err := f.primary.AppendCTZeroSample(ref, l, t, ct)
+	if err != nil {
+		return ref, err
+	}
+
+	for _, appender := range f.secondaries {
+		if _, err := appender.AppendCTZeroSample(ref, l, t, ct); err != nil {
+			return 0, err
+		}
+	}
+	return ref, nil
+}
+
 func (f *fanoutAppender) Commit() (err error) {
 	err = f.primary.Commit()
 
diff --git a/storage/interface.go b/storage/interface.go
index 2b1b6a63e..675e44c0e 100644
--- a/storage/interface.go
+++ b/storage/interface.go
@@ -43,6 +43,13 @@ var (
 	ErrExemplarLabelLength         = fmt.Errorf("label length for exemplar exceeds maximum of %d UTF-8 characters", exemplar.ExemplarMaxLabelSetLength)
 	ErrExemplarsDisabled           = fmt.Errorf("exemplar storage is disabled or max exemplars is less than or equal to 0")
 	ErrNativeHistogramsDisabled    = fmt.Errorf("native histograms are disabled")
+
+	// ErrOutOfOrderCT indicates failed append of CT to the storage
+	// due to CT being older the then newer sample.
+	// NOTE(bwplotka): This can be both an instrumentation failure or commonly expected
+	// behaviour, and we currently don't have a way to determine this. As a result
+	// it's recommended to ignore this error for now.
+	ErrOutOfOrderCT = fmt.Errorf("created timestamp out of order, ignoring")
 )
 
 // SeriesRef is a generic series reference. In prometheus it is either a
@@ -237,6 +244,7 @@ type Appender interface {
 	ExemplarAppender
 	HistogramAppender
 	MetadataUpdater
+	CreatedTimestampAppender
 }
 
 // GetRef is an extra interface on Appenders used by downstream projects
@@ -294,6 +302,24 @@ type MetadataUpdater interface {
 	UpdateMetadata(ref SeriesRef, l labels.Labels, m metadata.Metadata) (SeriesRef, error)
 }
 
+// CreatedTimestampAppender provides an interface for appending CT to storage.
+type CreatedTimestampAppender interface {
+	// AppendCTZeroSample adds synthetic zero sample for the given ct timestamp,
+	// which will be associated with given series, labels and the incoming
+	// sample's t (timestamp). AppendCTZeroSample returns error if zero sample can't be
+	// appended, for example when ct is too old, or when it would collide with
+	// incoming sample (sample has priority).
+	//
+	// AppendCTZeroSample has to be called before the corresponding sample Append.
+	// A series reference number is returned which can be used to modify the
+	// CT for the given series in the same or later transactions.
+	// Returned reference numbers are ephemeral and may be rejected in calls
+	// to AppendCTZeroSample() at any point.
+	//
+	// If the reference is 0 it must not be used for caching.
+	AppendCTZeroSample(ref SeriesRef, l labels.Labels, t, ct int64) (SeriesRef, error)
+}
+
 // SeriesSet contains a set of series.
 type SeriesSet interface {
 	Next() bool
diff --git a/storage/remote/write.go b/storage/remote/write.go
index 237f8caa9..66455cb4d 100644
--- a/storage/remote/write.go
+++ b/storage/remote/write.go
@@ -303,6 +303,11 @@ func (t *timestampTracker) UpdateMetadata(_ storage.SeriesRef, _ labels.Labels,
 	return 0, nil
 }
 
+func (t *timestampTracker) AppendCTZeroSample(_ storage.SeriesRef, _ labels.Labels, _, _ int64) (storage.SeriesRef, error) {
+	// AppendCTZeroSample is no-op for remote-write for now.
+	return 0, nil
+}
+
 // Commit implements storage.Appender.
 func (t *timestampTracker) Commit() error {
 	t.writeStorage.samplesIn.incr(t.samples + t.exemplars + t.histograms)
diff --git a/storage/remote/write_handler_test.go b/storage/remote/write_handler_test.go
index 839009b2a..fd5b34ecd 100644
--- a/storage/remote/write_handler_test.go
+++ b/storage/remote/write_handler_test.go
@@ -339,3 +339,8 @@ func (m *mockAppendable) UpdateMetadata(_ storage.SeriesRef, _ labels.Labels, _
 	// UpdateMetadata is no-op for remote write (where mockAppendable is being used to test) for now.
 	return 0, nil
 }
+
+func (m *mockAppendable) AppendCTZeroSample(_ storage.SeriesRef, _ labels.Labels, _, _ int64) (storage.SeriesRef, error) {
+	// AppendCTZeroSample is no-op for remote-write for now.
+	return 0, nil
+}
diff --git a/tsdb/agent/db.go b/tsdb/agent/db.go
index 6afef1389..557fb7854 100644
--- a/tsdb/agent/db.go
+++ b/tsdb/agent/db.go
@@ -962,6 +962,11 @@ func (a *appender) UpdateMetadata(storage.SeriesRef, labels.Labels, metadata.Met
 	return 0, nil
 }
 
+func (a *appender) AppendCTZeroSample(storage.SeriesRef, labels.Labels, int64, int64) (storage.SeriesRef, error) {
+	// TODO(bwplotka): Wire metadata in the Agent's appender.
+	return 0, nil
+}
+
 // Commit submits the collected samples and purges the batch.
 func (a *appender) Commit() error {
 	if err := a.log(); err != nil {
diff --git a/tsdb/head.go b/tsdb/head.go
index 3ff2bee71..848357359 100644
--- a/tsdb/head.go
+++ b/tsdb/head.go
@@ -149,6 +149,10 @@ type HeadOptions struct {
 	// EnableNativeHistograms enables the ingestion of native histograms.
 	EnableNativeHistograms atomic.Bool
 
+	// EnableCreatedTimestampZeroIngestion enables the ingestion of the created timestamp as a synthetic zero sample.
+	// See: https://github.com/prometheus/proposals/blob/main/proposals/2023-06-13_created-timestamp.md
+	EnableCreatedTimestampZeroIngestion bool
+
 	ChunkRange int64
 	// ChunkDirRoot is the parent directory of the chunks directory.
 	ChunkDirRoot         string
diff --git a/tsdb/head_append.go b/tsdb/head_append.go
index be53a4f3f..afb461afe 100644
--- a/tsdb/head_append.go
+++ b/tsdb/head_append.go
@@ -87,6 +87,17 @@ func (a *initAppender) UpdateMetadata(ref storage.SeriesRef, l labels.Labels, m
 	return a.app.UpdateMetadata(ref, l, m)
 }
 
+func (a *initAppender) AppendCTZeroSample(ref storage.SeriesRef, lset labels.Labels, t, ct int64) (storage.SeriesRef, error) {
+	if a.app != nil {
+		return a.app.AppendCTZeroSample(ref, lset, t, ct)
+	}
+
+	a.head.initTime(t)
+	a.app = a.head.appender()
+
+	return a.app.AppendCTZeroSample(ref, lset, t, ct)
+}
+
 // initTime initializes a head with the first timestamp. This only needs to be called
 // for a completely fresh head with an empty WAL.
 func (h *Head) initTime(t int64) {
@@ -319,28 +330,11 @@ func (a *headAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64
 
 	s := a.head.series.getByID(chunks.HeadSeriesRef(ref))
 	if s == nil {
-		// Ensure no empty labels have gotten through.
-		lset = lset.WithoutEmpty()
-		if lset.IsEmpty() {
-			return 0, errors.Wrap(ErrInvalidSample, "empty labelset")
-		}
-
-		if l, dup := lset.HasDuplicateLabelNames(); dup {
-			return 0, errors.Wrap(ErrInvalidSample, fmt.Sprintf(`label name "%s" is not unique`, l))
-		}
-
-		var created bool
 		var err error
-		s, created, err = a.head.getOrCreate(lset.Hash(), lset)
+		s, err = a.getOrCreate(lset)
 		if err != nil {
 			return 0, err
 		}
-		if created {
-			a.series = append(a.series, record.RefSeries{
-				Ref:    s.ref,
-				Labels: lset,
-			})
-		}
 	}
 
 	if value.IsStaleNaN(v) {
@@ -389,6 +383,71 @@ func (a *headAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64
 	return storage.SeriesRef(s.ref), nil
 }
 
+// AppendCTZeroSample appends synthetic zero sample for ct timestamp. It returns
+// error when sample can't be appended. See
+// storage.CreatedTimestampAppender.AppendCTZeroSample for further documentation.
+func (a *headAppender) AppendCTZeroSample(ref storage.SeriesRef, lset labels.Labels, t, ct int64) (storage.SeriesRef, error) {
+	if ct >= t {
+		return 0, fmt.Errorf("CT is newer or the same as sample's timestamp, ignoring")
+	}
+
+	s := a.head.series.getByID(chunks.HeadSeriesRef(ref))
+	if s == nil {
+		var err error
+		s, err = a.getOrCreate(lset)
+		if err != nil {
+			return 0, err
+		}
+	}
+
+	// Check if CT wouldn't be OOO vs samples we already might have for this series.
+	// NOTE(bwplotka): This will be often hit as it's expected for long living
+	// counters to share the same CT.
+	s.Lock()
+	isOOO, _, err := s.appendable(ct, 0, a.headMaxt, a.minValidTime, a.oooTimeWindow)
+	if err == nil {
+		s.pendingCommit = true
+	}
+	s.Unlock()
+	if err != nil {
+		return 0, err
+	}
+	if isOOO {
+		return storage.SeriesRef(s.ref), storage.ErrOutOfOrderCT
+	}
+
+	if ct > a.maxt {
+		a.maxt = ct
+	}
+	a.samples = append(a.samples, record.RefSample{Ref: s.ref, T: ct, V: 0.0})
+	a.sampleSeries = append(a.sampleSeries, s)
+	return storage.SeriesRef(s.ref), nil
+}
+
+func (a *headAppender) getOrCreate(lset labels.Labels) (*memSeries, error) {
+	// Ensure no empty labels have gotten through.
+	lset = lset.WithoutEmpty()
+	if lset.IsEmpty() {
+		return nil, errors.Wrap(ErrInvalidSample, "empty labelset")
+	}
+	if l, dup := lset.HasDuplicateLabelNames(); dup {
+		return nil, errors.Wrap(ErrInvalidSample, fmt.Sprintf(`label name "%s" is not unique`, l))
+	}
+	var created bool
+	var err error
+	s, created, err := a.head.getOrCreate(lset.Hash(), lset)
+	if err != nil {
+		return nil, err
+	}
+	if created {
+		a.series = append(a.series, record.RefSeries{
+			Ref:    s.ref,
+			Labels: lset,
+		})
+	}
+	return s, nil
+}
+
 // appendable checks whether the given sample is valid for appending to the series. (if we return false and no error)
 // The sample belongs to the out of order chunk if we return true and no error.
 // An error signifies the sample cannot be handled.
diff --git a/tsdb/head_test.go b/tsdb/head_test.go
index d444e1496..535647d3a 100644
--- a/tsdb/head_test.go
+++ b/tsdb/head_test.go
@@ -33,6 +33,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	prom_testutil "github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/atomic"
 	"golang.org/x/sync/errgroup"
@@ -5641,3 +5642,93 @@ func TestPostingsCardinalityStats(t *testing.T) {
 	// Using cache.
 	require.Equal(t, statsForSomeLabel1, head.PostingsCardinalityStats("n", 1))
 }
+
+func TestHeadAppender_AppendCTZeroSample(t *testing.T) {
+	type appendableSamples struct {
+		ts  int64
+		val float64
+		ct  int64
+	}
+	for _, tc := range []struct {
+		name              string
+		appendableSamples []appendableSamples
+		expectedSamples   []model.Sample
+	}{
+		{
+			name: "In order ct+normal sample",
+			appendableSamples: []appendableSamples{
+				{ts: 100, val: 10, ct: 1},
+			},
+			expectedSamples: []model.Sample{
+				{Timestamp: 1, Value: 0},
+				{Timestamp: 100, Value: 10},
+			},
+		},
+		{
+			name: "Consecutive appends with same ct ignore ct",
+			appendableSamples: []appendableSamples{
+				{ts: 100, val: 10, ct: 1},
+				{ts: 101, val: 10, ct: 1},
+			},
+			expectedSamples: []model.Sample{
+				{Timestamp: 1, Value: 0},
+				{Timestamp: 100, Value: 10},
+				{Timestamp: 101, Value: 10},
+			},
+		},
+		{
+			name: "Consecutive appends with newer ct do not ignore ct",
+			appendableSamples: []appendableSamples{
+				{ts: 100, val: 10, ct: 1},
+				{ts: 102, val: 10, ct: 101},
+			},
+			expectedSamples: []model.Sample{
+				{Timestamp: 1, Value: 0},
+				{Timestamp: 100, Value: 10},
+				{Timestamp: 101, Value: 0},
+				{Timestamp: 102, Value: 10},
+			},
+		},
+		{
+			name: "CT equals to previous sample timestamp is ignored",
+			appendableSamples: []appendableSamples{
+				{ts: 100, val: 10, ct: 1},
+				{ts: 101, val: 10, ct: 100},
+			},
+			expectedSamples: []model.Sample{
+				{Timestamp: 1, Value: 0},
+				{Timestamp: 100, Value: 10},
+				{Timestamp: 101, Value: 10},
+			},
+		},
+	} {
+		h, _ := newTestHead(t, DefaultBlockDuration, wlog.CompressionNone, false)
+		defer func() {
+			require.NoError(t, h.Close())
+		}()
+		a := h.Appender(context.Background())
+		lbls := labels.FromStrings("foo", "bar")
+		for _, sample := range tc.appendableSamples {
+			_, err := a.AppendCTZeroSample(0, lbls, sample.ts, sample.ct)
+			require.NoError(t, err)
+			_, err = a.Append(0, lbls, sample.ts, sample.val)
+			require.NoError(t, err)
+		}
+		require.NoError(t, a.Commit())
+
+		q, err := NewBlockQuerier(h, math.MinInt64, math.MaxInt64)
+		require.NoError(t, err)
+		ss := q.Select(context.Background(), false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
+		require.True(t, ss.Next())
+		s := ss.At()
+		require.False(t, ss.Next())
+		it := s.Iterator(nil)
+		for _, sample := range tc.expectedSamples {
+			require.Equal(t, chunkenc.ValFloat, it.Next())
+			timestamp, value := it.At()
+			require.Equal(t, sample.Timestamp, model.Time(timestamp))
+			require.Equal(t, sample.Value, model.SampleValue(value))
+		}
+		require.Equal(t, chunkenc.ValNone, it.Next())
+	}
+}
diff --git a/util/runutil/runutil.go b/util/runutil/runutil.go
new file mode 100644
index 000000000..5a77c332b
--- /dev/null
+++ b/util/runutil/runutil.go
@@ -0,0 +1,37 @@
+// Copyright 2013 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Copied from https://github.com/efficientgo/core/blob/a21078e2c723b69e05f95c65dbc5058712b4edd8/runutil/runutil.go#L39
+// and adjusted.
+
+package runutil
+
+import "time"
+
+// Retry executes f every interval seconds until timeout or no error is returned from f.
+func Retry(interval time.Duration, stopc <-chan struct{}, f func() error) error {
+	tick := time.NewTicker(interval)
+	defer tick.Stop()
+
+	var err error
+	for {
+		if err = f(); err == nil {
+			return nil
+		}
+		select {
+		case <-stopc:
+			return err
+		case <-tick.C:
+		}
+	}
+}

From 48df9fc0203c569241d1d0c38d12a4beab65883b Mon Sep 17 00:00:00 2001
From: Filip Petkovski <filip.petkovsky@gmail.com>
Date: Mon, 11 Dec 2023 11:12:11 +0100
Subject: [PATCH 017/115] Export SampleRingIterator

Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
---
 storage/buffer.go | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/storage/buffer.go b/storage/buffer.go
index 49dfbd299..75a935f9d 100644
--- a/storage/buffer.go
+++ b/storage/buffer.go
@@ -74,7 +74,7 @@ func (b *BufferedSeriesIterator) PeekBack(n int) (sample chunks.Sample, ok bool)
 
 // Buffer returns an iterator over the buffered data. Invalidates previously
 // returned iterators.
-func (b *BufferedSeriesIterator) Buffer() *sampleRingIterator {
+func (b *BufferedSeriesIterator) Buffer() *SampleRingIterator {
 	return b.buf.iterator()
 }
 
@@ -252,7 +252,7 @@ type sampleRing struct {
 	f int // Position of first element in ring buffer.
 	l int // Number of elements in buffer.
 
-	it sampleRingIterator
+	it SampleRingIterator
 }
 
 type bufType int
@@ -304,13 +304,13 @@ func (r *sampleRing) reset() {
 }
 
 // Returns the current iterator. Invalidates previously returned iterators.
-func (r *sampleRing) iterator() *sampleRingIterator {
+func (r *sampleRing) iterator() *SampleRingIterator {
 	r.it.r = r
 	r.it.i = -1
 	return &r.it
 }
 
-type sampleRingIterator struct {
+type SampleRingIterator struct {
 	r  *sampleRing
 	i  int
 	t  int64
@@ -319,7 +319,7 @@ type sampleRingIterator struct {
 	fh *histogram.FloatHistogram
 }
 
-func (it *sampleRingIterator) Next() chunkenc.ValueType {
+func (it *SampleRingIterator) Next() chunkenc.ValueType {
 	it.i++
 	if it.i >= it.r.l {
 		return chunkenc.ValNone
@@ -358,30 +358,30 @@ func (it *sampleRingIterator) Next() chunkenc.ValueType {
 	}
 }
 
-func (it *sampleRingIterator) Seek(int64) chunkenc.ValueType {
+func (it *SampleRingIterator) Seek(int64) chunkenc.ValueType {
 	return chunkenc.ValNone
 }
 
-func (it *sampleRingIterator) Err() error {
+func (it *SampleRingIterator) Err() error {
 	return nil
 }
 
-func (it *sampleRingIterator) At() (int64, float64) {
+func (it *SampleRingIterator) At() (int64, float64) {
 	return it.t, it.f
 }
 
-func (it *sampleRingIterator) AtHistogram() (int64, *histogram.Histogram) {
+func (it *SampleRingIterator) AtHistogram() (int64, *histogram.Histogram) {
 	return it.t, it.h
 }
 
-func (it *sampleRingIterator) AtFloatHistogram(fh *histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
+func (it *SampleRingIterator) AtFloatHistogram(fh *histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
 	if it.fh == nil {
 		return it.t, it.h.ToFloat(fh)
 	}
 	return it.t, it.fh
 }
 
-func (it *sampleRingIterator) AtT() int64 {
+func (it *SampleRingIterator) AtT() int64 {
 	return it.t
 }
 

From 6de80d7fb03f7cd5ed44b85559886e73d89e720d Mon Sep 17 00:00:00 2001
From: Paulin Todev <paulin.todev@gmail.com>
Date: Mon, 23 Oct 2023 14:55:36 +0100
Subject: [PATCH 018/115] Allow non-default registry to be used for metrics of
 SD components

Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
---
 cmd/prometheus/main.go                        |  28 +++-
 cmd/promtool/main.go                          |   3 +-
 cmd/promtool/sd.go                            |   5 +-
 discovery/README.md                           |   5 +
 discovery/aws/ec2.go                          |  16 +-
 discovery/aws/lightsail.go                    |  16 +-
 discovery/azure/azure.go                      |  63 ++++----
 discovery/consul/consul.go                    | 141 ++++++++++--------
 discovery/consul/consul_test.go               |  11 +-
 discovery/digitalocean/digitalocean.go        |  16 +-
 discovery/digitalocean/digitalocean_test.go   |   3 +-
 discovery/discovery.go                        |  10 ++
 discovery/dns/dns.go                          |  72 ++++-----
 discovery/dns/dns_test.go                     |   4 +-
 discovery/eureka/eureka.go                    |  16 +-
 discovery/eureka/eureka_test.go               |   3 +-
 discovery/file/file.go                        |  75 ++++++----
 discovery/file/file_test.go                   |   9 +-
 discovery/gce/gce.go                          |  16 +-
 discovery/hetzner/hetzner.go                  |  16 +-
 discovery/http/http.go                        |  41 ++---
 discovery/http/http_test.go                   |  23 ++-
 discovery/ionos/ionos.go                      |  17 ++-
 discovery/kubernetes/endpoints.go             |  21 +--
 discovery/kubernetes/endpointslice.go         |  21 +--
 discovery/kubernetes/ingress.go               |  15 +-
 discovery/kubernetes/kubernetes.go            |  69 +++++----
 discovery/kubernetes/kubernetes_test.go       |  18 ++-
 discovery/kubernetes/node.go                  |  15 +-
 discovery/kubernetes/pod.go                   |  15 +-
 discovery/kubernetes/service.go               |  15 +-
 discovery/legacymanager/manager.go            |  68 +++------
 discovery/legacymanager/manager_test.go       |  27 ++--
 discovery/linode/linode.go                    |  47 +++---
 discovery/linode/linode_test.go               |   3 +-
 discovery/manager.go                          |  66 ++------
 discovery/manager_test.go                     |  39 +++--
 discovery/marathon/marathon.go                |  16 +-
 discovery/marathon/marathon_test.go           |   5 +-
 discovery/metrics.go                          | 103 +++++++++++++
 ...lient_metrics.go => metrics_k8s_client.go} |  48 +++---
 discovery/moby/docker.go                      |  16 +-
 discovery/moby/docker_test.go                 |   3 +-
 discovery/moby/dockerswarm.go                 |  16 +-
 discovery/moby/nodes_test.go                  |   3 +-
 discovery/moby/services_test.go               |   5 +-
 discovery/moby/tasks_test.go                  |   3 +-
 discovery/nomad/nomad.go                      |  53 +++----
 discovery/nomad/nomad_test.go                 |   5 +-
 discovery/openstack/openstack.go              |  16 +-
 discovery/ovhcloud/ovhcloud.go                |  16 +-
 discovery/ovhcloud/ovhcloud_test.go           |   4 +-
 discovery/puppetdb/puppetdb.go                |  16 +-
 discovery/puppetdb/puppetdb_test.go           |  11 +-
 discovery/refresh/refresh.go                  |  85 +++++++----
 discovery/refresh/refresh_test.go             |  11 +-
 discovery/scaleway/scaleway.go                |  16 +-
 discovery/triton/triton.go                    |  16 +-
 discovery/triton/triton_test.go               |   3 +-
 discovery/util.go                             |  72 +++++++++
 discovery/uyuni/uyuni.go                      |  16 +-
 discovery/uyuni/uyuni_test.go                 |   6 +-
 discovery/vultr/vultr.go                      |  16 +-
 discovery/vultr/vultr_test.go                 |   3 +-
 discovery/xds/kuma.go                         |  81 +++++-----
 discovery/xds/kuma_test.go                    |   3 +-
 discovery/xds/xds.go                          |  13 +-
 .../examples/custom-sd/adapter-usage/main.go  |  10 +-
 .../examples/custom-sd/adapter/adapter.go     |   5 +-
 .../custom-sd/adapter/adapter_test.go         |   6 +-
 70 files changed, 1056 insertions(+), 693 deletions(-)
 create mode 100644 discovery/metrics.go
 rename discovery/{kubernetes/client_metrics.go => metrics_k8s_client.go} (85%)
 create mode 100644 discovery/util.go

diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go
index 106f9d05c..24fc0f8e4 100644
--- a/cmd/prometheus/main.go
+++ b/cmd/prometheus/main.go
@@ -620,14 +620,30 @@ func main() {
 		discoveryManagerNotify  discoveryManager
 	)
 
+	// Register the metrics used by both "scrape" and "notify" discovery managers.
+	// The same metrics are used for both discovery managers. Hence the registration
+	// needs to be done here, outside the NewManager() calls, to avoid duplicate
+	// metric registrations.
+	discoveryMetrics, err := discovery.NewMetrics(prometheus.DefaultRegisterer)
+	if err != nil {
+		level.Error(logger).Log("msg", "failed to create discovery metrics", "err", err)
+		os.Exit(1)
+	}
 	if cfg.enableNewSDManager {
-		discovery.RegisterMetrics()
-		discoveryManagerScrape = discovery.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), discovery.Name("scrape"))
-		discoveryManagerNotify = discovery.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), discovery.Name("notify"))
+		discoveryManagerScrape = discovery.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, discoveryMetrics, discovery.Name("scrape"))
+		discoveryManagerNotify = discovery.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, discoveryMetrics, discovery.Name("notify"))
 	} else {
-		legacymanager.RegisterMetrics()
-		discoveryManagerScrape = legacymanager.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), legacymanager.Name("scrape"))
-		discoveryManagerNotify = legacymanager.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), legacymanager.Name("notify"))
+		discoveryManagerScrape = legacymanager.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, discoveryMetrics, legacymanager.Name("scrape"))
+		discoveryManagerNotify = legacymanager.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, discoveryMetrics, legacymanager.Name("notify"))
+	}
+
+	if discoveryManagerScrape == nil {
+		level.Error(logger).Log("msg", "failed to create a discovery manager scrape")
+		os.Exit(1)
+	}
+	if discoveryManagerNotify == nil {
+		level.Error(logger).Log("msg", "failed to create a discovery manager notify")
+		os.Exit(1)
 	}
 
 	scrapeManager, err := scrape.NewManager(
diff --git a/cmd/promtool/main.go b/cmd/promtool/main.go
index f0b2719c9..508b681b8 100644
--- a/cmd/promtool/main.go
+++ b/cmd/promtool/main.go
@@ -36,6 +36,7 @@ import (
 	"github.com/google/pprof/profile"
 	"github.com/prometheus/client_golang/api"
 	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/prometheus/client_golang/prometheus/testutil/promlint"
 	config_util "github.com/prometheus/common/config"
@@ -317,7 +318,7 @@ func main() {
 
 	switch parsedCmd {
 	case sdCheckCmd.FullCommand():
-		os.Exit(CheckSD(*sdConfigFile, *sdJobName, *sdTimeout, noDefaultScrapePort))
+		os.Exit(CheckSD(*sdConfigFile, *sdJobName, *sdTimeout, noDefaultScrapePort, prometheus.DefaultRegisterer))
 
 	case checkConfigCmd.FullCommand():
 		os.Exit(CheckConfig(*agentMode, *checkConfigSyntaxOnly, newLintConfig(*checkConfigLint, *checkConfigLintFatal), *configFiles...))
diff --git a/cmd/promtool/sd.go b/cmd/promtool/sd.go
index 7c5ae7036..155152e1a 100644
--- a/cmd/promtool/sd.go
+++ b/cmd/promtool/sd.go
@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/discovery"
@@ -37,7 +38,7 @@ type sdCheckResult struct {
 }
 
 // CheckSD performs service discovery for the given job name and reports the results.
-func CheckSD(sdConfigFiles, sdJobName string, sdTimeout time.Duration, noDefaultScrapePort bool) int {
+func CheckSD(sdConfigFiles, sdJobName string, sdTimeout time.Duration, noDefaultScrapePort bool, registerer prometheus.Registerer) int {
 	logger := log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr))
 
 	cfg, err := config.LoadFile(sdConfigFiles, false, false, logger)
@@ -77,7 +78,7 @@ func CheckSD(sdConfigFiles, sdJobName string, sdTimeout time.Duration, noDefault
 	defer cancel()
 
 	for _, cfg := range scrapeConfig.ServiceDiscoveryConfigs {
-		d, err := cfg.NewDiscoverer(discovery.DiscovererOptions{Logger: logger})
+		d, err := cfg.NewDiscoverer(discovery.DiscovererOptions{Logger: logger, Registerer: registerer})
 		if err != nil {
 			fmt.Fprintln(os.Stderr, "Could not create new discoverer", err)
 			return failureExitCode
diff --git a/discovery/README.md b/discovery/README.md
index 19b579b39..4c0660862 100644
--- a/discovery/README.md
+++ b/discovery/README.md
@@ -234,6 +234,11 @@ type Config interface {
 
 type DiscovererOptions struct {
 	Logger log.Logger
+
+	// A registerer for the Discoverer's metrics.
+	Registerer prometheus.Registerer
+	
+	HTTPClientOptions []config.HTTPClientOption
 }
 ```
 
diff --git a/discovery/aws/ec2.go b/discovery/aws/ec2.go
index 64c8fdce6..40e6e7cb7 100644
--- a/discovery/aws/ec2.go
+++ b/discovery/aws/ec2.go
@@ -30,6 +30,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -101,7 +102,7 @@ func (*EC2SDConfig) Name() string { return "ec2" }
 
 // NewDiscoverer returns a Discoverer for the EC2 Config.
 func (c *EC2SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewEC2Discovery(c, opts.Logger), nil
+	return NewEC2Discovery(c, opts.Logger, opts.Registerer), nil
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface for the EC2 Config.
@@ -147,7 +148,7 @@ type EC2Discovery struct {
 }
 
 // NewEC2Discovery returns a new EC2Discovery which periodically refreshes its targets.
-func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger) *EC2Discovery {
+func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger, reg prometheus.Registerer) *EC2Discovery {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -156,10 +157,13 @@ func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger) *EC2Discovery {
 		cfg:    conf,
 	}
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"ec2",
-		time.Duration(d.cfg.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "ec2",
+			Interval: time.Duration(d.cfg.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+		},
 	)
 	return d
 }
diff --git a/discovery/aws/lightsail.go b/discovery/aws/lightsail.go
index c0198d6a7..5382ea015 100644
--- a/discovery/aws/lightsail.go
+++ b/discovery/aws/lightsail.go
@@ -29,6 +29,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/lightsail"
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -84,7 +85,7 @@ func (*LightsailSDConfig) Name() string { return "lightsail" }
 
 // NewDiscoverer returns a Discoverer for the Lightsail Config.
 func (c *LightsailSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewLightsailDiscovery(c, opts.Logger), nil
+	return NewLightsailDiscovery(c, opts.Logger, opts.Registerer), nil
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface for the Lightsail Config.
@@ -121,7 +122,7 @@ type LightsailDiscovery struct {
 }
 
 // NewLightsailDiscovery returns a new LightsailDiscovery which periodically refreshes its targets.
-func NewLightsailDiscovery(conf *LightsailSDConfig, logger log.Logger) *LightsailDiscovery {
+func NewLightsailDiscovery(conf *LightsailSDConfig, logger log.Logger, reg prometheus.Registerer) *LightsailDiscovery {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -129,10 +130,13 @@ func NewLightsailDiscovery(conf *LightsailSDConfig, logger log.Logger) *Lightsai
 		cfg: conf,
 	}
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"lightsail",
-		time.Duration(d.cfg.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "lightsail",
+			Interval: time.Duration(d.cfg.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+		},
 	)
 	return d
 }
diff --git a/discovery/azure/azure.go b/discovery/azure/azure.go
index ef953b802..3b67a8102 100644
--- a/discovery/azure/azure.go
+++ b/discovery/azure/azure.go
@@ -79,17 +79,6 @@ var (
 		AuthenticationMethod: authMethodOAuth,
 		HTTPClientConfig:     config_util.DefaultHTTPClientConfig,
 	}
-
-	failuresCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_azure_failures_total",
-			Help: "Number of Azure service discovery refresh failures.",
-		})
-	cacheHitCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_azure_cache_hit_total",
-			Help: "Number of cache hit during refresh.",
-		})
 )
 
 var environments = map[string]cloud.Configuration{
@@ -114,8 +103,6 @@ func CloudConfigurationFromName(name string) (cloud.Configuration, error) {
 
 func init() {
 	discovery.RegisterConfig(&SDConfig{})
-	prometheus.MustRegister(failuresCount)
-	prometheus.MustRegister(cacheHitCount)
 }
 
 // SDConfig is the configuration for Azure based service discovery.
@@ -138,7 +125,7 @@ func (*SDConfig) Name() string { return "azure" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger), nil
+	return NewDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 func validateAuthParam(param, name string) error {
@@ -181,14 +168,16 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 
 type Discovery struct {
 	*refresh.Discovery
-	logger log.Logger
-	cfg    *SDConfig
-	port   int
-	cache  *cache.Cache[string, *armnetwork.Interface]
+	logger        log.Logger
+	cfg           *SDConfig
+	port          int
+	cache         *cache.Cache[string, *armnetwork.Interface]
+	failuresCount prometheus.Counter
+	cacheHitCount prometheus.Counter
 }
 
 // NewDiscovery returns a new AzureDiscovery which periodically refreshes its targets.
-func NewDiscovery(cfg *SDConfig, logger log.Logger) *Discovery {
+func NewDiscovery(cfg *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -198,16 +187,30 @@ func NewDiscovery(cfg *SDConfig, logger log.Logger) *Discovery {
 		port:   cfg.Port,
 		logger: logger,
 		cache:  l,
+		failuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_azure_failures_total",
+				Help: "Number of Azure service discovery refresh failures.",
+			}),
+		cacheHitCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_azure_cache_hit_total",
+				Help: "Number of cache hit during refresh.",
+			}),
 	}
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"azure",
-		time.Duration(cfg.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "azure",
+			Interval: time.Duration(cfg.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+			Metrics:  []prometheus.Collector{d.failuresCount, d.cacheHitCount},
+		},
 	)
 
-	return d
+	return d, nil
 }
 
 // azureClient represents multiple Azure Resource Manager providers.
@@ -329,14 +332,14 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 
 	client, err := createAzureClient(*d.cfg)
 	if err != nil {
-		failuresCount.Inc()
+		d.failuresCount.Inc()
 		return nil, fmt.Errorf("could not create Azure client: %w", err)
 	}
 	client.logger = d.logger
 
 	machines, err := client.getVMs(ctx, d.cfg.ResourceGroup)
 	if err != nil {
-		failuresCount.Inc()
+		d.failuresCount.Inc()
 		return nil, fmt.Errorf("could not get virtual machines: %w", err)
 	}
 
@@ -345,14 +348,14 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 	// Load the vms managed by scale sets.
 	scaleSets, err := client.getScaleSets(ctx, d.cfg.ResourceGroup)
 	if err != nil {
-		failuresCount.Inc()
+		d.failuresCount.Inc()
 		return nil, fmt.Errorf("could not get virtual machine scale sets: %w", err)
 	}
 
 	for _, scaleSet := range scaleSets {
 		scaleSetVms, err := client.getScaleSetVMs(ctx, scaleSet)
 		if err != nil {
-			failuresCount.Inc()
+			d.failuresCount.Inc()
 			return nil, fmt.Errorf("could not get virtual machine scale set vms: %w", err)
 		}
 		machines = append(machines, scaleSetVms...)
@@ -403,7 +406,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 				var networkInterface *armnetwork.Interface
 				if v, ok := d.getFromCache(nicID); ok {
 					networkInterface = v
-					cacheHitCount.Add(1)
+					d.cacheHitCount.Add(1)
 				} else {
 					networkInterface, err = client.getNetworkInterfaceByID(ctx, nicID)
 					if err != nil {
@@ -462,7 +465,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 	var tg targetgroup.Group
 	for tgt := range ch {
 		if tgt.err != nil {
-			failuresCount.Inc()
+			d.failuresCount.Inc()
 			return nil, fmt.Errorf("unable to complete Azure service discovery: %w", tgt.err)
 		}
 		if tgt.labelSet != nil {
diff --git a/discovery/consul/consul.go b/discovery/consul/consul.go
index b4cb15229..50f171a78 100644
--- a/discovery/consul/consul.go
+++ b/discovery/consul/consul.go
@@ -71,41 +71,18 @@ const (
 	namespace = "prometheus"
 )
 
-var (
-	rpcFailuresCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Namespace: namespace,
-			Name:      "sd_consul_rpc_failures_total",
-			Help:      "The number of Consul RPC call failures.",
-		})
-	rpcDuration = prometheus.NewSummaryVec(
-		prometheus.SummaryOpts{
-			Namespace:  namespace,
-			Name:       "sd_consul_rpc_duration_seconds",
-			Help:       "The duration of a Consul RPC call in seconds.",
-			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
-		},
-		[]string{"endpoint", "call"},
-	)
-
-	// Initialize metric vectors.
-	servicesRPCDuration = rpcDuration.WithLabelValues("catalog", "services")
-	serviceRPCDuration  = rpcDuration.WithLabelValues("catalog", "service")
-
-	// DefaultSDConfig is the default Consul SD configuration.
-	DefaultSDConfig = SDConfig{
-		TagSeparator:     ",",
-		Scheme:           "http",
-		Server:           "localhost:8500",
-		AllowStale:       true,
-		RefreshInterval:  model.Duration(30 * time.Second),
-		HTTPClientConfig: config.DefaultHTTPClientConfig,
-	}
-)
+// DefaultSDConfig is the default Consul SD configuration.
+var DefaultSDConfig = SDConfig{
+	TagSeparator:     ",",
+	Scheme:           "http",
+	Server:           "localhost:8500",
+	AllowStale:       true,
+	RefreshInterval:  model.Duration(30 * time.Second),
+	HTTPClientConfig: config.DefaultHTTPClientConfig,
+}
 
 func init() {
 	discovery.RegisterConfig(&SDConfig{})
-	prometheus.MustRegister(rpcFailuresCount, rpcDuration)
 }
 
 // SDConfig is the configuration for Consul service discovery.
@@ -147,7 +124,7 @@ func (*SDConfig) Name() string { return "consul" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -184,22 +161,27 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 // Discovery retrieves target information from a Consul server
 // and updates them via watches.
 type Discovery struct {
-	client           *consul.Client
-	clientDatacenter string
-	clientNamespace  string
-	clientPartition  string
-	tagSeparator     string
-	watchedServices  []string // Set of services which will be discovered.
-	watchedTags      []string // Tags used to filter instances of a service.
-	watchedNodeMeta  map[string]string
-	allowStale       bool
-	refreshInterval  time.Duration
-	finalizer        func()
-	logger           log.Logger
+	client              *consul.Client
+	clientDatacenter    string
+	clientNamespace     string
+	clientPartition     string
+	tagSeparator        string
+	watchedServices     []string // Set of services which will be discovered.
+	watchedTags         []string // Tags used to filter instances of a service.
+	watchedNodeMeta     map[string]string
+	allowStale          bool
+	refreshInterval     time.Duration
+	finalizer           func()
+	logger              log.Logger
+	rpcFailuresCount    prometheus.Counter
+	rpcDuration         *prometheus.SummaryVec
+	servicesRPCDuration prometheus.Observer
+	serviceRPCDuration  prometheus.Observer
+	metricRegisterer    discovery.MetricRegisterer
 }
 
 // NewDiscovery returns a new Discovery for the given config.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -237,7 +219,35 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 		clientPartition:  conf.Partition,
 		finalizer:        wrapper.CloseIdleConnections,
 		logger:           logger,
+		rpcFailuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      "sd_consul_rpc_failures_total",
+				Help:      "The number of Consul RPC call failures.",
+			}),
+		rpcDuration: prometheus.NewSummaryVec(
+			prometheus.SummaryOpts{
+				Namespace:  namespace,
+				Name:       "sd_consul_rpc_duration_seconds",
+				Help:       "The duration of a Consul RPC call in seconds.",
+				Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+			},
+			[]string{"endpoint", "call"},
+		),
 	}
+
+	cd.metricRegisterer = discovery.NewMetricRegisterer(
+		reg,
+		[]prometheus.Collector{
+			cd.rpcFailuresCount,
+			cd.rpcDuration,
+		},
+	)
+
+	// Initialize metric vectors.
+	cd.servicesRPCDuration = cd.rpcDuration.WithLabelValues("catalog", "services")
+	cd.serviceRPCDuration = cd.rpcDuration.WithLabelValues("catalog", "service")
+
 	return cd, nil
 }
 
@@ -293,7 +303,7 @@ func (d *Discovery) getDatacenter() error {
 	info, err := d.client.Agent().Self()
 	if err != nil {
 		level.Error(d.logger).Log("msg", "Error retrieving datacenter name", "err", err)
-		rpcFailuresCount.Inc()
+		d.rpcFailuresCount.Inc()
 		return err
 	}
 
@@ -334,6 +344,13 @@ func (d *Discovery) initialize(ctx context.Context) {
 
 // Run implements the Discoverer interface.
 func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
+	err := d.metricRegisterer.RegisterMetrics()
+	if err != nil {
+		level.Error(d.logger).Log("msg", "Unable to register metrics", "err", err.Error())
+		return
+	}
+	defer d.metricRegisterer.UnregisterMetrics()
+
 	if d.finalizer != nil {
 		defer d.finalizer()
 	}
@@ -382,7 +399,7 @@ func (d *Discovery) watchServices(ctx context.Context, ch chan<- []*targetgroup.
 	t0 := time.Now()
 	srvs, meta, err := catalog.Services(opts.WithContext(ctx))
 	elapsed := time.Since(t0)
-	servicesRPCDuration.Observe(elapsed.Seconds())
+	d.servicesRPCDuration.Observe(elapsed.Seconds())
 
 	// Check the context before in order to exit early.
 	select {
@@ -393,7 +410,7 @@ func (d *Discovery) watchServices(ctx context.Context, ch chan<- []*targetgroup.
 
 	if err != nil {
 		level.Error(d.logger).Log("msg", "Error refreshing service list", "err", err)
-		rpcFailuresCount.Inc()
+		d.rpcFailuresCount.Inc()
 		time.Sleep(retryInterval)
 		return
 	}
@@ -449,13 +466,15 @@ func (d *Discovery) watchServices(ctx context.Context, ch chan<- []*targetgroup.
 
 // consulService contains data belonging to the same service.
 type consulService struct {
-	name         string
-	tags         []string
-	labels       model.LabelSet
-	discovery    *Discovery
-	client       *consul.Client
-	tagSeparator string
-	logger       log.Logger
+	name               string
+	tags               []string
+	labels             model.LabelSet
+	discovery          *Discovery
+	client             *consul.Client
+	tagSeparator       string
+	logger             log.Logger
+	rpcFailuresCount   prometheus.Counter
+	serviceRPCDuration prometheus.Observer
 }
 
 // Start watching a service.
@@ -469,8 +488,10 @@ func (d *Discovery) watchService(ctx context.Context, ch chan<- []*targetgroup.G
 			serviceLabel:    model.LabelValue(name),
 			datacenterLabel: model.LabelValue(d.clientDatacenter),
 		},
-		tagSeparator: d.tagSeparator,
-		logger:       d.logger,
+		tagSeparator:       d.tagSeparator,
+		logger:             d.logger,
+		rpcFailuresCount:   d.rpcFailuresCount,
+		serviceRPCDuration: d.serviceRPCDuration,
 	}
 
 	go func() {
@@ -508,7 +529,7 @@ func (srv *consulService) watch(ctx context.Context, ch chan<- []*targetgroup.Gr
 	t0 := time.Now()
 	serviceNodes, meta, err := health.ServiceMultipleTags(srv.name, srv.tags, false, opts.WithContext(ctx))
 	elapsed := time.Since(t0)
-	serviceRPCDuration.Observe(elapsed.Seconds())
+	srv.serviceRPCDuration.Observe(elapsed.Seconds())
 
 	// Check the context before in order to exit early.
 	select {
@@ -520,7 +541,7 @@ func (srv *consulService) watch(ctx context.Context, ch chan<- []*targetgroup.Gr
 
 	if err != nil {
 		level.Error(srv.logger).Log("msg", "Error refreshing service", "service", srv.name, "tags", strings.Join(srv.tags, ","), "err", err)
-		rpcFailuresCount.Inc()
+		srv.rpcFailuresCount.Inc()
 		time.Sleep(retryInterval)
 		return
 	}
diff --git a/discovery/consul/consul_test.go b/discovery/consul/consul_test.go
index 19f7d3c4a..97cb8fbc9 100644
--- a/discovery/consul/consul_test.go
+++ b/discovery/consul/consul_test.go
@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
@@ -39,7 +40,7 @@ func TestConfiguredService(t *testing.T) {
 	conf := &SDConfig{
 		Services: []string{"configuredServiceName"},
 	}
-	consulDiscovery, err := NewDiscovery(conf, nil)
+	consulDiscovery, err := NewDiscovery(conf, nil, prometheus.NewRegistry())
 	if err != nil {
 		t.Errorf("Unexpected error when initializing discovery %v", err)
 	}
@@ -56,7 +57,7 @@ func TestConfiguredServiceWithTag(t *testing.T) {
 		Services:    []string{"configuredServiceName"},
 		ServiceTags: []string{"http"},
 	}
-	consulDiscovery, err := NewDiscovery(conf, nil)
+	consulDiscovery, err := NewDiscovery(conf, nil, prometheus.NewRegistry())
 	if err != nil {
 		t.Errorf("Unexpected error when initializing discovery %v", err)
 	}
@@ -151,7 +152,7 @@ func TestConfiguredServiceWithTags(t *testing.T) {
 	}
 
 	for _, tc := range cases {
-		consulDiscovery, err := NewDiscovery(tc.conf, nil)
+		consulDiscovery, err := NewDiscovery(tc.conf, nil, prometheus.NewRegistry())
 		if err != nil {
 			t.Errorf("Unexpected error when initializing discovery %v", err)
 		}
@@ -165,7 +166,7 @@ func TestConfiguredServiceWithTags(t *testing.T) {
 
 func TestNonConfiguredService(t *testing.T) {
 	conf := &SDConfig{}
-	consulDiscovery, err := NewDiscovery(conf, nil)
+	consulDiscovery, err := NewDiscovery(conf, nil, prometheus.NewRegistry())
 	if err != nil {
 		t.Errorf("Unexpected error when initializing discovery %v", err)
 	}
@@ -262,7 +263,7 @@ func newServer(t *testing.T) (*httptest.Server, *SDConfig) {
 
 func newDiscovery(t *testing.T, config *SDConfig) *Discovery {
 	logger := log.NewNopLogger()
-	d, err := NewDiscovery(config, logger)
+	d, err := NewDiscovery(config, logger, prometheus.NewRegistry())
 	require.NoError(t, err)
 	return d
 }
diff --git a/discovery/digitalocean/digitalocean.go b/discovery/digitalocean/digitalocean.go
index e207388b3..970258de0 100644
--- a/discovery/digitalocean/digitalocean.go
+++ b/discovery/digitalocean/digitalocean.go
@@ -24,6 +24,7 @@ import (
 
 	"github.com/digitalocean/godo"
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/version"
@@ -75,7 +76,7 @@ func (*SDConfig) Name() string { return "digitalocean" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -103,7 +104,7 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	d := &Discovery{
 		port: conf.Port,
 	}
@@ -125,10 +126,13 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 	}
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"digitalocean",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "digitalocean",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+		},
 	)
 	return d, nil
 }
diff --git a/discovery/digitalocean/digitalocean_test.go b/discovery/digitalocean/digitalocean_test.go
index df2514ecb..a959b312c 100644
--- a/discovery/digitalocean/digitalocean_test.go
+++ b/discovery/digitalocean/digitalocean_test.go
@@ -20,6 +20,7 @@ import (
 	"testing"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 )
@@ -46,7 +47,7 @@ func TestDigitalOceanSDRefresh(t *testing.T) {
 
 	cfg := DefaultSDConfig
 	cfg.HTTPClientConfig.BearerToken = tokenID
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 	require.NoError(t, err)
 	endpoint, err := url.Parse(sdmock.Mock.Endpoint())
 	require.NoError(t, err)
diff --git a/discovery/discovery.go b/discovery/discovery.go
index 9dc010a09..acc4c1efe 100644
--- a/discovery/discovery.go
+++ b/discovery/discovery.go
@@ -18,6 +18,7 @@ import (
 	"reflect"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 
 	"github.com/prometheus/prometheus/discovery/targetgroup"
@@ -42,6 +43,15 @@ type Discoverer interface {
 type DiscovererOptions struct {
 	Logger log.Logger
 
+	// A registerer for the Discoverer's metrics.
+	// Some Discoverers may ignore this registerer and use the global one instead.
+	// For now this will work, because the Prometheus `main` function uses the global registry.
+	// However, in the future the Prometheus `main` function will be updated to not use the global registry.
+	// Hence, if a discoverer wants its metrics to be visible via the Prometheus executable's
+	// `/metrics` endpoint, it should use this explicit registerer.
+	// TODO(ptodev): Update this comment once the Prometheus `main` function does not use the global registry.
+	Registerer prometheus.Registerer
+
 	// Extra HTTP client options to expose to Discoverers. This field may be
 	// ignored; Discoverer implementations must opt-in to reading it.
 	HTTPClientOptions []config.HTTPClientOption
diff --git a/discovery/dns/dns.go b/discovery/dns/dns.go
index 4838a8954..9b6bd6741 100644
--- a/discovery/dns/dns.go
+++ b/discovery/dns/dns.go
@@ -49,30 +49,14 @@ const (
 	namespace = "prometheus"
 )
 
-var (
-	dnsSDLookupsCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Namespace: namespace,
-			Name:      "sd_dns_lookups_total",
-			Help:      "The number of DNS-SD lookups.",
-		})
-	dnsSDLookupFailuresCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Namespace: namespace,
-			Name:      "sd_dns_lookup_failures_total",
-			Help:      "The number of DNS-SD lookup failures.",
-		})
-
-	// DefaultSDConfig is the default DNS SD configuration.
-	DefaultSDConfig = SDConfig{
-		RefreshInterval: model.Duration(30 * time.Second),
-		Type:            "SRV",
-	}
-)
+// DefaultSDConfig is the default DNS SD configuration.
+var DefaultSDConfig = SDConfig{
+	RefreshInterval: model.Duration(30 * time.Second),
+	Type:            "SRV",
+}
 
 func init() {
 	discovery.RegisterConfig(&SDConfig{})
-	prometheus.MustRegister(dnsSDLookupFailuresCount, dnsSDLookupsCount)
 }
 
 // SDConfig is the configuration for DNS based service discovery.
@@ -88,7 +72,7 @@ func (*SDConfig) Name() string { return "dns" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(*c, opts.Logger), nil
+	return NewDiscovery(*c, opts.Logger, opts.Registerer)
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
@@ -118,16 +102,18 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 // the Discoverer interface.
 type Discovery struct {
 	*refresh.Discovery
-	names  []string
-	port   int
-	qtype  uint16
-	logger log.Logger
+	names                    []string
+	port                     int
+	qtype                    uint16
+	logger                   log.Logger
+	dnsSDLookupsCount        prometheus.Counter
+	dnsSDLookupFailuresCount prometheus.Counter
 
 	lookupFn func(name string, qtype uint16, logger log.Logger) (*dns.Msg, error)
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf SDConfig, logger log.Logger) *Discovery {
+func NewDiscovery(conf SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -151,14 +137,32 @@ func NewDiscovery(conf SDConfig, logger log.Logger) *Discovery {
 		port:     conf.Port,
 		logger:   logger,
 		lookupFn: lookupWithSearchPath,
+		dnsSDLookupsCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      "sd_dns_lookups_total",
+				Help:      "The number of DNS-SD lookups.",
+			}),
+		dnsSDLookupFailuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      "sd_dns_lookup_failures_total",
+				Help:      "The number of DNS-SD lookup failures.",
+			}),
 	}
+
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"dns",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "dns",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: prometheus.NewRegistry(),
+			Metrics:  []prometheus.Collector{d.dnsSDLookupsCount, d.dnsSDLookupFailuresCount},
+		},
 	)
-	return d
+
+	return d, nil
 }
 
 func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
@@ -191,9 +195,9 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 
 func (d *Discovery) refreshOne(ctx context.Context, name string, ch chan<- *targetgroup.Group) error {
 	response, err := d.lookupFn(name, d.qtype, d.logger)
-	dnsSDLookupsCount.Inc()
+	d.dnsSDLookupsCount.Inc()
 	if err != nil {
-		dnsSDLookupFailuresCount.Inc()
+		d.dnsSDLookupFailuresCount.Inc()
 		return err
 	}
 
diff --git a/discovery/dns/dns_test.go b/discovery/dns/dns_test.go
index 52ca72c79..b8dd2efaa 100644
--- a/discovery/dns/dns_test.go
+++ b/discovery/dns/dns_test.go
@@ -22,6 +22,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/miekg/dns"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
@@ -252,7 +253,8 @@ func TestDNS(t *testing.T) {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
-			sd := NewDiscovery(tc.config, nil)
+			sd, err := NewDiscovery(tc.config, nil, prometheus.NewRegistry())
+			require.NoError(t, err)
 			sd.lookupFn = tc.lookup
 
 			tgs, err := sd.refresh(context.Background())
diff --git a/discovery/eureka/eureka.go b/discovery/eureka/eureka.go
index 5d9d8d552..d3e4084e5 100644
--- a/discovery/eureka/eureka.go
+++ b/discovery/eureka/eureka.go
@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -80,7 +81,7 @@ func (*SDConfig) Name() string { return "eureka" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -117,7 +118,7 @@ type Discovery struct {
 }
 
 // NewDiscovery creates a new Eureka discovery for the given role.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "eureka_sd")
 	if err != nil {
 		return nil, err
@@ -128,10 +129,13 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 		server: conf.Server,
 	}
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"eureka",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "eureka",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+		},
 	)
 	return d, nil
 }
diff --git a/discovery/eureka/eureka_test.go b/discovery/eureka/eureka_test.go
index cb75e1428..1fe3c710e 100644
--- a/discovery/eureka/eureka_test.go
+++ b/discovery/eureka/eureka_test.go
@@ -20,6 +20,7 @@ import (
 	"net/http/httptest"
 	"testing"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 
@@ -35,7 +36,7 @@ func testUpdateServices(respHandler http.HandlerFunc) ([]*targetgroup.Group, err
 		Server: ts.URL,
 	}
 
-	md, err := NewDiscovery(&conf, nil)
+	md, err := NewDiscovery(&conf, nil, prometheus.NewRegistry())
 	if err != nil {
 		return nil, err
 	}
diff --git a/discovery/file/file.go b/discovery/file/file.go
index 60b63350f..ef6ed1f5e 100644
--- a/discovery/file/file.go
+++ b/discovery/file/file.go
@@ -39,24 +39,6 @@ import (
 )
 
 var (
-	fileSDReadErrorsCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_file_read_errors_total",
-			Help: "The number of File-SD read errors.",
-		})
-	fileSDScanDuration = prometheus.NewSummary(
-		prometheus.SummaryOpts{
-			Name:       "prometheus_sd_file_scan_duration_seconds",
-			Help:       "The duration of the File-SD scan in seconds.",
-			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
-		})
-	fileSDTimeStamp        = NewTimestampCollector()
-	fileWatcherErrorsCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_file_watcher_errors_total",
-			Help: "The number of File-SD errors caused by filesystem watch failures.",
-		})
-
 	patFileSDName = regexp.MustCompile(`^[^*]*(\*[^/]*)?\.(json|yml|yaml|JSON|YML|YAML)$`)
 
 	// DefaultSDConfig is the default file SD configuration.
@@ -67,7 +49,6 @@ var (
 
 func init() {
 	discovery.RegisterConfig(&SDConfig{})
-	prometheus.MustRegister(fileSDReadErrorsCount, fileSDScanDuration, fileSDTimeStamp, fileWatcherErrorsCount)
 }
 
 // SDConfig is the configuration for file based discovery.
@@ -81,7 +62,7 @@ func (*SDConfig) Name() string { return "file" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger), nil
+	return NewDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -187,10 +168,17 @@ type Discovery struct {
 	// This is used to detect deleted target groups.
 	lastRefresh map[string]int
 	logger      log.Logger
+
+	fileSDReadErrorsCount  prometheus.Counter
+	fileSDScanDuration     prometheus.Summary
+	fileWatcherErrorsCount prometheus.Counter
+	fileSDTimeStamp        *TimestampCollector
+
+	metricRegisterer discovery.MetricRegisterer
 }
 
 // NewDiscovery returns a new file discovery for the given paths.
-func NewDiscovery(conf *SDConfig, logger log.Logger) *Discovery {
+func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -200,9 +188,35 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) *Discovery {
 		interval:   time.Duration(conf.RefreshInterval),
 		timestamps: make(map[string]float64),
 		logger:     logger,
+		fileSDReadErrorsCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_file_read_errors_total",
+				Help: "The number of File-SD read errors.",
+			}),
+		fileSDScanDuration: prometheus.NewSummary(
+			prometheus.SummaryOpts{
+				Name:       "prometheus_sd_file_scan_duration_seconds",
+				Help:       "The duration of the File-SD scan in seconds.",
+				Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+			}),
+		fileWatcherErrorsCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_file_watcher_errors_total",
+				Help: "The number of File-SD errors caused by filesystem watch failures.",
+			}),
+		fileSDTimeStamp: NewTimestampCollector(),
 	}
-	fileSDTimeStamp.addDiscoverer(disc)
-	return disc
+
+	disc.fileSDTimeStamp.addDiscoverer(disc)
+
+	disc.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
+		disc.fileSDReadErrorsCount,
+		disc.fileSDScanDuration,
+		disc.fileWatcherErrorsCount,
+		disc.fileSDTimeStamp,
+	})
+
+	return disc, nil
 }
 
 // listFiles returns a list of all files that match the configured patterns.
@@ -239,10 +253,17 @@ func (d *Discovery) watchFiles() {
 
 // Run implements the Discoverer interface.
 func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
+	err := d.metricRegisterer.RegisterMetrics()
+	if err != nil {
+		level.Error(d.logger).Log("msg", "Unable to register metrics", "err", err.Error())
+		return
+	}
+	defer d.metricRegisterer.UnregisterMetrics()
+
 	watcher, err := fsnotify.NewWatcher()
 	if err != nil {
 		level.Error(d.logger).Log("msg", "Error adding file watcher", "err", err)
-		fileWatcherErrorsCount.Inc()
+		d.fileWatcherErrorsCount.Inc()
 		return
 	}
 	d.watcher = watcher
@@ -306,7 +327,7 @@ func (d *Discovery) stop() {
 	done := make(chan struct{})
 	defer close(done)
 
-	fileSDTimeStamp.removeDiscoverer(d)
+	d.fileSDTimeStamp.removeDiscoverer(d)
 
 	// Closing the watcher will deadlock unless all events and errors are drained.
 	go func() {
@@ -332,13 +353,13 @@ func (d *Discovery) stop() {
 func (d *Discovery) refresh(ctx context.Context, ch chan<- []*targetgroup.Group) {
 	t0 := time.Now()
 	defer func() {
-		fileSDScanDuration.Observe(time.Since(t0).Seconds())
+		d.fileSDScanDuration.Observe(time.Since(t0).Seconds())
 	}()
 	ref := map[string]int{}
 	for _, p := range d.listFiles() {
 		tgroups, err := d.readFile(p)
 		if err != nil {
-			fileSDReadErrorsCount.Inc()
+			d.fileSDReadErrorsCount.Inc()
 
 			level.Error(d.logger).Log("msg", "Error reading file", "path", p, "err", err)
 			// Prevent deletion down below.
diff --git a/discovery/file/file_test.go b/discovery/file/file_test.go
index 76e1cebed..c138fc8a9 100644
--- a/discovery/file/file_test.go
+++ b/discovery/file/file_test.go
@@ -24,6 +24,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
@@ -143,7 +144,7 @@ func (t *testRunner) run(files ...string) {
 	ctx, cancel := context.WithCancel(context.Background())
 	t.cancelSD = cancel
 	go func() {
-		NewDiscovery(
+		d, err := NewDiscovery(
 			&SDConfig{
 				Files: files,
 				// Setting a high refresh interval to make sure that the tests only
@@ -151,7 +152,11 @@ func (t *testRunner) run(files ...string) {
 				RefreshInterval: model.Duration(1 * time.Hour),
 			},
 			nil,
-		).Run(ctx, t.ch)
+			prometheus.NewRegistry(),
+		)
+		require.NoError(t, err)
+
+		d.Run(ctx, t.ch)
 	}()
 }
 
diff --git a/discovery/gce/gce.go b/discovery/gce/gce.go
index fa05fbbf3..21a95ee39 100644
--- a/discovery/gce/gce.go
+++ b/discovery/gce/gce.go
@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"golang.org/x/oauth2/google"
 	"google.golang.org/api/compute/v1"
@@ -86,7 +87,7 @@ func (*SDConfig) Name() string { return "gce" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(*c, opts.Logger)
+	return NewDiscovery(*c, opts.Logger, opts.Registerer)
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
@@ -121,7 +122,7 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf SDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	d := &Discovery{
 		project:      conf.Project,
 		zone:         conf.Zone,
@@ -141,10 +142,13 @@ func NewDiscovery(conf SDConfig, logger log.Logger) (*Discovery, error) {
 	d.isvc = compute.NewInstancesService(d.svc)
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"gce",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "gce",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+		},
 	)
 	return d, nil
 }
diff --git a/discovery/hetzner/hetzner.go b/discovery/hetzner/hetzner.go
index c3f7ec39c..9d3e6aa65 100644
--- a/discovery/hetzner/hetzner.go
+++ b/discovery/hetzner/hetzner.go
@@ -21,6 +21,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/hetznercloud/hcloud-go/v2/hcloud"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -67,7 +68,7 @@ func (*SDConfig) Name() string { return "hetzner" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 type refresher interface {
@@ -127,17 +128,20 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*refresh.Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*refresh.Discovery, error) {
 	r, err := newRefresher(conf, logger)
 	if err != nil {
 		return nil, err
 	}
 
 	return refresh.NewDiscovery(
-		logger,
-		"hetzner",
-		time.Duration(conf.RefreshInterval),
-		r.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "hetzner",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: r.refresh,
+			Registry: reg,
+		},
 	), nil
 }
 
diff --git a/discovery/http/http.go b/discovery/http/http.go
index 2980d7efd..c12fdb26d 100644
--- a/discovery/http/http.go
+++ b/discovery/http/http.go
@@ -45,17 +45,10 @@ var (
 	}
 	userAgent        = fmt.Sprintf("Prometheus/%s", version.Version)
 	matchContentType = regexp.MustCompile(`^(?i:application\/json(;\s*charset=("utf-8"|utf-8))?)$`)
-
-	failuresCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_http_failures_total",
-			Help: "Number of HTTP service discovery refresh failures.",
-		})
 )
 
 func init() {
 	discovery.RegisterConfig(&SDConfig{})
-	prometheus.MustRegister(failuresCount)
 }
 
 // SDConfig is the configuration for HTTP based discovery.
@@ -70,7 +63,7 @@ func (*SDConfig) Name() string { return "http" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.HTTPClientOptions)
+	return NewDiscovery(c, opts.Logger, opts.HTTPClientOptions, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -112,10 +105,11 @@ type Discovery struct {
 	client          *http.Client
 	refreshInterval time.Duration
 	tgLastLength    int
+	failuresCount   prometheus.Counter
 }
 
 // NewDiscovery returns a new HTTP discovery for the given config.
-func NewDiscovery(conf *SDConfig, logger log.Logger, clientOpts []config.HTTPClientOption) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, clientOpts []config.HTTPClientOption, reg prometheus.Registerer) (*Discovery, error) {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -130,13 +124,22 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, clientOpts []config.HTTPCli
 		url:             conf.URL,
 		client:          client,
 		refreshInterval: time.Duration(conf.RefreshInterval), // Stored to be sent as headers.
+		failuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_http_failures_total",
+				Help: "Number of HTTP service discovery refresh failures.",
+			}),
 	}
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"http",
-		time.Duration(conf.RefreshInterval),
-		d.Refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "http",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.Refresh,
+			Registry: reg,
+			Metrics:  []prometheus.Collector{d.failuresCount},
+		},
 	)
 	return d, nil
 }
@@ -152,7 +155,7 @@ func (d *Discovery) Refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 
 	resp, err := d.client.Do(req.WithContext(ctx))
 	if err != nil {
-		failuresCount.Inc()
+		d.failuresCount.Inc()
 		return nil, err
 	}
 	defer func() {
@@ -161,31 +164,31 @@ func (d *Discovery) Refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 	}()
 
 	if resp.StatusCode != http.StatusOK {
-		failuresCount.Inc()
+		d.failuresCount.Inc()
 		return nil, fmt.Errorf("server returned HTTP status %s", resp.Status)
 	}
 
 	if !matchContentType.MatchString(strings.TrimSpace(resp.Header.Get("Content-Type"))) {
-		failuresCount.Inc()
+		d.failuresCount.Inc()
 		return nil, fmt.Errorf("unsupported content type %q", resp.Header.Get("Content-Type"))
 	}
 
 	b, err := io.ReadAll(resp.Body)
 	if err != nil {
-		failuresCount.Inc()
+		d.failuresCount.Inc()
 		return nil, err
 	}
 
 	var targetGroups []*targetgroup.Group
 
 	if err := json.Unmarshal(b, &targetGroups); err != nil {
-		failuresCount.Inc()
+		d.failuresCount.Inc()
 		return nil, err
 	}
 
 	for i, tg := range targetGroups {
 		if tg == nil {
-			failuresCount.Inc()
+			d.failuresCount.Inc()
 			err = errors.New("nil target group item found")
 			return nil, err
 		}
diff --git a/discovery/http/http_test.go b/discovery/http/http_test.go
index 9bbda95b7..164719e90 100644
--- a/discovery/http/http_test.go
+++ b/discovery/http/http_test.go
@@ -41,7 +41,7 @@ func TestHTTPValidRefresh(t *testing.T) {
 		RefreshInterval:  model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil)
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, prometheus.NewRegistry())
 	require.NoError(t, err)
 
 	ctx := context.Background()
@@ -63,7 +63,7 @@ func TestHTTPValidRefresh(t *testing.T) {
 		},
 	}
 	require.Equal(t, expectedTargets, tgs)
-	require.Equal(t, 0.0, getFailureCount())
+	require.Equal(t, 0.0, getFailureCount(d.failuresCount))
 }
 
 func TestHTTPInvalidCode(t *testing.T) {
@@ -79,13 +79,13 @@ func TestHTTPInvalidCode(t *testing.T) {
 		RefreshInterval:  model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil)
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, prometheus.NewRegistry())
 	require.NoError(t, err)
 
 	ctx := context.Background()
 	_, err = d.Refresh(ctx)
 	require.EqualError(t, err, "server returned HTTP status 400 Bad Request")
-	require.Equal(t, 1.0, getFailureCount())
+	require.Equal(t, 1.0, getFailureCount(d.failuresCount))
 }
 
 func TestHTTPInvalidFormat(t *testing.T) {
@@ -101,18 +101,16 @@ func TestHTTPInvalidFormat(t *testing.T) {
 		RefreshInterval:  model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil)
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, prometheus.NewRegistry())
 	require.NoError(t, err)
 
 	ctx := context.Background()
 	_, err = d.Refresh(ctx)
 	require.EqualError(t, err, `unsupported content type "text/plain; charset=utf-8"`)
-	require.Equal(t, 1.0, getFailureCount())
+	require.Equal(t, 1.0, getFailureCount(d.failuresCount))
 }
 
-var lastFailureCount float64
-
-func getFailureCount() float64 {
+func getFailureCount(failuresCount prometheus.Counter) float64 {
 	failureChan := make(chan prometheus.Metric)
 
 	go func() {
@@ -129,10 +127,7 @@ func getFailureCount() float64 {
 		metric.Write(&counter)
 	}
 
-	// account for failures in prior tests
-	count := *counter.Counter.Value - lastFailureCount
-	lastFailureCount = *counter.Counter.Value
-	return count
+	return *counter.Counter.Value
 }
 
 func TestContentTypeRegex(t *testing.T) {
@@ -417,7 +412,7 @@ func TestSourceDisappeared(t *testing.T) {
 		URL:              ts.URL,
 		RefreshInterval:  model.Duration(1 * time.Second),
 	}
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil)
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, prometheus.NewRegistry())
 	require.NoError(t, err)
 	for _, test := range cases {
 		ctx := context.Background()
diff --git a/discovery/ionos/ionos.go b/discovery/ionos/ionos.go
index 3afed8d79..36623745a 100644
--- a/discovery/ionos/ionos.go
+++ b/discovery/ionos/ionos.go
@@ -23,6 +23,8 @@ import (
 
 	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/refresh"
+
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 const (
@@ -41,7 +43,7 @@ func init() {
 type Discovery struct{}
 
 // NewDiscovery returns a new refresh.Discovery for IONOS Cloud.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*refresh.Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*refresh.Discovery, error) {
 	if conf.ionosEndpoint == "" {
 		conf.ionosEndpoint = "https://api.ionos.com"
 	}
@@ -52,10 +54,13 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*refresh.Discovery, error)
 	}
 
 	return refresh.NewDiscovery(
-		logger,
-		"ionos",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "ionos",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+		},
 	), nil
 }
 
@@ -86,7 +91,7 @@ func (c SDConfig) Name() string {
 
 // NewDiscoverer returns a new discovery.Discoverer for IONOS Cloud.
 func (c SDConfig) NewDiscoverer(options discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(&c, options.Logger)
+	return NewDiscovery(&c, options.Logger, options.Registerer)
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
diff --git a/discovery/kubernetes/endpoints.go b/discovery/kubernetes/endpoints.go
index 708e229a2..801a45f7c 100644
--- a/discovery/kubernetes/endpoints.go
+++ b/discovery/kubernetes/endpoints.go
@@ -22,6 +22,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	apiv1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/tools/cache"
@@ -30,12 +31,6 @@ import (
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
-var (
-	epAddCount    = eventCount.WithLabelValues("endpoints", "add")
-	epUpdateCount = eventCount.WithLabelValues("endpoints", "update")
-	epDeleteCount = eventCount.WithLabelValues("endpoints", "delete")
-)
-
 // Endpoints discovers new endpoint targets.
 type Endpoints struct {
 	logger log.Logger
@@ -54,7 +49,7 @@ type Endpoints struct {
 }
 
 // NewEndpoints returns a new endpoints discovery.
-func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node cache.SharedInformer) *Endpoints {
+func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node cache.SharedInformer, eventCount *prometheus.CounterVec) *Endpoints {
 	if l == nil {
 		l = log.NewNopLogger()
 	}
@@ -73,15 +68,15 @@ func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node ca
 
 	_, err := e.endpointsInf.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(o interface{}) {
-			epAddCount.Inc()
+			eventCount.WithLabelValues("endpoints", "add").Inc()
 			e.enqueue(o)
 		},
 		UpdateFunc: func(_, o interface{}) {
-			epUpdateCount.Inc()
+			eventCount.WithLabelValues("endpoints", "update").Inc()
 			e.enqueue(o)
 		},
 		DeleteFunc: func(o interface{}) {
-			epDeleteCount.Inc()
+			eventCount.WithLabelValues("endpoints", "delete").Inc()
 			e.enqueue(o)
 		},
 	})
@@ -112,15 +107,15 @@ func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node ca
 		// TODO(fabxc): potentially remove add and delete event handlers. Those should
 		// be triggered via the endpoint handlers already.
 		AddFunc: func(o interface{}) {
-			svcAddCount.Inc()
+			eventCount.WithLabelValues("service", "add").Inc()
 			serviceUpdate(o)
 		},
 		UpdateFunc: func(_, o interface{}) {
-			svcUpdateCount.Inc()
+			eventCount.WithLabelValues("service", "update").Inc()
 			serviceUpdate(o)
 		},
 		DeleteFunc: func(o interface{}) {
-			svcDeleteCount.Inc()
+			eventCount.WithLabelValues("service", "delete").Inc()
 			serviceUpdate(o)
 		},
 	})
diff --git a/discovery/kubernetes/endpointslice.go b/discovery/kubernetes/endpointslice.go
index a16862380..e2ac1de42 100644
--- a/discovery/kubernetes/endpointslice.go
+++ b/discovery/kubernetes/endpointslice.go
@@ -22,6 +22,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	apiv1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/discovery/v1"
@@ -33,12 +34,6 @@ import (
 	"github.com/prometheus/prometheus/util/strutil"
 )
 
-var (
-	epslAddCount    = eventCount.WithLabelValues("endpointslice", "add")
-	epslUpdateCount = eventCount.WithLabelValues("endpointslice", "update")
-	epslDeleteCount = eventCount.WithLabelValues("endpointslice", "delete")
-)
-
 // EndpointSlice discovers new endpoint targets.
 type EndpointSlice struct {
 	logger log.Logger
@@ -57,7 +52,7 @@ type EndpointSlice struct {
 }
 
 // NewEndpointSlice returns a new endpointslice discovery.
-func NewEndpointSlice(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node cache.SharedInformer) *EndpointSlice {
+func NewEndpointSlice(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node cache.SharedInformer, eventCount *prometheus.CounterVec) *EndpointSlice {
 	if l == nil {
 		l = log.NewNopLogger()
 	}
@@ -76,15 +71,15 @@ func NewEndpointSlice(l log.Logger, eps cache.SharedIndexInformer, svc, pod, nod
 
 	_, err := e.endpointSliceInf.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(o interface{}) {
-			epslAddCount.Inc()
+			eventCount.WithLabelValues("endpointslice", "add").Inc()
 			e.enqueue(o)
 		},
 		UpdateFunc: func(_, o interface{}) {
-			epslUpdateCount.Inc()
+			eventCount.WithLabelValues("endpointslice", "update").Inc()
 			e.enqueue(o)
 		},
 		DeleteFunc: func(o interface{}) {
-			epslDeleteCount.Inc()
+			eventCount.WithLabelValues("endpointslice", "delete").Inc()
 			e.enqueue(o)
 		},
 	})
@@ -115,15 +110,15 @@ func NewEndpointSlice(l log.Logger, eps cache.SharedIndexInformer, svc, pod, nod
 	}
 	_, err = e.serviceInf.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(o interface{}) {
-			svcAddCount.Inc()
+			eventCount.WithLabelValues("service", "add").Inc()
 			serviceUpdate(o)
 		},
 		UpdateFunc: func(_, o interface{}) {
-			svcUpdateCount.Inc()
+			eventCount.WithLabelValues("service", "update").Inc()
 			serviceUpdate(o)
 		},
 		DeleteFunc: func(o interface{}) {
-			svcDeleteCount.Inc()
+			eventCount.WithLabelValues("service", "delete").Inc()
 			serviceUpdate(o)
 		},
 	})
diff --git a/discovery/kubernetes/ingress.go b/discovery/kubernetes/ingress.go
index fee4cc720..91b1fb0ee 100644
--- a/discovery/kubernetes/ingress.go
+++ b/discovery/kubernetes/ingress.go
@@ -21,6 +21,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	v1 "k8s.io/api/networking/v1"
 	"k8s.io/api/networking/v1beta1"
@@ -30,12 +31,6 @@ import (
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
-var (
-	ingressAddCount    = eventCount.WithLabelValues("ingress", "add")
-	ingressUpdateCount = eventCount.WithLabelValues("ingress", "update")
-	ingressDeleteCount = eventCount.WithLabelValues("ingress", "delete")
-)
-
 // Ingress implements discovery of Kubernetes ingress.
 type Ingress struct {
 	logger   log.Logger
@@ -45,19 +40,19 @@ type Ingress struct {
 }
 
 // NewIngress returns a new ingress discovery.
-func NewIngress(l log.Logger, inf cache.SharedInformer) *Ingress {
+func NewIngress(l log.Logger, inf cache.SharedInformer, eventCount *prometheus.CounterVec) *Ingress {
 	s := &Ingress{logger: l, informer: inf, store: inf.GetStore(), queue: workqueue.NewNamed("ingress")}
 	_, err := s.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(o interface{}) {
-			ingressAddCount.Inc()
+			eventCount.WithLabelValues("ingress", "add").Inc()
 			s.enqueue(o)
 		},
 		DeleteFunc: func(o interface{}) {
-			ingressDeleteCount.Inc()
+			eventCount.WithLabelValues("ingress", "delete").Inc()
 			s.enqueue(o)
 		},
 		UpdateFunc: func(_, o interface{}) {
-			ingressUpdateCount.Inc()
+			eventCount.WithLabelValues("ingress", "update").Inc()
 			s.enqueue(o)
 		},
 	})
diff --git a/discovery/kubernetes/kubernetes.go b/discovery/kubernetes/kubernetes.go
index 7bd96652f..4deaf3f68 100644
--- a/discovery/kubernetes/kubernetes.go
+++ b/discovery/kubernetes/kubernetes.go
@@ -58,24 +58,14 @@ import (
 const (
 	// metaLabelPrefix is the meta prefix used for all meta labels.
 	// in this discovery.
-	metaLabelPrefix  = model.MetaLabelPrefix + "kubernetes_"
-	namespaceLabel   = metaLabelPrefix + "namespace"
-	metricsNamespace = "prometheus_sd_kubernetes"
-	presentValue     = model.LabelValue("true")
+	metaLabelPrefix = model.MetaLabelPrefix + "kubernetes_"
+	namespaceLabel  = metaLabelPrefix + "namespace"
+	presentValue    = model.LabelValue("true")
 )
 
 var (
 	// Http header.
 	userAgent = fmt.Sprintf("Prometheus/%s", version.Version)
-	// Custom events metric.
-	eventCount = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Name:      "events_total",
-			Help:      "The number of Kubernetes events handled.",
-		},
-		[]string{"role", "event"},
-	)
 	// DefaultSDConfig is the default Kubernetes SD configuration.
 	DefaultSDConfig = SDConfig{
 		HTTPClientConfig: config.DefaultHTTPClientConfig,
@@ -84,15 +74,6 @@ var (
 
 func init() {
 	discovery.RegisterConfig(&SDConfig{})
-	prometheus.MustRegister(eventCount)
-	// Initialize metric vectors.
-	for _, role := range []string{"endpointslice", "endpoints", "node", "pod", "service", "ingress"} {
-		for _, evt := range []string{"add", "delete", "update"} {
-			eventCount.WithLabelValues(role, evt)
-		}
-	}
-	(&clientGoRequestMetricAdapter{}).Register(prometheus.DefaultRegisterer)
-	(&clientGoWorkqueueMetricsProvider{}).Register(prometheus.DefaultRegisterer)
 }
 
 // Role is role of the service in Kubernetes.
@@ -137,7 +118,7 @@ func (*SDConfig) Name() string { return "kubernetes" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return New(opts.Logger, c)
+	return New(opts.Logger, opts.Registerer, c)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -274,6 +255,8 @@ type Discovery struct {
 	selectors          roleSelector
 	ownNamespace       string
 	attachMetadata     AttachMetadataConfig
+	eventCount         *prometheus.CounterVec
+	metricRegisterer   discovery.MetricRegisterer
 }
 
 func (d *Discovery) getNamespaces() []string {
@@ -292,7 +275,7 @@ func (d *Discovery) getNamespaces() []string {
 }
 
 // New creates a new Kubernetes discovery for the given role.
-func New(l log.Logger, conf *SDConfig) (*Discovery, error) {
+func New(l log.Logger, reg prometheus.Registerer, conf *SDConfig) (*Discovery, error) {
 	if l == nil {
 		l = log.NewNopLogger()
 	}
@@ -346,7 +329,7 @@ func New(l log.Logger, conf *SDConfig) (*Discovery, error) {
 		return nil, err
 	}
 
-	return &Discovery{
+	d := &Discovery{
 		client:             c,
 		logger:             l,
 		role:               conf.Role,
@@ -355,7 +338,26 @@ func New(l log.Logger, conf *SDConfig) (*Discovery, error) {
 		selectors:          mapSelector(conf.Selectors),
 		ownNamespace:       ownNamespace,
 		attachMetadata:     conf.AttachMetadata,
-	}, nil
+		eventCount: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Namespace: discovery.KubernetesMetricsNamespace,
+				Name:      "events_total",
+				Help:      "The number of Kubernetes events handled.",
+			},
+			[]string{"role", "event"},
+		),
+	}
+
+	d.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{d.eventCount})
+
+	// Initialize metric vectors.
+	for _, role := range []string{"endpointslice", "endpoints", "node", "pod", "service", "ingress"} {
+		for _, evt := range []string{"add", "delete", "update"} {
+			d.eventCount.WithLabelValues(role, evt)
+		}
+	}
+
+	return d, nil
 }
 
 func mapSelector(rawSelector []SelectorConfig) roleSelector {
@@ -391,6 +393,14 @@ const resyncDisabled = 0
 // Run implements the discoverer interface.
 func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 	d.Lock()
+
+	err := d.metricRegisterer.RegisterMetrics()
+	if err != nil {
+		level.Error(d.logger).Log("msg", "Unable to register metrics", "err", err.Error())
+		return
+	}
+	defer d.metricRegisterer.UnregisterMetrics()
+
 	namespaces := d.getNamespaces()
 
 	switch d.role {
@@ -482,6 +492,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 				cache.NewSharedInformer(slw, &apiv1.Service{}, resyncDisabled),
 				cache.NewSharedInformer(plw, &apiv1.Pod{}, resyncDisabled),
 				nodeInf,
+				d.eventCount,
 			)
 			d.discoverers = append(d.discoverers, eps)
 			go eps.endpointSliceInf.Run(ctx.Done())
@@ -541,6 +552,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 				cache.NewSharedInformer(slw, &apiv1.Service{}, resyncDisabled),
 				cache.NewSharedInformer(plw, &apiv1.Pod{}, resyncDisabled),
 				nodeInf,
+				d.eventCount,
 			)
 			d.discoverers = append(d.discoverers, eps)
 			go eps.endpointsInf.Run(ctx.Done())
@@ -572,6 +584,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 				log.With(d.logger, "role", "pod"),
 				d.newPodsByNodeInformer(plw),
 				nodeInformer,
+				d.eventCount,
 			)
 			d.discoverers = append(d.discoverers, pod)
 			go pod.podInf.Run(ctx.Done())
@@ -594,6 +607,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 			svc := NewService(
 				log.With(d.logger, "role", "service"),
 				cache.NewSharedInformer(slw, &apiv1.Service{}, resyncDisabled),
+				d.eventCount,
 			)
 			d.discoverers = append(d.discoverers, svc)
 			go svc.informer.Run(ctx.Done())
@@ -651,13 +665,14 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 			ingress := NewIngress(
 				log.With(d.logger, "role", "ingress"),
 				informer,
+				d.eventCount,
 			)
 			d.discoverers = append(d.discoverers, ingress)
 			go ingress.informer.Run(ctx.Done())
 		}
 	case RoleNode:
 		nodeInformer := d.newNodeInformer(ctx)
-		node := NewNode(log.With(d.logger, "role", "node"), nodeInformer)
+		node := NewNode(log.With(d.logger, "role", "node"), nodeInformer, d.eventCount)
 		d.discoverers = append(d.discoverers, node)
 		go node.informer.Run(ctx.Done())
 	default:
diff --git a/discovery/kubernetes/kubernetes_test.go b/discovery/kubernetes/kubernetes_test.go
index d0ed4c6ca..71c937e94 100644
--- a/discovery/kubernetes/kubernetes_test.go
+++ b/discovery/kubernetes/kubernetes_test.go
@@ -29,6 +29,8 @@ import (
 	"k8s.io/client-go/kubernetes/fake"
 	"k8s.io/client-go/tools/cache"
 
+	"github.com/prometheus/client_golang/prometheus"
+
 	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 	"github.com/prometheus/prometheus/util/testutil"
@@ -49,13 +51,25 @@ func makeDiscoveryWithVersion(role Role, nsDiscovery NamespaceDiscovery, k8sVer
 	fakeDiscovery, _ := clientset.Discovery().(*fakediscovery.FakeDiscovery)
 	fakeDiscovery.FakedServerVersion = &version.Info{GitVersion: k8sVer}
 
-	return &Discovery{
+	d := &Discovery{
 		client:             clientset,
 		logger:             log.NewNopLogger(),
 		role:               role,
 		namespaceDiscovery: &nsDiscovery,
 		ownNamespace:       "own-ns",
-	}, clientset
+		eventCount: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Namespace: discovery.KubernetesMetricsNamespace,
+				Name:      "events_total",
+				Help:      "The number of Kubernetes events handled.",
+			},
+			[]string{"role", "event"},
+		),
+	}
+
+	d.metricRegisterer = discovery.NewMetricRegisterer(prometheus.NewRegistry(), []prometheus.Collector{d.eventCount})
+
+	return d, clientset
 }
 
 // makeDiscoveryWithMetadata creates a kubernetes.Discovery instance with the specified metadata config.
diff --git a/discovery/kubernetes/node.go b/discovery/kubernetes/node.go
index b188a3ceb..f1e37e6fa 100644
--- a/discovery/kubernetes/node.go
+++ b/discovery/kubernetes/node.go
@@ -22,6 +22,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	apiv1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/tools/cache"
@@ -35,12 +36,6 @@ const (
 	NodeLegacyHostIP = "LegacyHostIP"
 )
 
-var (
-	nodeAddCount    = eventCount.WithLabelValues("node", "add")
-	nodeUpdateCount = eventCount.WithLabelValues("node", "update")
-	nodeDeleteCount = eventCount.WithLabelValues("node", "delete")
-)
-
 // Node discovers Kubernetes nodes.
 type Node struct {
 	logger   log.Logger
@@ -50,22 +45,22 @@ type Node struct {
 }
 
 // NewNode returns a new node discovery.
-func NewNode(l log.Logger, inf cache.SharedInformer) *Node {
+func NewNode(l log.Logger, inf cache.SharedInformer, eventCount *prometheus.CounterVec) *Node {
 	if l == nil {
 		l = log.NewNopLogger()
 	}
 	n := &Node{logger: l, informer: inf, store: inf.GetStore(), queue: workqueue.NewNamed("node")}
 	_, err := n.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(o interface{}) {
-			nodeAddCount.Inc()
+			eventCount.WithLabelValues("node", "add").Inc()
 			n.enqueue(o)
 		},
 		DeleteFunc: func(o interface{}) {
-			nodeDeleteCount.Inc()
+			eventCount.WithLabelValues("node", "delete").Inc()
 			n.enqueue(o)
 		},
 		UpdateFunc: func(_, o interface{}) {
-			nodeUpdateCount.Inc()
+			eventCount.WithLabelValues("node", "update").Inc()
 			n.enqueue(o)
 		},
 	})
diff --git a/discovery/kubernetes/pod.go b/discovery/kubernetes/pod.go
index 88da7bba6..cc809b29c 100644
--- a/discovery/kubernetes/pod.go
+++ b/discovery/kubernetes/pod.go
@@ -23,6 +23,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	apiv1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -34,12 +35,6 @@ import (
 
 const nodeIndex = "node"
 
-var (
-	podAddCount    = eventCount.WithLabelValues("pod", "add")
-	podUpdateCount = eventCount.WithLabelValues("pod", "update")
-	podDeleteCount = eventCount.WithLabelValues("pod", "delete")
-)
-
 // Pod discovers new pod targets.
 type Pod struct {
 	podInf           cache.SharedIndexInformer
@@ -51,7 +46,7 @@ type Pod struct {
 }
 
 // NewPod creates a new pod discovery.
-func NewPod(l log.Logger, pods cache.SharedIndexInformer, nodes cache.SharedInformer) *Pod {
+func NewPod(l log.Logger, pods cache.SharedIndexInformer, nodes cache.SharedInformer, eventCount *prometheus.CounterVec) *Pod {
 	if l == nil {
 		l = log.NewNopLogger()
 	}
@@ -66,15 +61,15 @@ func NewPod(l log.Logger, pods cache.SharedIndexInformer, nodes cache.SharedInfo
 	}
 	_, err := p.podInf.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(o interface{}) {
-			podAddCount.Inc()
+			eventCount.WithLabelValues("pod", "add").Inc()
 			p.enqueue(o)
 		},
 		DeleteFunc: func(o interface{}) {
-			podDeleteCount.Inc()
+			eventCount.WithLabelValues("pod", "delete").Inc()
 			p.enqueue(o)
 		},
 		UpdateFunc: func(_, o interface{}) {
-			podUpdateCount.Inc()
+			eventCount.WithLabelValues("pod", "update").Inc()
 			p.enqueue(o)
 		},
 	})
diff --git a/discovery/kubernetes/service.go b/discovery/kubernetes/service.go
index 9fcc6644c..a680ebee8 100644
--- a/discovery/kubernetes/service.go
+++ b/discovery/kubernetes/service.go
@@ -22,6 +22,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	apiv1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/tools/cache"
@@ -30,12 +31,6 @@ import (
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
-var (
-	svcAddCount    = eventCount.WithLabelValues("service", "add")
-	svcUpdateCount = eventCount.WithLabelValues("service", "update")
-	svcDeleteCount = eventCount.WithLabelValues("service", "delete")
-)
-
 // Service implements discovery of Kubernetes services.
 type Service struct {
 	logger   log.Logger
@@ -45,22 +40,22 @@ type Service struct {
 }
 
 // NewService returns a new service discovery.
-func NewService(l log.Logger, inf cache.SharedInformer) *Service {
+func NewService(l log.Logger, inf cache.SharedInformer, eventCount *prometheus.CounterVec) *Service {
 	if l == nil {
 		l = log.NewNopLogger()
 	}
 	s := &Service{logger: l, informer: inf, store: inf.GetStore(), queue: workqueue.NewNamed("service")}
 	_, err := s.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(o interface{}) {
-			svcAddCount.Inc()
+			eventCount.WithLabelValues("service", "add").Inc()
 			s.enqueue(o)
 		},
 		DeleteFunc: func(o interface{}) {
-			svcDeleteCount.Inc()
+			eventCount.WithLabelValues("service", "delete").Inc()
 			s.enqueue(o)
 		},
 		UpdateFunc: func(_, o interface{}) {
-			svcUpdateCount.Inc()
+			eventCount.WithLabelValues("service", "update").Inc()
 			s.enqueue(o)
 		},
 	})
diff --git a/discovery/legacymanager/manager.go b/discovery/legacymanager/manager.go
index 74c544e72..101012daf 100644
--- a/discovery/legacymanager/manager.go
+++ b/discovery/legacymanager/manager.go
@@ -28,48 +28,6 @@ import (
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
-var (
-	failedConfigs = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "prometheus_sd_failed_configs",
-			Help: "Current number of service discovery configurations that failed to load.",
-		},
-		[]string{"name"},
-	)
-	discoveredTargets = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "prometheus_sd_discovered_targets",
-			Help: "Current number of discovered targets.",
-		},
-		[]string{"name", "config"},
-	)
-	receivedUpdates = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_received_updates_total",
-			Help: "Total number of update events received from the SD providers.",
-		},
-		[]string{"name"},
-	)
-	delayedUpdates = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_updates_delayed_total",
-			Help: "Total number of update events that couldn't be sent immediately.",
-		},
-		[]string{"name"},
-	)
-	sentUpdates = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_updates_total",
-			Help: "Total number of update events sent to the SD consumers.",
-		},
-		[]string{"name"},
-	)
-)
-
-func RegisterMetrics() {
-	prometheus.MustRegister(failedConfigs, discoveredTargets, receivedUpdates, delayedUpdates, sentUpdates)
-}
-
 type poolKey struct {
 	setName  string
 	provider string
@@ -84,7 +42,7 @@ type provider struct {
 }
 
 // NewManager is the Discovery Manager constructor.
-func NewManager(ctx context.Context, logger log.Logger, options ...func(*Manager)) *Manager {
+func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Registerer, metrics *discovery.Metrics, options ...func(*Manager)) *Manager {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -96,6 +54,8 @@ func NewManager(ctx context.Context, logger log.Logger, options ...func(*Manager
 		ctx:            ctx,
 		updatert:       5 * time.Second,
 		triggerSend:    make(chan struct{}, 1),
+		registerer:     registerer,
+		metrics:        metrics,
 	}
 	for _, option := range options {
 		option(mgr)
@@ -135,6 +95,11 @@ type Manager struct {
 
 	// The triggerSend channel signals to the manager that new updates have been received from providers.
 	triggerSend chan struct{}
+
+	// A registerer for all service discovery metrics.
+	registerer prometheus.Registerer
+
+	metrics *discovery.Metrics
 }
 
 // Run starts the background processing.
@@ -157,7 +122,7 @@ func (m *Manager) ApplyConfig(cfg map[string]discovery.Configs) error {
 
 	for pk := range m.targets {
 		if _, ok := cfg[pk.setName]; !ok {
-			discoveredTargets.DeleteLabelValues(m.name, pk.setName)
+			m.metrics.DiscoveredTargets.DeleteLabelValues(m.name, pk.setName)
 		}
 	}
 	m.cancelDiscoverers()
@@ -168,9 +133,9 @@ func (m *Manager) ApplyConfig(cfg map[string]discovery.Configs) error {
 	failedCount := 0
 	for name, scfg := range cfg {
 		failedCount += m.registerProviders(scfg, name)
-		discoveredTargets.WithLabelValues(m.name, name).Set(0)
+		m.metrics.DiscoveredTargets.WithLabelValues(m.name, name).Set(0)
 	}
-	failedConfigs.WithLabelValues(m.name).Set(float64(failedCount))
+	m.metrics.FailedConfigs.WithLabelValues(m.name).Set(float64(failedCount))
 
 	for _, prov := range m.providers {
 		m.startProvider(m.ctx, prov)
@@ -207,7 +172,7 @@ func (m *Manager) updater(ctx context.Context, p *provider, updates chan []*targ
 		case <-ctx.Done():
 			return
 		case tgs, ok := <-updates:
-			receivedUpdates.WithLabelValues(m.name).Inc()
+			m.metrics.ReceivedUpdates.WithLabelValues(m.name).Inc()
 			if !ok {
 				level.Debug(m.logger).Log("msg", "Discoverer channel closed", "provider", p.name)
 				return
@@ -236,11 +201,11 @@ func (m *Manager) sender() {
 		case <-ticker.C: // Some discoverers send updates too often so we throttle these with the ticker.
 			select {
 			case <-m.triggerSend:
-				sentUpdates.WithLabelValues(m.name).Inc()
+				m.metrics.SentUpdates.WithLabelValues(m.name).Inc()
 				select {
 				case m.syncCh <- m.allGroups():
 				default:
-					delayedUpdates.WithLabelValues(m.name).Inc()
+					m.metrics.DelayedUpdates.WithLabelValues(m.name).Inc()
 					level.Debug(m.logger).Log("msg", "Discovery receiver's channel was full so will retry the next cycle")
 					select {
 					case m.triggerSend <- struct{}{}:
@@ -288,7 +253,7 @@ func (m *Manager) allGroups() map[string][]*targetgroup.Group {
 		}
 	}
 	for setName, v := range n {
-		discoveredTargets.WithLabelValues(m.name, setName).Set(float64(v))
+		m.metrics.DiscoveredTargets.WithLabelValues(m.name, setName).Set(float64(v))
 	}
 	return tSets
 }
@@ -309,7 +274,8 @@ func (m *Manager) registerProviders(cfgs discovery.Configs, setName string) int
 		}
 		typ := cfg.Name()
 		d, err := cfg.NewDiscoverer(discovery.DiscovererOptions{
-			Logger: log.With(m.logger, "discovery", typ, "config", setName),
+			Logger:     log.With(m.logger, "discovery", typ, "config", setName),
+			Registerer: m.registerer,
 		})
 		if err != nil {
 			level.Error(m.logger).Log("msg", "Cannot create service discovery", "err", err, "type", typ, "config", setName)
diff --git a/discovery/legacymanager/manager_test.go b/discovery/legacymanager/manager_test.go
index 13b84e6e3..dccb687c2 100644
--- a/discovery/legacymanager/manager_test.go
+++ b/discovery/legacymanager/manager_test.go
@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	client_testutil "github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
@@ -35,6 +36,12 @@ func TestMain(m *testing.M) {
 	testutil.TolerantVerifyLeak(m)
 }
 
+func newTestDiscoveryMetrics(t *testing.T) *discovery.Metrics {
+	metrics, err := discovery.NewMetrics(prometheus.NewRegistry())
+	require.NoError(t, err)
+	return metrics
+}
+
 // TestTargetUpdatesOrder checks that the target updates are received in the expected order.
 func TestTargetUpdatesOrder(t *testing.T) {
 	// The order by which the updates are send is determined by the interval passed to the mock discovery adapter
@@ -664,7 +671,7 @@ func TestTargetUpdatesOrder(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel()
 
-			discoveryManager := NewManager(ctx, log.NewNopLogger())
+			discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
 			discoveryManager.updatert = 100 * time.Millisecond
 
 			var totalUpdatesCount int
@@ -746,7 +753,7 @@ func verifyPresence(t *testing.T, tSets map[poolKey]map[string]*targetgroup.Grou
 func TestTargetSetRecreatesTargetGroupsEveryRun(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -774,7 +781,7 @@ func TestTargetSetRecreatesTargetGroupsEveryRun(t *testing.T) {
 func TestDiscovererConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -798,7 +805,7 @@ func TestDiscovererConfigs(t *testing.T) {
 func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -837,7 +844,7 @@ func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 func TestIdenticalConfigurationsAreCoalesced(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, nil)
+	discoveryManager := NewManager(ctx, nil, prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -868,7 +875,7 @@ func TestApplyConfigDoesNotModifyStaticTargets(t *testing.T) {
 	}
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -893,7 +900,7 @@ func (e errorConfig) NewDiscoverer(discovery.DiscovererOptions) (discovery.Disco
 func TestGaugeFailedConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -907,7 +914,7 @@ func TestGaugeFailedConfigs(t *testing.T) {
 	discoveryManager.ApplyConfig(c)
 	<-discoveryManager.SyncCh()
 
-	failedCount := client_testutil.ToFloat64(failedConfigs)
+	failedCount := client_testutil.ToFloat64(discoveryManager.metrics.FailedConfigs)
 	if failedCount != 3 {
 		t.Fatalf("Expected to have 3 failed configs, got: %v", failedCount)
 	}
@@ -918,7 +925,7 @@ func TestGaugeFailedConfigs(t *testing.T) {
 	discoveryManager.ApplyConfig(c)
 	<-discoveryManager.SyncCh()
 
-	failedCount = client_testutil.ToFloat64(failedConfigs)
+	failedCount = client_testutil.ToFloat64(discoveryManager.metrics.FailedConfigs)
 	if failedCount != 0 {
 		t.Fatalf("Expected to get no failed config, got: %v", failedCount)
 	}
@@ -1049,7 +1056,7 @@ func TestCoordinationWithReceiver(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel()
 
-			mgr := NewManager(ctx, nil)
+			mgr := NewManager(ctx, nil, prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
 			mgr.updatert = updateDelay
 			go mgr.Run()
 
diff --git a/discovery/linode/linode.go b/discovery/linode/linode.go
index a5e047b94..38a5cdad4 100644
--- a/discovery/linode/linode.go
+++ b/discovery/linode/linode.go
@@ -67,24 +67,15 @@ const (
 )
 
 // DefaultSDConfig is the default Linode SD configuration.
-var (
-	DefaultSDConfig = SDConfig{
-		TagSeparator:     ",",
-		Port:             80,
-		RefreshInterval:  model.Duration(60 * time.Second),
-		HTTPClientConfig: config.DefaultHTTPClientConfig,
-	}
-
-	failuresCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_linode_failures_total",
-			Help: "Number of Linode service discovery refresh failures.",
-		})
-)
+var DefaultSDConfig = SDConfig{
+	TagSeparator:     ",",
+	Port:             80,
+	RefreshInterval:  model.Duration(60 * time.Second),
+	HTTPClientConfig: config.DefaultHTTPClientConfig,
+}
 
 func init() {
 	discovery.RegisterConfig(&SDConfig{})
-	prometheus.MustRegister(failuresCount)
 }
 
 // SDConfig is the configuration for Linode based service discovery.
@@ -101,7 +92,7 @@ func (*SDConfig) Name() string { return "linode" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -131,16 +122,22 @@ type Discovery struct {
 	pollCount            int
 	lastResults          []*targetgroup.Group
 	eventPollingEnabled  bool
+	failuresCount        prometheus.Counter
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	d := &Discovery{
 		port:                 conf.Port,
 		tagSeparator:         conf.TagSeparator,
 		pollCount:            0,
 		lastRefreshTimestamp: time.Now().UTC(),
 		eventPollingEnabled:  true,
+		failuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_linode_failures_total",
+				Help: "Number of Linode service discovery refresh failures.",
+			}),
 	}
 
 	rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "linode_sd")
@@ -158,10 +155,14 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 	d.client = &client
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"linode",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "linode",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+			Metrics:  []prometheus.Collector{d.failuresCount},
+		},
 	)
 	return d, nil
 }
@@ -222,14 +223,14 @@ func (d *Discovery) refreshData(ctx context.Context) ([]*targetgroup.Group, erro
 	// Gather all linode instances.
 	instances, err := d.client.ListInstances(ctx, &linodego.ListOptions{PageSize: 500})
 	if err != nil {
-		failuresCount.Inc()
+		d.failuresCount.Inc()
 		return nil, err
 	}
 
 	// Gather detailed IP address info for all IPs on all linode instances.
 	detailedIPs, err := d.client.ListIPAddresses(ctx, &linodego.ListOptions{PageSize: 500})
 	if err != nil {
-		failuresCount.Inc()
+		d.failuresCount.Inc()
 		return nil, err
 	}
 
diff --git a/discovery/linode/linode_test.go b/discovery/linode/linode_test.go
index db4ee9bf8..536b12090 100644
--- a/discovery/linode/linode_test.go
+++ b/discovery/linode/linode_test.go
@@ -20,6 +20,7 @@ import (
 	"testing"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
@@ -52,7 +53,7 @@ func TestLinodeSDRefresh(t *testing.T) {
 		Credentials: tokenID,
 		Type:        "Bearer",
 	}
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 	require.NoError(t, err)
 	endpoint, err := url.Parse(sdmock.Mock.Endpoint())
 	require.NoError(t, err)
diff --git a/discovery/manager.go b/discovery/manager.go
index 86439d2c9..6afa1b622 100644
--- a/discovery/manager.go
+++ b/discovery/manager.go
@@ -28,48 +28,6 @@ import (
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
-var (
-	failedConfigs = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "prometheus_sd_failed_configs",
-			Help: "Current number of service discovery configurations that failed to load.",
-		},
-		[]string{"name"},
-	)
-	discoveredTargets = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "prometheus_sd_discovered_targets",
-			Help: "Current number of discovered targets.",
-		},
-		[]string{"name", "config"},
-	)
-	receivedUpdates = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_received_updates_total",
-			Help: "Total number of update events received from the SD providers.",
-		},
-		[]string{"name"},
-	)
-	delayedUpdates = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_updates_delayed_total",
-			Help: "Total number of update events that couldn't be sent immediately.",
-		},
-		[]string{"name"},
-	)
-	sentUpdates = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_updates_total",
-			Help: "Total number of update events sent to the SD consumers.",
-		},
-		[]string{"name"},
-	)
-)
-
-func RegisterMetrics() {
-	prometheus.MustRegister(failedConfigs, discoveredTargets, receivedUpdates, delayedUpdates, sentUpdates)
-}
-
 type poolKey struct {
 	setName  string
 	provider string
@@ -107,7 +65,7 @@ func (p *Provider) Config() interface{} {
 }
 
 // NewManager is the Discovery Manager constructor.
-func NewManager(ctx context.Context, logger log.Logger, options ...func(*Manager)) *Manager {
+func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Registerer, metrics *Metrics, options ...func(*Manager)) *Manager {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -118,6 +76,8 @@ func NewManager(ctx context.Context, logger log.Logger, options ...func(*Manager
 		ctx:         ctx,
 		updatert:    5 * time.Second,
 		triggerSend: make(chan struct{}, 1),
+		registerer:  registerer,
+		metrics:     metrics,
 	}
 	for _, option := range options {
 		option(mgr)
@@ -170,6 +130,11 @@ type Manager struct {
 
 	// lastProvider counts providers registered during Manager's lifetime.
 	lastProvider uint
+
+	// A registerer for all service discovery metrics.
+	registerer prometheus.Registerer
+
+	metrics *Metrics
 }
 
 // Providers returns the currently configured SD providers.
@@ -200,7 +165,7 @@ func (m *Manager) ApplyConfig(cfg map[string]Configs) error {
 	for name, scfg := range cfg {
 		failedCount += m.registerProviders(scfg, name)
 	}
-	failedConfigs.WithLabelValues(m.name).Set(float64(failedCount))
+	m.metrics.FailedConfigs.WithLabelValues(m.name).Set(float64(failedCount))
 
 	var (
 		wg sync.WaitGroup
@@ -230,13 +195,13 @@ func (m *Manager) ApplyConfig(cfg map[string]Configs) error {
 			// Remove obsolete subs' targets.
 			if _, ok := prov.newSubs[s]; !ok {
 				delete(m.targets, poolKey{s, prov.name})
-				discoveredTargets.DeleteLabelValues(m.name, s)
+				m.metrics.DiscoveredTargets.DeleteLabelValues(m.name, s)
 			}
 		}
 		// Set metrics and targets for new subs.
 		for s := range prov.newSubs {
 			if _, ok := prov.subs[s]; !ok {
-				discoveredTargets.WithLabelValues(m.name, s).Set(0)
+				m.metrics.DiscoveredTargets.WithLabelValues(m.name, s).Set(0)
 			}
 			if l := len(refTargets); l > 0 {
 				m.targets[poolKey{s, prov.name}] = make(map[string]*targetgroup.Group, l)
@@ -316,7 +281,7 @@ func (m *Manager) updater(ctx context.Context, p *Provider, updates chan []*targ
 		case <-ctx.Done():
 			return
 		case tgs, ok := <-updates:
-			receivedUpdates.WithLabelValues(m.name).Inc()
+			m.metrics.ReceivedUpdates.WithLabelValues(m.name).Inc()
 			if !ok {
 				level.Debug(m.logger).Log("msg", "Discoverer channel closed", "provider", p.name)
 				// Wait for provider cancellation to ensure targets are cleaned up when expected.
@@ -349,11 +314,11 @@ func (m *Manager) sender() {
 		case <-ticker.C: // Some discoverers send updates too often, so we throttle these with the ticker.
 			select {
 			case <-m.triggerSend:
-				sentUpdates.WithLabelValues(m.name).Inc()
+				m.metrics.SentUpdates.WithLabelValues(m.name).Inc()
 				select {
 				case m.syncCh <- m.allGroups():
 				default:
-					delayedUpdates.WithLabelValues(m.name).Inc()
+					m.metrics.DelayedUpdates.WithLabelValues(m.name).Inc()
 					level.Debug(m.logger).Log("msg", "Discovery receiver's channel was full so will retry the next cycle")
 					select {
 					case m.triggerSend <- struct{}{}:
@@ -405,7 +370,7 @@ func (m *Manager) allGroups() map[string][]*targetgroup.Group {
 		}
 	}
 	for setName, v := range n {
-		discoveredTargets.WithLabelValues(m.name, setName).Set(float64(v))
+		m.metrics.DiscoveredTargets.WithLabelValues(m.name, setName).Set(float64(v))
 	}
 	return tSets
 }
@@ -428,6 +393,7 @@ func (m *Manager) registerProviders(cfgs Configs, setName string) int {
 		d, err := cfg.NewDiscoverer(DiscovererOptions{
 			Logger:            log.With(m.logger, "discovery", typ, "config", setName),
 			HTTPClientOptions: m.httpOpts,
+			Registerer:        m.registerer,
 		})
 		if err != nil {
 			level.Error(m.logger).Log("msg", "Cannot create service discovery", "err", err, "type", typ, "config", setName)
diff --git a/discovery/manager_test.go b/discovery/manager_test.go
index 796b01458..3cb277343 100644
--- a/discovery/manager_test.go
+++ b/discovery/manager_test.go
@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	client_testutil "github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
@@ -35,6 +36,12 @@ func TestMain(m *testing.M) {
 	testutil.TolerantVerifyLeak(m)
 }
 
+func newTestDiscoveryMetrics(t *testing.T) *Metrics {
+	metrics, err := NewMetrics(prometheus.NewRegistry())
+	require.NoError(t, err)
+	return metrics
+}
+
 // TestTargetUpdatesOrder checks that the target updates are received in the expected order.
 func TestTargetUpdatesOrder(t *testing.T) {
 	// The order by which the updates are send is determined by the interval passed to the mock discovery adapter
@@ -664,7 +671,7 @@ func TestTargetUpdatesOrder(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel()
 
-			discoveryManager := NewManager(ctx, log.NewNopLogger())
+			discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
 			discoveryManager.updatert = 100 * time.Millisecond
 
 			var totalUpdatesCount int
@@ -778,7 +785,7 @@ func pk(provider, setName string, n int) poolKey {
 func TestTargetSetTargetGroupsPresentOnConfigReload(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -810,7 +817,7 @@ func TestTargetSetTargetGroupsPresentOnConfigReload(t *testing.T) {
 func TestTargetSetTargetGroupsPresentOnConfigRename(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -845,7 +852,7 @@ func TestTargetSetTargetGroupsPresentOnConfigRename(t *testing.T) {
 func TestTargetSetTargetGroupsPresentOnConfigDuplicateAndDeleteOriginal(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -883,7 +890,7 @@ func TestTargetSetTargetGroupsPresentOnConfigDuplicateAndDeleteOriginal(t *testi
 func TestTargetSetTargetGroupsPresentOnConfigChange(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -944,7 +951,7 @@ func TestTargetSetTargetGroupsPresentOnConfigChange(t *testing.T) {
 func TestTargetSetRecreatesTargetGroupsOnConfigChange(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -983,7 +990,7 @@ func TestTargetSetRecreatesTargetGroupsOnConfigChange(t *testing.T) {
 func TestDiscovererConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -1015,7 +1022,7 @@ func TestDiscovererConfigs(t *testing.T) {
 func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -1062,7 +1069,7 @@ func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 func TestIdenticalConfigurationsAreCoalesced(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, nil)
+	discoveryManager := NewManager(ctx, nil, prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -1098,7 +1105,7 @@ func TestApplyConfigDoesNotModifyStaticTargets(t *testing.T) {
 	}
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -1144,7 +1151,7 @@ func (s lockStaticDiscoverer) Run(ctx context.Context, up chan<- []*targetgroup.
 func TestGaugeFailedConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -1158,7 +1165,7 @@ func TestGaugeFailedConfigs(t *testing.T) {
 	discoveryManager.ApplyConfig(c)
 	<-discoveryManager.SyncCh()
 
-	failedCount := client_testutil.ToFloat64(failedConfigs)
+	failedCount := client_testutil.ToFloat64(discoveryManager.metrics.FailedConfigs)
 	if failedCount != 3 {
 		t.Fatalf("Expected to have 3 failed configs, got: %v", failedCount)
 	}
@@ -1169,7 +1176,7 @@ func TestGaugeFailedConfigs(t *testing.T) {
 	discoveryManager.ApplyConfig(c)
 	<-discoveryManager.SyncCh()
 
-	failedCount = client_testutil.ToFloat64(failedConfigs)
+	failedCount = client_testutil.ToFloat64(discoveryManager.metrics.FailedConfigs)
 	if failedCount != 0 {
 		t.Fatalf("Expected to get no failed config, got: %v", failedCount)
 	}
@@ -1300,7 +1307,7 @@ func TestCoordinationWithReceiver(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel()
 
-			mgr := NewManager(ctx, nil)
+			mgr := NewManager(ctx, nil, prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
 			mgr.updatert = updateDelay
 			go mgr.Run()
 
@@ -1392,10 +1399,10 @@ func (o onceProvider) Run(_ context.Context, ch chan<- []*targetgroup.Group) {
 
 // TestTargetSetTargetGroupsUpdateDuringApplyConfig is used to detect races when
 // ApplyConfig happens at the same time as targets update.
-func TestTargetSetTargetGroupsUpdateDuringApplyConfig(*testing.T) {
+func TestTargetSetTargetGroupsUpdateDuringApplyConfig(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger())
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
diff --git a/discovery/marathon/marathon.go b/discovery/marathon/marathon.go
index 27947fa8a..a6a6252fd 100644
--- a/discovery/marathon/marathon.go
+++ b/discovery/marathon/marathon.go
@@ -28,6 +28,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -83,7 +84,7 @@ func (*SDConfig) Name() string { return "marathon" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(*c, opts.Logger)
+	return NewDiscovery(*c, opts.Logger, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -132,7 +133,7 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Marathon Discovery.
-func NewDiscovery(conf SDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "marathon_sd")
 	if err != nil {
 		return nil, err
@@ -154,10 +155,13 @@ func NewDiscovery(conf SDConfig, logger log.Logger) (*Discovery, error) {
 		appsClient: fetchApps,
 	}
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"marathon",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "marathon",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+		},
 	)
 	return d, nil
 }
diff --git a/discovery/marathon/marathon_test.go b/discovery/marathon/marathon_test.go
index 258e3c8dd..a1ddce930 100644
--- a/discovery/marathon/marathon_test.go
+++ b/discovery/marathon/marathon_test.go
@@ -21,6 +21,7 @@ import (
 	"net/http/httptest"
 	"testing"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 
 	"github.com/prometheus/prometheus/discovery/targetgroup"
@@ -36,7 +37,7 @@ func testConfig() SDConfig {
 }
 
 func testUpdateServices(client appListClient) ([]*targetgroup.Group, error) {
-	md, err := NewDiscovery(testConfig(), nil)
+	md, err := NewDiscovery(testConfig(), nil, prometheus.NewRegistry())
 	if err != nil {
 		return nil, err
 	}
@@ -129,7 +130,7 @@ func TestMarathonSDSendGroup(t *testing.T) {
 }
 
 func TestMarathonSDRemoveApp(t *testing.T) {
-	md, err := NewDiscovery(testConfig(), nil)
+	md, err := NewDiscovery(testConfig(), nil, prometheus.NewRegistry())
 	if err != nil {
 		t.Fatalf("%s", err)
 	}
diff --git a/discovery/metrics.go b/discovery/metrics.go
new file mode 100644
index 000000000..04f54b542
--- /dev/null
+++ b/discovery/metrics.go
@@ -0,0 +1,103 @@
+// Copyright 2016 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package discovery
+
+import (
+	"fmt"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	clientGoRequestMetrics  = &clientGoRequestMetricAdapter{}
+	clientGoWorkloadMetrics = &clientGoWorkqueueMetricsProvider{}
+)
+
+func init() {
+	clientGoRequestMetrics.RegisterWithK8sGoClient()
+	clientGoWorkloadMetrics.RegisterWithK8sGoClient()
+}
+
+// Metrics to be used with a discovery manager.
+type Metrics struct {
+	FailedConfigs     *prometheus.GaugeVec
+	DiscoveredTargets *prometheus.GaugeVec
+	ReceivedUpdates   *prometheus.CounterVec
+	DelayedUpdates    *prometheus.CounterVec
+	SentUpdates       *prometheus.CounterVec
+}
+
+func NewMetrics(registerer prometheus.Registerer) (*Metrics, error) {
+	m := &Metrics{}
+
+	m.FailedConfigs = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "prometheus_sd_failed_configs",
+			Help: "Current number of service discovery configurations that failed to load.",
+		},
+		[]string{"name"},
+	)
+
+	m.DiscoveredTargets = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "prometheus_sd_discovered_targets",
+			Help: "Current number of discovered targets.",
+		},
+		[]string{"name", "config"},
+	)
+
+	m.ReceivedUpdates = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "prometheus_sd_received_updates_total",
+			Help: "Total number of update events received from the SD providers.",
+		},
+		[]string{"name"},
+	)
+
+	m.DelayedUpdates = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "prometheus_sd_updates_delayed_total",
+			Help: "Total number of update events that couldn't be sent immediately.",
+		},
+		[]string{"name"},
+	)
+
+	m.SentUpdates = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "prometheus_sd_updates_total",
+			Help: "Total number of update events sent to the SD consumers.",
+		},
+		[]string{"name"},
+	)
+
+	metrics := append(
+		[]prometheus.Collector{
+			m.FailedConfigs,
+			m.DiscoveredTargets,
+			m.ReceivedUpdates,
+			m.DelayedUpdates,
+			m.SentUpdates,
+		},
+		clientGoMetrics()...,
+	)
+
+	for _, collector := range metrics {
+		err := registerer.Register(collector)
+		if err != nil {
+			return nil, fmt.Errorf("failed to register discovery manager metrics: %w", err)
+		}
+	}
+
+	return m, nil
+}
diff --git a/discovery/kubernetes/client_metrics.go b/discovery/metrics_k8s_client.go
similarity index 85%
rename from discovery/kubernetes/client_metrics.go
rename to discovery/metrics_k8s_client.go
index 7b097b14a..4f161bc3e 100644
--- a/discovery/kubernetes/client_metrics.go
+++ b/discovery/metrics_k8s_client.go
@@ -11,7 +11,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package kubernetes
+package discovery
 
 import (
 	"context"
@@ -23,13 +23,22 @@ import (
 	"k8s.io/client-go/util/workqueue"
 )
 
-const workqueueMetricsNamespace = metricsNamespace + "_workqueue"
+// This file registers metrics used by the Kubernetes Go client (k8s.io/client-go).
+// Unfortunately, k8s.io/client-go metrics are global.
+// If we instantiate multiple k8s SD instances, their k8s/client-go metrics will overlap.
+// To prevent us from displaying misleading metrics, we register k8s.io/client-go metrics
+// outside of the Kubernetes SD.
+
+const (
+	KubernetesMetricsNamespace = "prometheus_sd_kubernetes"
+	workqueueMetricsNamespace  = KubernetesMetricsNamespace + "_workqueue"
+)
 
 var (
 	// Metrics for client-go's HTTP requests.
 	clientGoRequestResultMetricVec = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
+			Namespace: KubernetesMetricsNamespace,
 			Name:      "http_request_total",
 			Help:      "Total number of HTTP requests to the Kubernetes API by status code.",
 		},
@@ -37,7 +46,7 @@ var (
 	)
 	clientGoRequestLatencyMetricVec = prometheus.NewSummaryVec(
 		prometheus.SummaryOpts{
-			Namespace:  metricsNamespace,
+			Namespace:  KubernetesMetricsNamespace,
 			Name:       "http_request_duration_seconds",
 			Help:       "Summary of latencies for HTTP requests to the Kubernetes API by endpoint.",
 			Objectives: map[float64]float64{},
@@ -109,17 +118,28 @@ func (noopMetric) Set(float64)     {}
 // Definition of client-go metrics adapters for HTTP requests observation.
 type clientGoRequestMetricAdapter struct{}
 
-func (f *clientGoRequestMetricAdapter) Register(registerer prometheus.Registerer) {
+// Returns all of the Prometheus metrics derived from k8s.io/client-go.
+// This may be used tu register and unregister the metrics.
+func clientGoMetrics() []prometheus.Collector {
+	return []prometheus.Collector{
+		clientGoRequestResultMetricVec,
+		clientGoRequestLatencyMetricVec,
+		clientGoWorkqueueDepthMetricVec,
+		clientGoWorkqueueAddsMetricVec,
+		clientGoWorkqueueLatencyMetricVec,
+		clientGoWorkqueueUnfinishedWorkSecondsMetricVec,
+		clientGoWorkqueueLongestRunningProcessorMetricVec,
+		clientGoWorkqueueWorkDurationMetricVec,
+	}
+}
+
+func (f *clientGoRequestMetricAdapter) RegisterWithK8sGoClient() {
 	metrics.Register(
 		metrics.RegisterOpts{
 			RequestLatency: f,
 			RequestResult:  f,
 		},
 	)
-	registerer.MustRegister(
-		clientGoRequestResultMetricVec,
-		clientGoRequestLatencyMetricVec,
-	)
 }
 
 func (clientGoRequestMetricAdapter) Increment(_ context.Context, code, _, _ string) {
@@ -133,16 +153,8 @@ func (clientGoRequestMetricAdapter) Observe(_ context.Context, _ string, u url.U
 // Definition of client-go workqueue metrics provider definition.
 type clientGoWorkqueueMetricsProvider struct{}
 
-func (f *clientGoWorkqueueMetricsProvider) Register(registerer prometheus.Registerer) {
+func (f *clientGoWorkqueueMetricsProvider) RegisterWithK8sGoClient() {
 	workqueue.SetProvider(f)
-	registerer.MustRegister(
-		clientGoWorkqueueDepthMetricVec,
-		clientGoWorkqueueAddsMetricVec,
-		clientGoWorkqueueLatencyMetricVec,
-		clientGoWorkqueueWorkDurationMetricVec,
-		clientGoWorkqueueUnfinishedWorkSecondsMetricVec,
-		clientGoWorkqueueLongestRunningProcessorMetricVec,
-	)
 }
 
 func (f *clientGoWorkqueueMetricsProvider) NewDepthMetric(name string) workqueue.GaugeMetric {
diff --git a/discovery/moby/docker.go b/discovery/moby/docker.go
index 162833ece..a13bb8704 100644
--- a/discovery/moby/docker.go
+++ b/discovery/moby/docker.go
@@ -26,6 +26,7 @@ import (
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -80,7 +81,7 @@ func (*DockerSDConfig) Name() string { return "docker" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *DockerSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDockerDiscovery(c, opts.Logger)
+	return NewDockerDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -114,7 +115,7 @@ type DockerDiscovery struct {
 }
 
 // NewDockerDiscovery returns a new DockerDiscovery which periodically refreshes its targets.
-func NewDockerDiscovery(conf *DockerSDConfig, logger log.Logger) (*DockerDiscovery, error) {
+func NewDockerDiscovery(conf *DockerSDConfig, logger log.Logger, reg prometheus.Registerer) (*DockerDiscovery, error) {
 	var err error
 
 	d := &DockerDiscovery{
@@ -165,10 +166,13 @@ func NewDockerDiscovery(conf *DockerSDConfig, logger log.Logger) (*DockerDiscove
 	}
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"docker",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "docker",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+		},
 	)
 	return d, nil
 }
diff --git a/discovery/moby/docker_test.go b/discovery/moby/docker_test.go
index f80c53b61..1a87ad2a1 100644
--- a/discovery/moby/docker_test.go
+++ b/discovery/moby/docker_test.go
@@ -19,6 +19,7 @@ import (
 	"testing"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
@@ -37,7 +38,7 @@ host: %s
 	var cfg DockerSDConfig
 	require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg))
 
-	d, err := NewDockerDiscovery(&cfg, log.NewNopLogger())
+	d, err := NewDockerDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 	require.NoError(t, err)
 
 	ctx := context.Background()
diff --git a/discovery/moby/dockerswarm.go b/discovery/moby/dockerswarm.go
index 371f9d5ed..bd87fea5a 100644
--- a/discovery/moby/dockerswarm.go
+++ b/discovery/moby/dockerswarm.go
@@ -23,6 +23,7 @@ import (
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/version"
@@ -74,7 +75,7 @@ func (*DockerSwarmSDConfig) Name() string { return "dockerswarm" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *DockerSwarmSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -117,7 +118,7 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *DockerSwarmSDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf *DockerSwarmSDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	var err error
 
 	d := &Discovery{
@@ -168,10 +169,13 @@ func NewDiscovery(conf *DockerSwarmSDConfig, logger log.Logger) (*Discovery, err
 	}
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"dockerswarm",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "dockerswarm",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+		},
 	)
 	return d, nil
 }
diff --git a/discovery/moby/nodes_test.go b/discovery/moby/nodes_test.go
index 2bc383374..512ff7049 100644
--- a/discovery/moby/nodes_test.go
+++ b/discovery/moby/nodes_test.go
@@ -19,6 +19,7 @@ import (
 	"testing"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
@@ -38,7 +39,7 @@ host: %s
 	var cfg DockerSwarmSDConfig
 	require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg))
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 	require.NoError(t, err)
 
 	ctx := context.Background()
diff --git a/discovery/moby/services_test.go b/discovery/moby/services_test.go
index 81c8d31f1..816586dd7 100644
--- a/discovery/moby/services_test.go
+++ b/discovery/moby/services_test.go
@@ -19,6 +19,7 @@ import (
 	"testing"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
@@ -38,7 +39,7 @@ host: %s
 	var cfg DockerSwarmSDConfig
 	require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg))
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 	require.NoError(t, err)
 
 	ctx := context.Background()
@@ -332,7 +333,7 @@ filters:
 	var cfg DockerSwarmSDConfig
 	require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg))
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 	require.NoError(t, err)
 
 	ctx := context.Background()
diff --git a/discovery/moby/tasks_test.go b/discovery/moby/tasks_test.go
index eed5f2924..764fda343 100644
--- a/discovery/moby/tasks_test.go
+++ b/discovery/moby/tasks_test.go
@@ -19,6 +19,7 @@ import (
 	"testing"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
@@ -38,7 +39,7 @@ host: %s
 	var cfg DockerSwarmSDConfig
 	require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg))
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 	require.NoError(t, err)
 
 	ctx := context.Background()
diff --git a/discovery/nomad/nomad.go b/discovery/nomad/nomad.go
index 7013f0737..3fdcf714e 100644
--- a/discovery/nomad/nomad.go
+++ b/discovery/nomad/nomad.go
@@ -49,27 +49,18 @@ const (
 )
 
 // DefaultSDConfig is the default nomad SD configuration.
-var (
-	DefaultSDConfig = SDConfig{
-		AllowStale:       true,
-		HTTPClientConfig: config.DefaultHTTPClientConfig,
-		Namespace:        "default",
-		RefreshInterval:  model.Duration(60 * time.Second),
-		Region:           "global",
-		Server:           "http://localhost:4646",
-		TagSeparator:     ",",
-	}
-
-	failuresCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_nomad_failures_total",
-			Help: "Number of nomad service discovery refresh failures.",
-		})
-)
+var DefaultSDConfig = SDConfig{
+	AllowStale:       true,
+	HTTPClientConfig: config.DefaultHTTPClientConfig,
+	Namespace:        "default",
+	RefreshInterval:  model.Duration(60 * time.Second),
+	Region:           "global",
+	Server:           "http://localhost:4646",
+	TagSeparator:     ",",
+}
 
 func init() {
 	discovery.RegisterConfig(&SDConfig{})
-	prometheus.MustRegister(failuresCount)
 }
 
 // SDConfig is the configuration for nomad based service discovery.
@@ -88,7 +79,7 @@ func (*SDConfig) Name() string { return "nomad" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -121,10 +112,11 @@ type Discovery struct {
 	region          string
 	server          string
 	tagSeparator    string
+	failuresCount   prometheus.Counter
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	d := &Discovery{
 		allowStale:      conf.AllowStale,
 		namespace:       conf.Namespace,
@@ -132,6 +124,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 		region:          conf.Region,
 		server:          conf.Server,
 		tagSeparator:    conf.TagSeparator,
+		failuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_nomad_failures_total",
+				Help: "Number of nomad service discovery refresh failures.",
+			}),
 	}
 
 	HTTPClient, err := config.NewClientFromConfig(conf.HTTPClientConfig, "nomad_sd")
@@ -153,10 +150,14 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 	d.client = client
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"nomad",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "nomad",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+			Metrics:  []prometheus.Collector{d.failuresCount},
+		},
 	)
 	return d, nil
 }
@@ -167,7 +168,7 @@ func (d *Discovery) refresh(context.Context) ([]*targetgroup.Group, error) {
 	}
 	stubs, _, err := d.client.Services().List(opts)
 	if err != nil {
-		failuresCount.Inc()
+		d.failuresCount.Inc()
 		return nil, err
 	}
 
@@ -179,7 +180,7 @@ func (d *Discovery) refresh(context.Context) ([]*targetgroup.Group, error) {
 		for _, service := range stub.Services {
 			instances, _, err := d.client.Services().Get(service.ServiceName, opts)
 			if err != nil {
-				failuresCount.Inc()
+				d.failuresCount.Inc()
 				return nil, fmt.Errorf("failed to fetch services: %w", err)
 			}
 
diff --git a/discovery/nomad/nomad_test.go b/discovery/nomad/nomad_test.go
index f9490f476..ca67a877e 100644
--- a/discovery/nomad/nomad_test.go
+++ b/discovery/nomad/nomad_test.go
@@ -22,6 +22,7 @@ import (
 	"testing"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 )
@@ -127,7 +128,7 @@ func TestConfiguredService(t *testing.T) {
 	conf := &SDConfig{
 		Server: "http://localhost:4646",
 	}
-	_, err := NewDiscovery(conf, nil)
+	_, err := NewDiscovery(conf, nil, prometheus.NewRegistry())
 	require.NoError(t, err)
 }
 
@@ -141,7 +142,7 @@ func TestNomadSDRefresh(t *testing.T) {
 
 	cfg := DefaultSDConfig
 	cfg.Server = endpoint.String()
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 	require.NoError(t, err)
 
 	tgs, err := d.refresh(context.Background())
diff --git a/discovery/openstack/openstack.go b/discovery/openstack/openstack.go
index 92c83a4cf..9544a7c0f 100644
--- a/discovery/openstack/openstack.go
+++ b/discovery/openstack/openstack.go
@@ -24,6 +24,7 @@ import (
 	"github.com/gophercloud/gophercloud"
 	"github.com/gophercloud/gophercloud/openstack"
 	"github.com/mwitkow/go-conntrack"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -70,7 +71,7 @@ func (*SDConfig) Name() string { return "openstack" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -134,16 +135,19 @@ type refresher interface {
 }
 
 // NewDiscovery returns a new OpenStack Discoverer which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, l log.Logger) (*refresh.Discovery, error) {
+func NewDiscovery(conf *SDConfig, l log.Logger, reg prometheus.Registerer) (*refresh.Discovery, error) {
 	r, err := newRefresher(conf, l)
 	if err != nil {
 		return nil, err
 	}
 	return refresh.NewDiscovery(
-		l,
-		"openstack",
-		time.Duration(conf.RefreshInterval),
-		r.refresh,
+		refresh.Options{
+			Logger:   l,
+			Mech:     "openstack",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: r.refresh,
+			Registry: reg,
+		},
 	), nil
 }
 
diff --git a/discovery/ovhcloud/ovhcloud.go b/discovery/ovhcloud/ovhcloud.go
index 535ade4df..eca284a85 100644
--- a/discovery/ovhcloud/ovhcloud.go
+++ b/discovery/ovhcloud/ovhcloud.go
@@ -22,6 +22,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/ovh/go-ovh/ovh"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -93,7 +94,7 @@ func createClient(config *SDConfig) (*ovh.Client, error) {
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(options discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, options.Logger)
+	return NewDiscovery(c, options.Logger, options.Registerer)
 }
 
 func init() {
@@ -140,16 +141,19 @@ func newRefresher(conf *SDConfig, logger log.Logger) (refresher, error) {
 }
 
 // NewDiscovery returns a new OVHcloud Discoverer which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*refresh.Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*refresh.Discovery, error) {
 	r, err := newRefresher(conf, logger)
 	if err != nil {
 		return nil, err
 	}
 
 	return refresh.NewDiscovery(
-		logger,
-		"ovhcloud",
-		time.Duration(conf.RefreshInterval),
-		r.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "ovhcloud",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: r.refresh,
+			Registry: reg,
+		},
 	), nil
 }
diff --git a/discovery/ovhcloud/ovhcloud_test.go b/discovery/ovhcloud/ovhcloud_test.go
index efcd95bb0..9bd9ea954 100644
--- a/discovery/ovhcloud/ovhcloud_test.go
+++ b/discovery/ovhcloud/ovhcloud_test.go
@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
@@ -122,7 +123,8 @@ func TestDiscoverer(t *testing.T) {
 	conf, _ := getMockConf("vps")
 	logger := testutil.NewLogger(t)
 	_, err := conf.NewDiscoverer(discovery.DiscovererOptions{
-		Logger: logger,
+		Logger:     logger,
+		Registerer: prometheus.NewRegistry(),
 	})
 
 	require.NoError(t, err)
diff --git a/discovery/puppetdb/puppetdb.go b/discovery/puppetdb/puppetdb.go
index 9484a0aa6..616f2c61e 100644
--- a/discovery/puppetdb/puppetdb.go
+++ b/discovery/puppetdb/puppetdb.go
@@ -29,6 +29,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/grafana/regexp"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/version"
@@ -83,7 +84,7 @@ func (*SDConfig) Name() string { return "puppetdb" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -130,7 +131,7 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new PuppetDB discovery for the given config.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -156,10 +157,13 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 	}
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"http",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "http",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+		},
 	)
 	return d, nil
 }
diff --git a/discovery/puppetdb/puppetdb_test.go b/discovery/puppetdb/puppetdb_test.go
index 236efec16..edd9b9d04 100644
--- a/discovery/puppetdb/puppetdb_test.go
+++ b/discovery/puppetdb/puppetdb_test.go
@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
@@ -62,7 +63,7 @@ func TestPuppetSlashInURL(t *testing.T) {
 			Port:             80,
 			RefreshInterval:  model.Duration(30 * time.Second),
 		}
-		d, err := NewDiscovery(&cfg, log.NewNopLogger())
+		d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 		require.NoError(t, err)
 		require.Equal(t, apiURL, d.url)
 	}
@@ -79,7 +80,7 @@ func TestPuppetDBRefresh(t *testing.T) {
 		RefreshInterval:  model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 	require.NoError(t, err)
 
 	ctx := context.Background()
@@ -120,7 +121,7 @@ func TestPuppetDBRefreshWithParameters(t *testing.T) {
 		RefreshInterval:   model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 	require.NoError(t, err)
 
 	ctx := context.Background()
@@ -172,7 +173,7 @@ func TestPuppetDBInvalidCode(t *testing.T) {
 		RefreshInterval:  model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 	require.NoError(t, err)
 
 	ctx := context.Background()
@@ -193,7 +194,7 @@ func TestPuppetDBInvalidFormat(t *testing.T) {
 		RefreshInterval:  model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 	require.NoError(t, err)
 
 	ctx := context.Background()
diff --git a/discovery/refresh/refresh.go b/discovery/refresh/refresh.go
index 919567a53..0b0e5a921 100644
--- a/discovery/refresh/refresh.go
+++ b/discovery/refresh/refresh.go
@@ -22,29 +22,17 @@ import (
 	"github.com/go-kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
-var (
-	failuresCount = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "prometheus_sd_refresh_failures_total",
-			Help: "Number of refresh failures for the given SD mechanism.",
-		},
-		[]string{"mechanism"},
-	)
-	duration = prometheus.NewSummaryVec(
-		prometheus.SummaryOpts{
-			Name:       "prometheus_sd_refresh_duration_seconds",
-			Help:       "The duration of a refresh in seconds for the given SD mechanism.",
-			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
-		},
-		[]string{"mechanism"},
-	)
-)
-
-func init() {
-	prometheus.MustRegister(duration, failuresCount)
+type Options struct {
+	Logger   log.Logger
+	Mech     string
+	Interval time.Duration
+	RefreshF func(ctx context.Context) ([]*targetgroup.Group, error)
+	Registry prometheus.Registerer
+	Metrics  []prometheus.Collector
 }
 
 // Discovery implements the Discoverer interface.
@@ -54,25 +42,62 @@ type Discovery struct {
 	refreshf func(ctx context.Context) ([]*targetgroup.Group, error)
 
 	failures prometheus.Counter
-	duration prometheus.Observer
+	duration prometheus.Summary
+
+	metricRegisterer discovery.MetricRegisterer
 }
 
 // NewDiscovery returns a Discoverer function that calls a refresh() function at every interval.
-func NewDiscovery(l log.Logger, mech string, interval time.Duration, refreshf func(ctx context.Context) ([]*targetgroup.Group, error)) *Discovery {
-	if l == nil {
-		l = log.NewNopLogger()
+func NewDiscovery(opts Options) *Discovery {
+	var logger log.Logger
+	if opts.Logger == nil {
+		logger = log.NewNopLogger()
+	} else {
+		logger = opts.Logger
 	}
-	return &Discovery{
-		logger:   l,
-		interval: interval,
-		refreshf: refreshf,
-		failures: failuresCount.WithLabelValues(mech),
-		duration: duration.WithLabelValues(mech),
+
+	d := Discovery{
+		logger:   logger,
+		interval: opts.Interval,
+		refreshf: opts.RefreshF,
+		failures: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_refresh_failures_total",
+				Help: "Number of refresh failures for the given SD mechanism.",
+				ConstLabels: prometheus.Labels{
+					"mechanism": opts.Mech,
+				},
+			}),
+		duration: prometheus.NewSummary(
+			prometheus.SummaryOpts{
+				Name:       "prometheus_sd_refresh_duration_seconds",
+				Help:       "The duration of a refresh in seconds for the given SD mechanism.",
+				Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+				ConstLabels: prometheus.Labels{
+					"mechanism": opts.Mech,
+				},
+			}),
 	}
+
+	metrics := []prometheus.Collector{d.failures, d.duration}
+	if opts.Metrics != nil {
+		metrics = append(metrics, opts.Metrics...)
+	}
+
+	d.metricRegisterer = discovery.NewMetricRegisterer(opts.Registry, metrics)
+
+	return &d
 }
 
 // Run implements the Discoverer interface.
 func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
+	err := d.metricRegisterer.RegisterMetrics()
+	if err != nil {
+		level.Error(d.logger).Log("msg", "Unable to register metrics", "err", err.Error())
+		return
+	}
+	defer d.metricRegisterer.UnregisterMetrics()
+
 	// Get an initial set right away.
 	tgs, err := d.refresh(ctx)
 	if err != nil {
diff --git a/discovery/refresh/refresh_test.go b/discovery/refresh/refresh_test.go
index 6decef19f..12e7ab3be 100644
--- a/discovery/refresh/refresh_test.go
+++ b/discovery/refresh/refresh_test.go
@@ -19,6 +19,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
@@ -65,7 +66,15 @@ func TestRefresh(t *testing.T) {
 		return nil, fmt.Errorf("some error")
 	}
 	interval := time.Millisecond
-	d := NewDiscovery(nil, "test", interval, refresh)
+	d := NewDiscovery(
+		Options{
+			Logger:   nil,
+			Mech:     "test",
+			Interval: interval,
+			RefreshF: refresh,
+			Registry: prometheus.NewRegistry(),
+		},
+	)
 
 	ch := make(chan []*targetgroup.Group)
 	ctx, cancel := context.WithCancel(context.Background())
diff --git a/discovery/scaleway/scaleway.go b/discovery/scaleway/scaleway.go
index 90091b317..86527b34e 100644
--- a/discovery/scaleway/scaleway.go
+++ b/discovery/scaleway/scaleway.go
@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/scaleway/scaleway-sdk-go/scw"
@@ -160,7 +161,7 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 }
 
 func (c SDConfig) NewDiscoverer(options discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(&c, options.Logger)
+	return NewDiscovery(&c, options.Logger, options.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -177,17 +178,20 @@ func init() {
 // the Discoverer interface.
 type Discovery struct{}
 
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*refresh.Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*refresh.Discovery, error) {
 	r, err := newRefresher(conf)
 	if err != nil {
 		return nil, err
 	}
 
 	return refresh.NewDiscovery(
-		logger,
-		"scaleway",
-		time.Duration(conf.RefreshInterval),
-		r.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "scaleway",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: r.refresh,
+			Registry: reg,
+		},
 	), nil
 }
 
diff --git a/discovery/triton/triton.go b/discovery/triton/triton.go
index c83f3b34a..4839827ad 100644
--- a/discovery/triton/triton.go
+++ b/discovery/triton/triton.go
@@ -26,6 +26,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/mwitkow/go-conntrack"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -74,7 +75,7 @@ func (*SDConfig) Name() string { return "triton" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return New(opts.Logger, c)
+	return New(opts.Logger, c, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -138,7 +139,7 @@ type Discovery struct {
 }
 
 // New returns a new Discovery which periodically refreshes its targets.
-func New(logger log.Logger, conf *SDConfig) (*Discovery, error) {
+func New(logger log.Logger, conf *SDConfig, reg prometheus.Registerer) (*Discovery, error) {
 	tls, err := config.NewTLSConfig(&conf.TLSConfig)
 	if err != nil {
 		return nil, err
@@ -159,10 +160,13 @@ func New(logger log.Logger, conf *SDConfig) (*Discovery, error) {
 		sdConfig: conf,
 	}
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"triton",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "triton",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+		},
 	)
 	return d, nil
 }
diff --git a/discovery/triton/triton_test.go b/discovery/triton/triton_test.go
index 0ed9daa68..fa51a2e47 100644
--- a/discovery/triton/triton_test.go
+++ b/discovery/triton/triton_test.go
@@ -24,6 +24,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
@@ -79,7 +80,7 @@ var (
 )
 
 func newTritonDiscovery(c SDConfig) (*Discovery, error) {
-	return New(nil, &c)
+	return New(nil, &c, prometheus.NewRegistry())
 }
 
 func TestTritonSDNew(t *testing.T) {
diff --git a/discovery/util.go b/discovery/util.go
new file mode 100644
index 000000000..83cc640dd
--- /dev/null
+++ b/discovery/util.go
@@ -0,0 +1,72 @@
+// Copyright 2020 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package discovery
+
+import (
+	"fmt"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// A utility to be used by implementations of discovery.Discoverer
+// which need to manage the lifetime of their metrics.
+type MetricRegisterer interface {
+	RegisterMetrics() error
+	UnregisterMetrics()
+}
+
+// metricRegistererImpl is an implementation of MetricRegisterer.
+type metricRegistererImpl struct {
+	reg     prometheus.Registerer
+	metrics []prometheus.Collector
+}
+
+var _ MetricRegisterer = &metricRegistererImpl{}
+
+// Creates an instance of a MetricRegisterer.
+// Typically called inside the implementation of the NewDiscoverer() method.
+func NewMetricRegisterer(reg prometheus.Registerer, metrics []prometheus.Collector) MetricRegisterer {
+	return &metricRegistererImpl{
+		reg:     reg,
+		metrics: metrics,
+	}
+}
+
+// RegisterMetrics registers the metrics with a Prometheus registerer.
+// If any metric fails to register, it will unregister all metrics that
+// were registered so far, and return an error.
+// Typically called at the start of the SD's Run() method.
+func (rh *metricRegistererImpl) RegisterMetrics() error {
+	for _, collector := range rh.metrics {
+		err := rh.reg.Register(collector)
+		if err != nil {
+			// Unregister all metrics that were registered so far.
+			// This is so that if RegisterMetrics() gets called again,
+			// there will not be an error due to a duplicate registration.
+			rh.UnregisterMetrics()
+
+			return fmt.Errorf("failed to register metric: %w", err)
+		}
+	}
+	return nil
+}
+
+// UnregisterMetrics unregisters the metrics from the same Prometheus
+// registerer which was used to register them.
+// Typically called at the end of the SD's Run() method by a defer statement.
+func (rh *metricRegistererImpl) UnregisterMetrics() {
+	for _, collector := range rh.metrics {
+		rh.reg.Unregister(collector)
+	}
+}
diff --git a/discovery/uyuni/uyuni.go b/discovery/uyuni/uyuni.go
index bc33d28cb..744f3f96c 100644
--- a/discovery/uyuni/uyuni.go
+++ b/discovery/uyuni/uyuni.go
@@ -25,6 +25,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/kolo/xmlrpc"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -115,7 +116,7 @@ func (*SDConfig) Name() string { return "uyuni" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -203,7 +204,7 @@ func getEndpointInfoForSystems(
 }
 
 // NewDiscovery returns a uyuni discovery for the given configuration.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	apiURL, err := url.Parse(conf.Server)
 	if err != nil {
 		return nil, err
@@ -227,10 +228,13 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 	}
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"uyuni",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "uyuni",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+		},
 	)
 	return d, nil
 }
diff --git a/discovery/uyuni/uyuni_test.go b/discovery/uyuni/uyuni_test.go
index 9c910a3a3..fd03c88f1 100644
--- a/discovery/uyuni/uyuni_test.go
+++ b/discovery/uyuni/uyuni_test.go
@@ -23,6 +23,8 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/prometheus/client_golang/prometheus"
+
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -35,7 +37,7 @@ func testUpdateServices(respHandler http.HandlerFunc) ([]*targetgroup.Group, err
 		Server: ts.URL,
 	}
 
-	md, err := NewDiscovery(&conf, nil)
+	md, err := NewDiscovery(&conf, nil, prometheus.NewRegistry())
 	if err != nil {
 		return nil, err
 	}
@@ -108,7 +110,7 @@ func TestUyuniSDSkipLogin(t *testing.T) {
 		Server: ts.URL,
 	}
 
-	md, err := NewDiscovery(&conf, nil)
+	md, err := NewDiscovery(&conf, nil, prometheus.NewRegistry())
 	if err != nil {
 		t.Error(err)
 	}
diff --git a/discovery/vultr/vultr.go b/discovery/vultr/vultr.go
index 42881d3c1..129800048 100644
--- a/discovery/vultr/vultr.go
+++ b/discovery/vultr/vultr.go
@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/version"
@@ -78,7 +79,7 @@ func (*SDConfig) Name() string { return "vultr" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger)
+	return NewDiscovery(c, opts.Logger, opts.Registerer)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -106,7 +107,7 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
 	d := &Discovery{
 		port: conf.Port,
 	}
@@ -128,10 +129,13 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
 	}
 
 	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"vultr",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
+		refresh.Options{
+			Logger:   logger,
+			Mech:     "vultr",
+			Interval: time.Duration(conf.RefreshInterval),
+			RefreshF: d.refresh,
+			Registry: reg,
+		},
 	)
 	return d, nil
 }
diff --git a/discovery/vultr/vultr_test.go b/discovery/vultr/vultr_test.go
index 0977238e0..c50b11d2d 100644
--- a/discovery/vultr/vultr_test.go
+++ b/discovery/vultr/vultr_test.go
@@ -20,6 +20,7 @@ import (
 	"testing"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 )
@@ -46,7 +47,7 @@ func TestVultrSDRefresh(t *testing.T) {
 
 	cfg := DefaultSDConfig
 	cfg.HTTPClientConfig.BearerToken = APIKey
-	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
 	require.NoError(t, err)
 	endpoint, err := url.Parse(sdMock.Mock.Endpoint())
 	require.NoError(t, err)
diff --git a/discovery/xds/kuma.go b/discovery/xds/kuma.go
index bc88ba554..5ac4a42a8 100644
--- a/discovery/xds/kuma.go
+++ b/discovery/xds/kuma.go
@@ -30,35 +30,12 @@ import (
 	"github.com/prometheus/prometheus/util/strutil"
 )
 
-var (
-	// DefaultKumaSDConfig is the default Kuma MADS SD configuration.
-	DefaultKumaSDConfig = KumaSDConfig{
-		HTTPClientConfig: config.DefaultHTTPClientConfig,
-		RefreshInterval:  model.Duration(15 * time.Second),
-		FetchTimeout:     model.Duration(2 * time.Minute),
-	}
-
-	kumaFetchFailuresCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Namespace: namespace,
-			Name:      "sd_kuma_fetch_failures_total",
-			Help:      "The number of Kuma MADS fetch call failures.",
-		})
-	kumaFetchSkipUpdateCount = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Namespace: namespace,
-			Name:      "sd_kuma_fetch_skipped_updates_total",
-			Help:      "The number of Kuma MADS fetch calls that result in no updates to the targets.",
-		})
-	kumaFetchDuration = prometheus.NewSummary(
-		prometheus.SummaryOpts{
-			Namespace:  namespace,
-			Name:       "sd_kuma_fetch_duration_seconds",
-			Help:       "The duration of a Kuma MADS fetch call.",
-			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
-		},
-	)
-)
+// DefaultKumaSDConfig is the default Kuma MADS SD configuration.
+var DefaultKumaSDConfig = KumaSDConfig{
+	HTTPClientConfig: config.DefaultHTTPClientConfig,
+	RefreshInterval:  model.Duration(15 * time.Second),
+	FetchTimeout:     model.Duration(2 * time.Minute),
+}
 
 const (
 	// kumaMetaLabelPrefix is the meta prefix used for all kuma meta labels.
@@ -120,7 +97,7 @@ func (c *KumaSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discover
 		logger = log.NewNopLogger()
 	}
 
-	return NewKumaHTTPDiscovery(c, logger)
+	return NewKumaHTTPDiscovery(c, logger, opts.Registerer)
 }
 
 func convertKumaV1MonitoringAssignment(assignment *MonitoringAssignment) []model.LabelSet {
@@ -176,7 +153,7 @@ func kumaMadsV1ResourceParser(resources []*anypb.Any, typeURL string) ([]model.L
 	return targets, nil
 }
 
-func NewKumaHTTPDiscovery(conf *KumaSDConfig, logger log.Logger) (discovery.Discoverer, error) {
+func NewKumaHTTPDiscovery(conf *KumaSDConfig, logger log.Logger, reg prometheus.Registerer) (discovery.Discoverer, error) {
 	// Default to "prometheus" if hostname is unavailable.
 	clientID, err := osutil.GetFQDN()
 	if err != nil {
@@ -203,15 +180,41 @@ func NewKumaHTTPDiscovery(conf *KumaSDConfig, logger log.Logger) (discovery.Disc
 	}
 
 	d := &fetchDiscovery{
-		client:               client,
-		logger:               logger,
-		refreshInterval:      time.Duration(conf.RefreshInterval),
-		source:               "kuma",
-		parseResources:       kumaMadsV1ResourceParser,
-		fetchFailuresCount:   kumaFetchFailuresCount,
-		fetchSkipUpdateCount: kumaFetchSkipUpdateCount,
-		fetchDuration:        kumaFetchDuration,
+		client:          client,
+		logger:          logger,
+		refreshInterval: time.Duration(conf.RefreshInterval),
+		source:          "kuma",
+		parseResources:  kumaMadsV1ResourceParser,
+		fetchFailuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      "sd_kuma_fetch_failures_total",
+				Help:      "The number of Kuma MADS fetch call failures.",
+			}),
+		fetchSkipUpdateCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      "sd_kuma_fetch_skipped_updates_total",
+				Help:      "The number of Kuma MADS fetch calls that result in no updates to the targets.",
+			}),
+		fetchDuration: prometheus.NewSummary(
+			prometheus.SummaryOpts{
+				Namespace:  namespace,
+				Name:       "sd_kuma_fetch_duration_seconds",
+				Help:       "The duration of a Kuma MADS fetch call.",
+				Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+			},
+		),
 	}
 
+	d.metricRegisterer = discovery.NewMetricRegisterer(
+		reg,
+		[]prometheus.Collector{
+			d.fetchFailuresCount,
+			d.fetchSkipUpdateCount,
+			d.fetchDuration,
+		},
+	)
+
 	return d, nil
 }
diff --git a/discovery/xds/kuma_test.go b/discovery/xds/kuma_test.go
index 581be9fb1..0626f82a0 100644
--- a/discovery/xds/kuma_test.go
+++ b/discovery/xds/kuma_test.go
@@ -21,6 +21,7 @@ import (
 	"time"
 
 	v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/protobuf/proto"
@@ -107,7 +108,7 @@ func getKumaMadsV1DiscoveryResponse(resources ...*MonitoringAssignment) (*v3.Dis
 }
 
 func newKumaTestHTTPDiscovery(c KumaSDConfig) (*fetchDiscovery, error) {
-	kd, err := NewKumaHTTPDiscovery(&c, nopLogger)
+	kd, err := NewKumaHTTPDiscovery(&c, nopLogger, prometheus.NewRegistry())
 	if err != nil {
 		return nil, err
 	}
diff --git a/discovery/xds/xds.go b/discovery/xds/xds.go
index 48bdbab02..a04fe6862 100644
--- a/discovery/xds/xds.go
+++ b/discovery/xds/xds.go
@@ -69,9 +69,6 @@ func init() {
 	// Register top-level SD Configs.
 	discovery.RegisterConfig(&KumaSDConfig{})
 
-	// Register metrics.
-	prometheus.MustRegister(kumaFetchDuration, kumaFetchSkipUpdateCount, kumaFetchFailuresCount)
-
 	// Register protobuf types that need to be marshalled/ unmarshalled.
 	mustRegisterMessage(protoTypes, (&v3.DiscoveryRequest{}).ProtoReflect().Type())
 	mustRegisterMessage(protoTypes, (&v3.DiscoveryResponse{}).ProtoReflect().Type())
@@ -109,12 +106,20 @@ type fetchDiscovery struct {
 	parseResources resourceParser
 	logger         log.Logger
 
-	fetchDuration        prometheus.Observer
+	fetchDuration        prometheus.Summary
 	fetchSkipUpdateCount prometheus.Counter
 	fetchFailuresCount   prometheus.Counter
+
+	metricRegisterer discovery.MetricRegisterer
 }
 
 func (d *fetchDiscovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
+	err := d.metricRegisterer.RegisterMetrics()
+	if err != nil {
+		level.Error(d.logger).Log("msg", "Unable to register metrics", "err", err.Error())
+		return
+	}
+	defer d.metricRegisterer.UnregisterMetrics()
 	defer d.client.Close()
 
 	ticker := time.NewTicker(d.refreshInterval)
diff --git a/documentation/examples/custom-sd/adapter-usage/main.go b/documentation/examples/custom-sd/adapter-usage/main.go
index ae656db19..b712749df 100644
--- a/documentation/examples/custom-sd/adapter-usage/main.go
+++ b/documentation/examples/custom-sd/adapter-usage/main.go
@@ -28,8 +28,10 @@ import (
 	"github.com/alecthomas/kingpin/v2"
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 
+	prom_discovery "github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 	"github.com/prometheus/prometheus/documentation/examples/custom-sd/adapter"
 	"github.com/prometheus/prometheus/util/strutil"
@@ -268,7 +270,13 @@ func main() {
 	if err != nil {
 		fmt.Println("err: ", err)
 	}
-	sdAdapter := adapter.NewAdapter(ctx, *outputFile, "exampleSD", disc, logger)
+
+	discoveryMetrics, err := prom_discovery.NewMetrics(prometheus.DefaultRegisterer)
+	if err != nil {
+		level.Error(logger).Log("msg", "failed to create discovery metrics", "err", err)
+		os.Exit(1)
+	}
+	sdAdapter := adapter.NewAdapter(ctx, *outputFile, "exampleSD", disc, logger, discoveryMetrics)
 	sdAdapter.Run()
 
 	<-ctx.Done()
diff --git a/documentation/examples/custom-sd/adapter/adapter.go b/documentation/examples/custom-sd/adapter/adapter.go
index 57c32ce49..8aedf0084 100644
--- a/documentation/examples/custom-sd/adapter/adapter.go
+++ b/documentation/examples/custom-sd/adapter/adapter.go
@@ -25,6 +25,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 
 	"github.com/prometheus/prometheus/discovery"
@@ -162,12 +163,12 @@ func (a *Adapter) Run() {
 }
 
 // NewAdapter creates a new instance of Adapter.
-func NewAdapter(ctx context.Context, file, name string, d discovery.Discoverer, logger log.Logger) *Adapter {
+func NewAdapter(ctx context.Context, file, name string, d discovery.Discoverer, logger log.Logger, metrics *discovery.Metrics) *Adapter {
 	return &Adapter{
 		ctx:     ctx,
 		disc:    d,
 		groups:  make(map[string]*customSD),
-		manager: discovery.NewManager(ctx, logger),
+		manager: discovery.NewManager(ctx, logger, prometheus.NewRegistry(), metrics),
 		output:  file,
 		name:    name,
 		logger:  logger,
diff --git a/documentation/examples/custom-sd/adapter/adapter_test.go b/documentation/examples/custom-sd/adapter/adapter_test.go
index eaf34c667..14cae47b4 100644
--- a/documentation/examples/custom-sd/adapter/adapter_test.go
+++ b/documentation/examples/custom-sd/adapter/adapter_test.go
@@ -18,9 +18,11 @@ import (
 	"os"
 	"testing"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -226,6 +228,8 @@ func TestWriteOutput(t *testing.T) {
 	require.NoError(t, err)
 	defer os.Remove(tmpfile.Name())
 	tmpfile.Close()
-	adapter := NewAdapter(ctx, tmpfile.Name(), "test_sd", nil, nil)
+	metrics, err := discovery.NewMetrics(prometheus.NewRegistry())
+	require.NoError(t, err)
+	adapter := NewAdapter(ctx, tmpfile.Name(), "test_sd", nil, nil, metrics)
 	require.NoError(t, adapter.writeOutput())
 }

From 6a5306a53c437b9e5817423b8431937167b410d0 Mon Sep 17 00:00:00 2001
From: Paulin Todev <paulin.todev@gmail.com>
Date: Wed, 15 Nov 2023 17:13:40 +0000
Subject: [PATCH 019/115] Use const labels for Discovery Manager metrics.

Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
---
 cmd/prometheus/main.go                        | 64 ++++++++++++------
 discovery/legacymanager/manager.go            | 25 ++++---
 discovery/legacymanager/manager_test.go       | 30 +++++----
 discovery/manager.go                          | 25 ++++---
 discovery/manager_test.go                     | 45 +++++++------
 discovery/metrics.go                          | 66 +++++++++----------
 discovery/metrics_k8s_client.go               | 11 ++++
 .../examples/custom-sd/adapter-usage/main.go  |  5 +-
 .../examples/custom-sd/adapter/adapter.go     |  4 +-
 .../custom-sd/adapter/adapter_test.go         |  5 +-
 10 files changed, 168 insertions(+), 112 deletions(-)

diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go
index 24fc0f8e4..f30fbecd9 100644
--- a/cmd/prometheus/main.go
+++ b/cmd/prometheus/main.go
@@ -620,30 +620,56 @@ func main() {
 		discoveryManagerNotify  discoveryManager
 	)
 
-	// Register the metrics used by both "scrape" and "notify" discovery managers.
-	// The same metrics are used for both discovery managers. Hence the registration
-	// needs to be done here, outside the NewManager() calls, to avoid duplicate
-	// metric registrations.
-	discoveryMetrics, err := discovery.NewMetrics(prometheus.DefaultRegisterer)
+	// Kubernetes client metrics are used by Kubernetes SD.
+	// They are registered here in the main function, because SD mechanisms
+	// can only register metrics specific to a SD instance.
+	// Kubernetes client metrics are the same for the whole process -
+	// they are not specific to an SD instance.
+	err = discovery.RegisterK8sClientMetricsWithPrometheus(prometheus.DefaultRegisterer)
 	if err != nil {
-		level.Error(logger).Log("msg", "failed to create discovery metrics", "err", err)
+		level.Error(logger).Log("msg", "failed to register Kubernetes client metrics", "err", err)
 		os.Exit(1)
 	}
 	if cfg.enableNewSDManager {
-		discoveryManagerScrape = discovery.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, discoveryMetrics, discovery.Name("scrape"))
-		discoveryManagerNotify = discovery.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, discoveryMetrics, discovery.Name("notify"))
-	} else {
-		discoveryManagerScrape = legacymanager.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, discoveryMetrics, legacymanager.Name("scrape"))
-		discoveryManagerNotify = legacymanager.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, discoveryMetrics, legacymanager.Name("notify"))
-	}
+		{
+			discMgr := discovery.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, discovery.Name("scrape"))
+			if discMgr == nil {
+				level.Error(logger).Log("msg", "failed to create a discovery manager scrape")
+				os.Exit(1)
+			} else {
+				discoveryManagerScrape = discMgr
+			}
+		}
 
-	if discoveryManagerScrape == nil {
-		level.Error(logger).Log("msg", "failed to create a discovery manager scrape")
-		os.Exit(1)
-	}
-	if discoveryManagerNotify == nil {
-		level.Error(logger).Log("msg", "failed to create a discovery manager notify")
-		os.Exit(1)
+		{
+			discMgr := discovery.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, discovery.Name("notify"))
+			if discMgr == nil {
+				level.Error(logger).Log("msg", "failed to create a discovery manager notify")
+				os.Exit(1)
+			} else {
+				discoveryManagerNotify = discMgr
+			}
+		}
+	} else {
+		{
+			discMgr := legacymanager.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, legacymanager.Name("scrape"))
+			if discMgr == nil {
+				level.Error(logger).Log("msg", "failed to create a discovery manager scrape")
+				os.Exit(1)
+			} else {
+				discoveryManagerScrape = discMgr
+			}
+		}
+
+		{
+			discMgr := legacymanager.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, legacymanager.Name("notify"))
+			if discMgr == nil {
+				level.Error(logger).Log("msg", "failed to create a discovery manager notify")
+				os.Exit(1)
+			} else {
+				discoveryManagerNotify = discMgr
+			}
+		}
 	}
 
 	scrapeManager, err := scrape.NewManager(
diff --git a/discovery/legacymanager/manager.go b/discovery/legacymanager/manager.go
index 101012daf..9c80f305a 100644
--- a/discovery/legacymanager/manager.go
+++ b/discovery/legacymanager/manager.go
@@ -42,7 +42,7 @@ type provider struct {
 }
 
 // NewManager is the Discovery Manager constructor.
-func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Registerer, metrics *discovery.Metrics, options ...func(*Manager)) *Manager {
+func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Registerer, options ...func(*Manager)) *Manager {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -55,11 +55,20 @@ func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Re
 		updatert:       5 * time.Second,
 		triggerSend:    make(chan struct{}, 1),
 		registerer:     registerer,
-		metrics:        metrics,
 	}
 	for _, option := range options {
 		option(mgr)
 	}
+
+	// Register the metrics.
+	// We have to do this after setting all options, so that the name of the Manager is set.
+	if metrics, err := discovery.NewMetrics(registerer, mgr.name); err == nil {
+		mgr.metrics = metrics
+	} else {
+		level.Error(logger).Log("msg", "Failed to create discovery manager metrics", "manager", mgr.name, "err", err)
+		return nil
+	}
+
 	return mgr
 }
 
@@ -133,9 +142,9 @@ func (m *Manager) ApplyConfig(cfg map[string]discovery.Configs) error {
 	failedCount := 0
 	for name, scfg := range cfg {
 		failedCount += m.registerProviders(scfg, name)
-		m.metrics.DiscoveredTargets.WithLabelValues(m.name, name).Set(0)
+		m.metrics.DiscoveredTargets.WithLabelValues(name).Set(0)
 	}
-	m.metrics.FailedConfigs.WithLabelValues(m.name).Set(float64(failedCount))
+	m.metrics.FailedConfigs.Set(float64(failedCount))
 
 	for _, prov := range m.providers {
 		m.startProvider(m.ctx, prov)
@@ -172,7 +181,7 @@ func (m *Manager) updater(ctx context.Context, p *provider, updates chan []*targ
 		case <-ctx.Done():
 			return
 		case tgs, ok := <-updates:
-			m.metrics.ReceivedUpdates.WithLabelValues(m.name).Inc()
+			m.metrics.ReceivedUpdates.Inc()
 			if !ok {
 				level.Debug(m.logger).Log("msg", "Discoverer channel closed", "provider", p.name)
 				return
@@ -201,11 +210,11 @@ func (m *Manager) sender() {
 		case <-ticker.C: // Some discoverers send updates too often so we throttle these with the ticker.
 			select {
 			case <-m.triggerSend:
-				m.metrics.SentUpdates.WithLabelValues(m.name).Inc()
+				m.metrics.SentUpdates.Inc()
 				select {
 				case m.syncCh <- m.allGroups():
 				default:
-					m.metrics.DelayedUpdates.WithLabelValues(m.name).Inc()
+					m.metrics.DelayedUpdates.Inc()
 					level.Debug(m.logger).Log("msg", "Discovery receiver's channel was full so will retry the next cycle")
 					select {
 					case m.triggerSend <- struct{}{}:
@@ -253,7 +262,7 @@ func (m *Manager) allGroups() map[string][]*targetgroup.Group {
 		}
 	}
 	for setName, v := range n {
-		m.metrics.DiscoveredTargets.WithLabelValues(m.name, setName).Set(float64(v))
+		m.metrics.DiscoveredTargets.WithLabelValues(setName).Set(float64(v))
 	}
 	return tSets
 }
diff --git a/discovery/legacymanager/manager_test.go b/discovery/legacymanager/manager_test.go
index dccb687c2..7a2e8feea 100644
--- a/discovery/legacymanager/manager_test.go
+++ b/discovery/legacymanager/manager_test.go
@@ -36,12 +36,6 @@ func TestMain(m *testing.M) {
 	testutil.TolerantVerifyLeak(m)
 }
 
-func newTestDiscoveryMetrics(t *testing.T) *discovery.Metrics {
-	metrics, err := discovery.NewMetrics(prometheus.NewRegistry())
-	require.NoError(t, err)
-	return metrics
-}
-
 // TestTargetUpdatesOrder checks that the target updates are received in the expected order.
 func TestTargetUpdatesOrder(t *testing.T) {
 	// The order by which the updates are send is determined by the interval passed to the mock discovery adapter
@@ -671,7 +665,8 @@ func TestTargetUpdatesOrder(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel()
 
-			discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
+			discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+			require.NotNil(t, discoveryManager)
 			discoveryManager.updatert = 100 * time.Millisecond
 
 			var totalUpdatesCount int
@@ -753,7 +748,8 @@ func verifyPresence(t *testing.T, tSets map[poolKey]map[string]*targetgroup.Grou
 func TestTargetSetRecreatesTargetGroupsEveryRun(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -781,7 +777,8 @@ func TestTargetSetRecreatesTargetGroupsEveryRun(t *testing.T) {
 func TestDiscovererConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -805,7 +802,8 @@ func TestDiscovererConfigs(t *testing.T) {
 func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -844,7 +842,8 @@ func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 func TestIdenticalConfigurationsAreCoalesced(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, nil, prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
+	discoveryManager := NewManager(ctx, nil, prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -875,7 +874,8 @@ func TestApplyConfigDoesNotModifyStaticTargets(t *testing.T) {
 	}
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -900,7 +900,8 @@ func (e errorConfig) NewDiscoverer(discovery.DiscovererOptions) (discovery.Disco
 func TestGaugeFailedConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -1056,7 +1057,8 @@ func TestCoordinationWithReceiver(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel()
 
-			mgr := NewManager(ctx, nil, prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
+			mgr := NewManager(ctx, nil, prometheus.NewRegistry())
+			require.NotNil(t, mgr)
 			mgr.updatert = updateDelay
 			go mgr.Run()
 
diff --git a/discovery/manager.go b/discovery/manager.go
index 6afa1b622..67e326c41 100644
--- a/discovery/manager.go
+++ b/discovery/manager.go
@@ -65,7 +65,7 @@ func (p *Provider) Config() interface{} {
 }
 
 // NewManager is the Discovery Manager constructor.
-func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Registerer, metrics *Metrics, options ...func(*Manager)) *Manager {
+func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Registerer, options ...func(*Manager)) *Manager {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -77,11 +77,20 @@ func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Re
 		updatert:    5 * time.Second,
 		triggerSend: make(chan struct{}, 1),
 		registerer:  registerer,
-		metrics:     metrics,
 	}
 	for _, option := range options {
 		option(mgr)
 	}
+
+	// Register the metrics.
+	// We have to do this after setting all options, so that the name of the Manager is set.
+	if metrics, err := NewMetrics(registerer, mgr.name); err == nil {
+		mgr.metrics = metrics
+	} else {
+		level.Error(logger).Log("msg", "Failed to create discovery manager metrics", "manager", mgr.name, "err", err)
+		return nil
+	}
+
 	return mgr
 }
 
@@ -165,7 +174,7 @@ func (m *Manager) ApplyConfig(cfg map[string]Configs) error {
 	for name, scfg := range cfg {
 		failedCount += m.registerProviders(scfg, name)
 	}
-	m.metrics.FailedConfigs.WithLabelValues(m.name).Set(float64(failedCount))
+	m.metrics.FailedConfigs.Set(float64(failedCount))
 
 	var (
 		wg sync.WaitGroup
@@ -201,7 +210,7 @@ func (m *Manager) ApplyConfig(cfg map[string]Configs) error {
 		// Set metrics and targets for new subs.
 		for s := range prov.newSubs {
 			if _, ok := prov.subs[s]; !ok {
-				m.metrics.DiscoveredTargets.WithLabelValues(m.name, s).Set(0)
+				m.metrics.DiscoveredTargets.WithLabelValues(s).Set(0)
 			}
 			if l := len(refTargets); l > 0 {
 				m.targets[poolKey{s, prov.name}] = make(map[string]*targetgroup.Group, l)
@@ -281,7 +290,7 @@ func (m *Manager) updater(ctx context.Context, p *Provider, updates chan []*targ
 		case <-ctx.Done():
 			return
 		case tgs, ok := <-updates:
-			m.metrics.ReceivedUpdates.WithLabelValues(m.name).Inc()
+			m.metrics.ReceivedUpdates.Inc()
 			if !ok {
 				level.Debug(m.logger).Log("msg", "Discoverer channel closed", "provider", p.name)
 				// Wait for provider cancellation to ensure targets are cleaned up when expected.
@@ -314,11 +323,11 @@ func (m *Manager) sender() {
 		case <-ticker.C: // Some discoverers send updates too often, so we throttle these with the ticker.
 			select {
 			case <-m.triggerSend:
-				m.metrics.SentUpdates.WithLabelValues(m.name).Inc()
+				m.metrics.SentUpdates.Inc()
 				select {
 				case m.syncCh <- m.allGroups():
 				default:
-					m.metrics.DelayedUpdates.WithLabelValues(m.name).Inc()
+					m.metrics.DelayedUpdates.Inc()
 					level.Debug(m.logger).Log("msg", "Discovery receiver's channel was full so will retry the next cycle")
 					select {
 					case m.triggerSend <- struct{}{}:
@@ -370,7 +379,7 @@ func (m *Manager) allGroups() map[string][]*targetgroup.Group {
 		}
 	}
 	for setName, v := range n {
-		m.metrics.DiscoveredTargets.WithLabelValues(m.name, setName).Set(float64(v))
+		m.metrics.DiscoveredTargets.WithLabelValues(setName).Set(float64(v))
 	}
 	return tSets
 }
diff --git a/discovery/manager_test.go b/discovery/manager_test.go
index 3cb277343..f22de75a4 100644
--- a/discovery/manager_test.go
+++ b/discovery/manager_test.go
@@ -36,12 +36,6 @@ func TestMain(m *testing.M) {
 	testutil.TolerantVerifyLeak(m)
 }
 
-func newTestDiscoveryMetrics(t *testing.T) *Metrics {
-	metrics, err := NewMetrics(prometheus.NewRegistry())
-	require.NoError(t, err)
-	return metrics
-}
-
 // TestTargetUpdatesOrder checks that the target updates are received in the expected order.
 func TestTargetUpdatesOrder(t *testing.T) {
 	// The order by which the updates are send is determined by the interval passed to the mock discovery adapter
@@ -671,7 +665,8 @@ func TestTargetUpdatesOrder(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel()
 
-			discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
+			discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+			require.NotNil(t, discoveryManager)
 			discoveryManager.updatert = 100 * time.Millisecond
 
 			var totalUpdatesCount int
@@ -785,7 +780,8 @@ func pk(provider, setName string, n int) poolKey {
 func TestTargetSetTargetGroupsPresentOnConfigReload(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -817,7 +813,8 @@ func TestTargetSetTargetGroupsPresentOnConfigReload(t *testing.T) {
 func TestTargetSetTargetGroupsPresentOnConfigRename(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -852,7 +849,8 @@ func TestTargetSetTargetGroupsPresentOnConfigRename(t *testing.T) {
 func TestTargetSetTargetGroupsPresentOnConfigDuplicateAndDeleteOriginal(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -890,7 +888,8 @@ func TestTargetSetTargetGroupsPresentOnConfigDuplicateAndDeleteOriginal(t *testi
 func TestTargetSetTargetGroupsPresentOnConfigChange(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -951,7 +950,8 @@ func TestTargetSetTargetGroupsPresentOnConfigChange(t *testing.T) {
 func TestTargetSetRecreatesTargetGroupsOnConfigChange(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -990,7 +990,8 @@ func TestTargetSetRecreatesTargetGroupsOnConfigChange(t *testing.T) {
 func TestDiscovererConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -1022,7 +1023,8 @@ func TestDiscovererConfigs(t *testing.T) {
 func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -1069,7 +1071,8 @@ func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 func TestIdenticalConfigurationsAreCoalesced(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, nil, prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
+	discoveryManager := NewManager(ctx, nil, prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -1105,7 +1108,8 @@ func TestApplyConfigDoesNotModifyStaticTargets(t *testing.T) {
 	}
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -1151,7 +1155,8 @@ func (s lockStaticDiscoverer) Run(ctx context.Context, up chan<- []*targetgroup.
 func TestGaugeFailedConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
@@ -1307,7 +1312,8 @@ func TestCoordinationWithReceiver(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel()
 
-			mgr := NewManager(ctx, nil, prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
+			mgr := NewManager(ctx, nil, prometheus.NewRegistry())
+			require.NotNil(t, mgr)
 			mgr.updatert = updateDelay
 			go mgr.Run()
 
@@ -1402,7 +1408,8 @@ func (o onceProvider) Run(_ context.Context, ch chan<- []*targetgroup.Group) {
 func TestTargetSetTargetGroupsUpdateDuringApplyConfig(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry(), newTestDiscoveryMetrics(t))
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
 
diff --git a/discovery/metrics.go b/discovery/metrics.go
index 04f54b542..6a6060395 100644
--- a/discovery/metrics.go
+++ b/discovery/metrics.go
@@ -31,66 +31,64 @@ func init() {
 
 // Metrics to be used with a discovery manager.
 type Metrics struct {
-	FailedConfigs     *prometheus.GaugeVec
+	FailedConfigs     prometheus.Gauge
 	DiscoveredTargets *prometheus.GaugeVec
-	ReceivedUpdates   *prometheus.CounterVec
-	DelayedUpdates    *prometheus.CounterVec
-	SentUpdates       *prometheus.CounterVec
+	ReceivedUpdates   prometheus.Counter
+	DelayedUpdates    prometheus.Counter
+	SentUpdates       prometheus.Counter
 }
 
-func NewMetrics(registerer prometheus.Registerer) (*Metrics, error) {
+func NewMetrics(registerer prometheus.Registerer, sdManagerName string) (*Metrics, error) {
 	m := &Metrics{}
 
-	m.FailedConfigs = prometheus.NewGaugeVec(
+	m.FailedConfigs = prometheus.NewGauge(
 		prometheus.GaugeOpts{
-			Name: "prometheus_sd_failed_configs",
-			Help: "Current number of service discovery configurations that failed to load.",
+			Name:        "prometheus_sd_failed_configs",
+			Help:        "Current number of service discovery configurations that failed to load.",
+			ConstLabels: prometheus.Labels{"name": sdManagerName},
 		},
-		[]string{"name"},
 	)
 
 	m.DiscoveredTargets = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "prometheus_sd_discovered_targets",
-			Help: "Current number of discovered targets.",
+			Name:        "prometheus_sd_discovered_targets",
+			Help:        "Current number of discovered targets.",
+			ConstLabels: prometheus.Labels{"name": sdManagerName},
 		},
-		[]string{"name", "config"},
+		[]string{"config"},
 	)
 
-	m.ReceivedUpdates = prometheus.NewCounterVec(
+	m.ReceivedUpdates = prometheus.NewCounter(
 		prometheus.CounterOpts{
-			Name: "prometheus_sd_received_updates_total",
-			Help: "Total number of update events received from the SD providers.",
+			Name:        "prometheus_sd_received_updates_total",
+			Help:        "Total number of update events received from the SD providers.",
+			ConstLabels: prometheus.Labels{"name": sdManagerName},
 		},
-		[]string{"name"},
 	)
 
-	m.DelayedUpdates = prometheus.NewCounterVec(
+	m.DelayedUpdates = prometheus.NewCounter(
 		prometheus.CounterOpts{
-			Name: "prometheus_sd_updates_delayed_total",
-			Help: "Total number of update events that couldn't be sent immediately.",
+			Name:        "prometheus_sd_updates_delayed_total",
+			Help:        "Total number of update events that couldn't be sent immediately.",
+			ConstLabels: prometheus.Labels{"name": sdManagerName},
 		},
-		[]string{"name"},
 	)
 
-	m.SentUpdates = prometheus.NewCounterVec(
+	m.SentUpdates = prometheus.NewCounter(
 		prometheus.CounterOpts{
-			Name: "prometheus_sd_updates_total",
-			Help: "Total number of update events sent to the SD consumers.",
+			Name:        "prometheus_sd_updates_total",
+			Help:        "Total number of update events sent to the SD consumers.",
+			ConstLabels: prometheus.Labels{"name": sdManagerName},
 		},
-		[]string{"name"},
 	)
 
-	metrics := append(
-		[]prometheus.Collector{
-			m.FailedConfigs,
-			m.DiscoveredTargets,
-			m.ReceivedUpdates,
-			m.DelayedUpdates,
-			m.SentUpdates,
-		},
-		clientGoMetrics()...,
-	)
+	metrics := []prometheus.Collector{
+		m.FailedConfigs,
+		m.DiscoveredTargets,
+		m.ReceivedUpdates,
+		m.DelayedUpdates,
+		m.SentUpdates,
+	}
 
 	for _, collector := range metrics {
 		err := registerer.Register(collector)
diff --git a/discovery/metrics_k8s_client.go b/discovery/metrics_k8s_client.go
index 4f161bc3e..f16245684 100644
--- a/discovery/metrics_k8s_client.go
+++ b/discovery/metrics_k8s_client.go
@@ -15,6 +15,7 @@ package discovery
 
 import (
 	"context"
+	"fmt"
 	"net/url"
 	"time"
 
@@ -133,6 +134,16 @@ func clientGoMetrics() []prometheus.Collector {
 	}
 }
 
+func RegisterK8sClientMetricsWithPrometheus(registerer prometheus.Registerer) error {
+	for _, collector := range clientGoMetrics() {
+		err := registerer.Register(collector)
+		if err != nil {
+			return fmt.Errorf("failed to register Kubernetes Go Client metrics: %w", err)
+		}
+	}
+	return nil
+}
+
 func (f *clientGoRequestMetricAdapter) RegisterWithK8sGoClient() {
 	metrics.Register(
 		metrics.RegisterOpts{
diff --git a/documentation/examples/custom-sd/adapter-usage/main.go b/documentation/examples/custom-sd/adapter-usage/main.go
index b712749df..f4bba7394 100644
--- a/documentation/examples/custom-sd/adapter-usage/main.go
+++ b/documentation/examples/custom-sd/adapter-usage/main.go
@@ -28,10 +28,8 @@ import (
 	"github.com/alecthomas/kingpin/v2"
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
-	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 
-	prom_discovery "github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 	"github.com/prometheus/prometheus/documentation/examples/custom-sd/adapter"
 	"github.com/prometheus/prometheus/util/strutil"
@@ -271,12 +269,11 @@ func main() {
 		fmt.Println("err: ", err)
 	}
 
-	discoveryMetrics, err := prom_discovery.NewMetrics(prometheus.DefaultRegisterer)
 	if err != nil {
 		level.Error(logger).Log("msg", "failed to create discovery metrics", "err", err)
 		os.Exit(1)
 	}
-	sdAdapter := adapter.NewAdapter(ctx, *outputFile, "exampleSD", disc, logger, discoveryMetrics)
+	sdAdapter := adapter.NewAdapter(ctx, *outputFile, "exampleSD", disc, logger)
 	sdAdapter.Run()
 
 	<-ctx.Done()
diff --git a/documentation/examples/custom-sd/adapter/adapter.go b/documentation/examples/custom-sd/adapter/adapter.go
index 8aedf0084..7fbf94aa9 100644
--- a/documentation/examples/custom-sd/adapter/adapter.go
+++ b/documentation/examples/custom-sd/adapter/adapter.go
@@ -163,12 +163,12 @@ func (a *Adapter) Run() {
 }
 
 // NewAdapter creates a new instance of Adapter.
-func NewAdapter(ctx context.Context, file, name string, d discovery.Discoverer, logger log.Logger, metrics *discovery.Metrics) *Adapter {
+func NewAdapter(ctx context.Context, file, name string, d discovery.Discoverer, logger log.Logger) *Adapter {
 	return &Adapter{
 		ctx:     ctx,
 		disc:    d,
 		groups:  make(map[string]*customSD),
-		manager: discovery.NewManager(ctx, logger, prometheus.NewRegistry(), metrics),
+		manager: discovery.NewManager(ctx, logger, prometheus.NewRegistry()),
 		output:  file,
 		name:    name,
 		logger:  logger,
diff --git a/documentation/examples/custom-sd/adapter/adapter_test.go b/documentation/examples/custom-sd/adapter/adapter_test.go
index 14cae47b4..8e5920eb4 100644
--- a/documentation/examples/custom-sd/adapter/adapter_test.go
+++ b/documentation/examples/custom-sd/adapter/adapter_test.go
@@ -18,11 +18,9 @@ import (
 	"os"
 	"testing"
 
-	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 
-	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -228,8 +226,7 @@ func TestWriteOutput(t *testing.T) {
 	require.NoError(t, err)
 	defer os.Remove(tmpfile.Name())
 	tmpfile.Close()
-	metrics, err := discovery.NewMetrics(prometheus.NewRegistry())
 	require.NoError(t, err)
-	adapter := NewAdapter(ctx, tmpfile.Name(), "test_sd", nil, nil, metrics)
+	adapter := NewAdapter(ctx, tmpfile.Name(), "test_sd", nil, nil)
 	require.NoError(t, adapter.writeOutput())
 }

From 6279497124673eda4de899b08bbf6d66e6041892 Mon Sep 17 00:00:00 2001
From: Paulin Todev <paulin.todev@gmail.com>
Date: Tue, 5 Dec 2023 18:01:39 +0000
Subject: [PATCH 020/115] Remove unnecessary else. This was flagged by the
 linter.

Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
---
 cmd/prometheus/main.go | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go
index f30fbecd9..7438476c3 100644
--- a/cmd/prometheus/main.go
+++ b/cmd/prometheus/main.go
@@ -636,9 +636,8 @@ func main() {
 			if discMgr == nil {
 				level.Error(logger).Log("msg", "failed to create a discovery manager scrape")
 				os.Exit(1)
-			} else {
-				discoveryManagerScrape = discMgr
 			}
+			discoveryManagerScrape = discMgr
 		}
 
 		{
@@ -646,9 +645,8 @@ func main() {
 			if discMgr == nil {
 				level.Error(logger).Log("msg", "failed to create a discovery manager notify")
 				os.Exit(1)
-			} else {
-				discoveryManagerNotify = discMgr
 			}
+			discoveryManagerNotify = discMgr
 		}
 	} else {
 		{
@@ -656,9 +654,8 @@ func main() {
 			if discMgr == nil {
 				level.Error(logger).Log("msg", "failed to create a discovery manager scrape")
 				os.Exit(1)
-			} else {
-				discoveryManagerScrape = discMgr
 			}
+			discoveryManagerScrape = discMgr
 		}
 
 		{
@@ -666,9 +663,8 @@ func main() {
 			if discMgr == nil {
 				level.Error(logger).Log("msg", "failed to create a discovery manager notify")
 				os.Exit(1)
-			} else {
-				discoveryManagerNotify = discMgr
 			}
+			discoveryManagerNotify = discMgr
 		}
 	}
 

From 8f6cf3aabb1b9c0b385d57c922f8e2f571bba61b Mon Sep 17 00:00:00 2001
From: Matthieu MOREL <matthieu.morel35@gmail.com>
Date: Thu, 16 Nov 2023 19:54:41 +0100
Subject: [PATCH 021/115] tsdb: use Go standard errors

Signed-off-by: Matthieu MOREL <matthieu.morel35@gmail.com>
---
 .golangci.yml               |   6 +-
 go.mod                      |   2 +-
 tsdb/agent/db.go            |   2 +-
 tsdb/block.go               |  29 ++++---
 tsdb/blockwriter.go         |  11 +--
 tsdb/chunkenc/chunk_test.go |   5 +-
 tsdb/chunks/head_chunks.go  |   9 ++-
 tsdb/compact.go             |  50 ++++++------
 tsdb/compact_test.go        |   2 +-
 tsdb/db.go                  | 154 +++++++++++++++++++-----------------
 tsdb/db_test.go             |   3 +-
 tsdb/exemplar.go            |   3 +-
 tsdb/head.go                |  48 +++++------
 tsdb/head_append.go         |  50 ++++++------
 tsdb/head_bench_test.go     |   2 +-
 tsdb/head_read.go           |  15 ++--
 tsdb/head_test.go           |  15 ++--
 tsdb/head_wal.go            | 103 +++++++++++++-----------
 tsdb/index/index.go         |   6 +-
 tsdb/mocks_test.go          |   4 +-
 tsdb/querier.go             |  40 +++++-----
 tsdb/querier_test.go        |   4 +-
 tsdb/repair.go              |  43 ++++++----
 tsdb/wal.go                 | 101 ++++++++++++-----------
 tsdb/wlog/wlog.go           |   4 +
 25 files changed, 382 insertions(+), 329 deletions(-)

diff --git a/.golangci.yml b/.golangci.yml
index 01ba9deb0..166b2e0d4 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -36,13 +36,9 @@ issues:
     - path: _test.go
       linters:
         - errcheck
-    - path: tsdb/
+    - path: "tsdb/head_wal.go"
       linters:
         - errorlint
-    - path: tsdb/
-      text: "import 'github.com/pkg/errors' is not allowed"
-      linters:
-        - depguard
     - linters:
         - godot
       source: "^// ==="
diff --git a/go.mod b/go.mod
index 95cc7b8af..f785de723 100644
--- a/go.mod
+++ b/go.mod
@@ -43,7 +43,6 @@ require (
 	github.com/oklog/run v1.1.0
 	github.com/oklog/ulid v1.3.1
 	github.com/ovh/go-ovh v1.4.3
-	github.com/pkg/errors v0.9.1
 	github.com/prometheus/alertmanager v0.26.0
 	github.com/prometheus/client_golang v1.17.0
 	github.com/prometheus/client_model v0.5.0
@@ -167,6 +166,7 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.2 // indirect
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/procfs v0.11.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
diff --git a/tsdb/agent/db.go b/tsdb/agent/db.go
index 557fb7854..d39989713 100644
--- a/tsdb/agent/db.go
+++ b/tsdb/agent/db.go
@@ -359,7 +359,7 @@ func (db *DB) replayWAL() error {
 	start := time.Now()
 
 	dir, startFrom, err := wlog.LastCheckpoint(db.wal.Dir())
-	if err != nil && err != record.ErrNotFound {
+	if err != nil && !errors.Is(err, record.ErrNotFound) {
 		return fmt.Errorf("find last checkpoint: %w", err)
 	}
 
diff --git a/tsdb/block.go b/tsdb/block.go
index a586536b1..e2562de03 100644
--- a/tsdb/block.go
+++ b/tsdb/block.go
@@ -17,6 +17,7 @@ package tsdb
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -26,7 +27,6 @@ import (
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/oklog/ulid"
-	"github.com/pkg/errors"
 	"golang.org/x/exp/slices"
 
 	"github.com/prometheus/prometheus/model/labels"
@@ -479,14 +479,19 @@ func (r blockIndexReader) SortedLabelValues(ctx context.Context, name string, ma
 			slices.Sort(st)
 		}
 	}
-
-	return st, errors.Wrapf(err, "block: %s", r.b.Meta().ULID)
+	if err != nil {
+		return st, fmt.Errorf("block: %s: %w", r.b.Meta().ULID, err)
+	}
+	return st, nil
 }
 
 func (r blockIndexReader) LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, error) {
 	if len(matchers) == 0 {
 		st, err := r.ir.LabelValues(ctx, name)
-		return st, errors.Wrapf(err, "block: %s", r.b.Meta().ULID)
+		if err != nil {
+			return st, fmt.Errorf("block: %s: %w", r.b.Meta().ULID, err)
+		}
+		return st, nil
 	}
 
 	return labelValuesWithMatchers(ctx, r.ir, name, matchers...)
@@ -503,7 +508,7 @@ func (r blockIndexReader) LabelNames(ctx context.Context, matchers ...*labels.Ma
 func (r blockIndexReader) Postings(ctx context.Context, name string, values ...string) (index.Postings, error) {
 	p, err := r.ir.Postings(ctx, name, values...)
 	if err != nil {
-		return p, errors.Wrapf(err, "block: %s", r.b.Meta().ULID)
+		return p, fmt.Errorf("block: %s: %w", r.b.Meta().ULID, err)
 	}
 	return p, nil
 }
@@ -514,7 +519,7 @@ func (r blockIndexReader) SortedPostings(p index.Postings) index.Postings {
 
 func (r blockIndexReader) Series(ref storage.SeriesRef, builder *labels.ScratchBuilder, chks *[]chunks.Meta) error {
 	if err := r.ir.Series(ref, builder, chks); err != nil {
-		return errors.Wrapf(err, "block: %s", r.b.Meta().ULID)
+		return fmt.Errorf("block: %s: %w", r.b.Meta().ULID, err)
 	}
 	return nil
 }
@@ -566,7 +571,7 @@ func (pb *Block) Delete(ctx context.Context, mint, maxt int64, ms ...*labels.Mat
 
 	p, err := PostingsForMatchers(ctx, pb.indexr, ms...)
 	if err != nil {
-		return errors.Wrap(err, "select series")
+		return fmt.Errorf("select series: %w", err)
 	}
 
 	ir := pb.indexr
@@ -654,12 +659,12 @@ func (pb *Block) CleanTombstones(dest string, c Compactor) (*ulid.ULID, bool, er
 func (pb *Block) Snapshot(dir string) error {
 	blockDir := filepath.Join(dir, pb.meta.ULID.String())
 	if err := os.MkdirAll(blockDir, 0o777); err != nil {
-		return errors.Wrap(err, "create snapshot block dir")
+		return fmt.Errorf("create snapshot block dir: %w", err)
 	}
 
 	chunksDir := chunkDir(blockDir)
 	if err := os.MkdirAll(chunksDir, 0o777); err != nil {
-		return errors.Wrap(err, "create snapshot chunk dir")
+		return fmt.Errorf("create snapshot chunk dir: %w", err)
 	}
 
 	// Hardlink meta, index and tombstones
@@ -669,7 +674,7 @@ func (pb *Block) Snapshot(dir string) error {
 		tombstones.TombstonesFilename,
 	} {
 		if err := os.Link(filepath.Join(pb.dir, fname), filepath.Join(blockDir, fname)); err != nil {
-			return errors.Wrapf(err, "create snapshot %s", fname)
+			return fmt.Errorf("create snapshot %s: %w", fname, err)
 		}
 	}
 
@@ -677,13 +682,13 @@ func (pb *Block) Snapshot(dir string) error {
 	curChunkDir := chunkDir(pb.dir)
 	files, err := os.ReadDir(curChunkDir)
 	if err != nil {
-		return errors.Wrap(err, "ReadDir the current chunk dir")
+		return fmt.Errorf("ReadDir the current chunk dir: %w", err)
 	}
 
 	for _, f := range files {
 		err := os.Link(filepath.Join(curChunkDir, f.Name()), filepath.Join(chunksDir, f.Name()))
 		if err != nil {
-			return errors.Wrap(err, "hardlink a chunk")
+			return fmt.Errorf("hardlink a chunk: %w", err)
 		}
 	}
 
diff --git a/tsdb/blockwriter.go b/tsdb/blockwriter.go
index 0d017e095..73bc5f1e3 100644
--- a/tsdb/blockwriter.go
+++ b/tsdb/blockwriter.go
@@ -15,13 +15,14 @@ package tsdb
 
 import (
 	"context"
+	"errors"
+	"fmt"
 	"math"
 	"os"
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/oklog/ulid"
-	"github.com/pkg/errors"
 
 	"github.com/prometheus/prometheus/model/timestamp"
 	"github.com/prometheus/prometheus/storage"
@@ -65,7 +66,7 @@ func NewBlockWriter(logger log.Logger, dir string, blockSize int64) (*BlockWrite
 func (w *BlockWriter) initHead() error {
 	chunkDir, err := os.MkdirTemp(os.TempDir(), "head")
 	if err != nil {
-		return errors.Wrap(err, "create temp dir")
+		return fmt.Errorf("create temp dir: %w", err)
 	}
 	w.chunkDir = chunkDir
 	opts := DefaultHeadOptions()
@@ -74,7 +75,7 @@ func (w *BlockWriter) initHead() error {
 	opts.EnableNativeHistograms.Store(true)
 	h, err := NewHead(nil, w.logger, nil, nil, opts, NewHeadStats())
 	if err != nil {
-		return errors.Wrap(err, "tsdb.NewHead")
+		return fmt.Errorf("tsdb.NewHead: %w", err)
 	}
 
 	w.head = h
@@ -102,11 +103,11 @@ func (w *BlockWriter) Flush(ctx context.Context) (ulid.ULID, error) {
 		[]int64{w.blockSize},
 		chunkenc.NewPool(), nil)
 	if err != nil {
-		return ulid.ULID{}, errors.Wrap(err, "create leveled compactor")
+		return ulid.ULID{}, fmt.Errorf("create leveled compactor: %w", err)
 	}
 	id, err := compactor.Write(w.destinationDir, w.head, mint, maxt, nil)
 	if err != nil {
-		return ulid.ULID{}, errors.Wrap(err, "compactor write")
+		return ulid.ULID{}, fmt.Errorf("compactor write: %w", err)
 	}
 
 	return id, nil
diff --git a/tsdb/chunkenc/chunk_test.go b/tsdb/chunkenc/chunk_test.go
index 3d22f74cc..9db1bf364 100644
--- a/tsdb/chunkenc/chunk_test.go
+++ b/tsdb/chunkenc/chunk_test.go
@@ -14,6 +14,7 @@
 package chunkenc
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"math/rand"
@@ -153,8 +154,8 @@ func benchmarkIterator(b *testing.B, newChunk func() Chunk) {
 			res = v
 			i++
 		}
-		if it.Err() != io.EOF {
-			require.NoError(b, it.Err())
+		if err := it.Err(); err != nil && !errors.Is(err, io.EOF) {
+			require.NoError(b, err)
 		}
 		_ = res
 	}
diff --git a/tsdb/chunks/head_chunks.go b/tsdb/chunks/head_chunks.go
index b495b6182..12c3e7b90 100644
--- a/tsdb/chunks/head_chunks.go
+++ b/tsdb/chunks/head_chunks.go
@@ -111,6 +111,10 @@ func (e *CorruptionErr) Error() string {
 	return fmt.Errorf("corruption in head chunk file %s: %w", segmentFile(e.Dir, e.FileIndex), e.Err).Error()
 }
 
+func (e *CorruptionErr) Unwrap() error {
+	return e.Err
+}
+
 // chunkPos keeps track of the position in the head chunk files.
 // chunkPos is not thread-safe, a lock must be used to protect it.
 type chunkPos struct {
@@ -400,7 +404,7 @@ func repairLastChunkFile(files map[int]string) (_ map[int]string, returnErr erro
 
 	buf := make([]byte, MagicChunksSize)
 	size, err := f.Read(buf)
-	if err != nil && err != io.EOF {
+	if err != nil && !errors.Is(err, io.EOF) {
 		return files, fmt.Errorf("failed to read magic number during last head chunk file repair: %w", err)
 	}
 	if err := f.Close(); err != nil {
@@ -892,7 +896,8 @@ func (cdm *ChunkDiskMapper) IterateAllChunks(f func(seriesRef HeadSeriesRef, chu
 			// Extract the encoding from the byte. ChunkDiskMapper uses only the last 7 bits for the encoding.
 			chkEnc = cdm.RemoveMasks(chkEnc)
 			if err := f(seriesRef, chunkRef, mint, maxt, numSamples, chkEnc, isOOO); err != nil {
-				if cerr, ok := err.(*CorruptionErr); ok {
+				var cerr *CorruptionErr
+				if errors.As(err, &cerr) {
 					cerr.Dir = cdm.dir.Name()
 					cerr.FileIndex = segID
 					return cerr
diff --git a/tsdb/compact.go b/tsdb/compact.go
index 32c88d2cc..7b4ff9c1b 100644
--- a/tsdb/compact.go
+++ b/tsdb/compact.go
@@ -16,6 +16,7 @@ package tsdb
 import (
 	"context"
 	"crypto/rand"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -25,7 +26,6 @@ import (
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/oklog/ulid"
-	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"golang.org/x/exp/slices"
 
@@ -485,7 +485,7 @@ func (c *LeveledCompactor) CompactWithBlockPopulator(dest string, dirs []string,
 	if !errors.Is(err, context.Canceled) {
 		for _, b := range bs {
 			if err := b.setCompactionFailed(); err != nil {
-				errs.Add(errors.Wrapf(err, "setting compaction failed for block: %s", b.Dir()))
+				errs.Add(fmt.Errorf("setting compaction failed for block: %s: %w", b.Dir(), err))
 			}
 		}
 	}
@@ -586,7 +586,7 @@ func (c *LeveledCompactor) write(dest string, meta *BlockMeta, blockPopulator Bl
 
 	chunkw, err = chunks.NewWriterWithSegSize(chunkDir(tmp), c.maxBlockChunkSegmentSize)
 	if err != nil {
-		return errors.Wrap(err, "open chunk writer")
+		return fmt.Errorf("open chunk writer: %w", err)
 	}
 	closers = append(closers, chunkw)
 	// Record written chunk sizes on level 1 compactions.
@@ -601,12 +601,12 @@ func (c *LeveledCompactor) write(dest string, meta *BlockMeta, blockPopulator Bl
 
 	indexw, err := index.NewWriter(c.ctx, filepath.Join(tmp, indexFilename))
 	if err != nil {
-		return errors.Wrap(err, "open index writer")
+		return fmt.Errorf("open index writer: %w", err)
 	}
 	closers = append(closers, indexw)
 
 	if err := blockPopulator.PopulateBlock(c.ctx, c.metrics, c.logger, c.chunkPool, c.mergeFunc, blocks, meta, indexw, chunkw); err != nil {
-		return errors.Wrap(err, "populate block")
+		return fmt.Errorf("populate block: %w", err)
 	}
 
 	select {
@@ -634,17 +634,17 @@ func (c *LeveledCompactor) write(dest string, meta *BlockMeta, blockPopulator Bl
 	}
 
 	if _, err = writeMetaFile(c.logger, tmp, meta); err != nil {
-		return errors.Wrap(err, "write merged meta")
+		return fmt.Errorf("write merged meta: %w", err)
 	}
 
 	// Create an empty tombstones file.
 	if _, err := tombstones.WriteFile(c.logger, tmp, tombstones.NewMemTombstones()); err != nil {
-		return errors.Wrap(err, "write new tombstones file")
+		return fmt.Errorf("write new tombstones file: %w", err)
 	}
 
 	df, err := fileutil.OpenDir(tmp)
 	if err != nil {
-		return errors.Wrap(err, "open temporary block dir")
+		return fmt.Errorf("open temporary block dir: %w", err)
 	}
 	defer func() {
 		if df != nil {
@@ -653,18 +653,18 @@ func (c *LeveledCompactor) write(dest string, meta *BlockMeta, blockPopulator Bl
 	}()
 
 	if err := df.Sync(); err != nil {
-		return errors.Wrap(err, "sync temporary dir file")
+		return fmt.Errorf("sync temporary dir file: %w", err)
 	}
 
 	// Close temp dir before rename block dir (for windows platform).
 	if err = df.Close(); err != nil {
-		return errors.Wrap(err, "close temporary dir")
+		return fmt.Errorf("close temporary dir: %w", err)
 	}
 	df = nil
 
 	// Block successfully written, make it visible in destination dir by moving it from tmp one.
 	if err := fileutil.Replace(tmp, dir); err != nil {
-		return errors.Wrap(err, "rename block dir")
+		return fmt.Errorf("rename block dir: %w", err)
 	}
 
 	return nil
@@ -693,7 +693,7 @@ func (c DefaultBlockPopulator) PopulateBlock(ctx context.Context, metrics *Compa
 	defer func() {
 		errs := tsdb_errors.NewMulti(err)
 		if cerr := tsdb_errors.CloseAll(closers); cerr != nil {
-			errs.Add(errors.Wrap(cerr, "close"))
+			errs.Add(fmt.Errorf("close: %w", cerr))
 		}
 		err = errs.Err()
 		metrics.PopulatingBlocks.Set(0)
@@ -721,19 +721,19 @@ func (c DefaultBlockPopulator) PopulateBlock(ctx context.Context, metrics *Compa
 
 		indexr, err := b.Index()
 		if err != nil {
-			return errors.Wrapf(err, "open index reader for block %+v", b.Meta())
+			return fmt.Errorf("open index reader for block %+v: %w", b.Meta(), err)
 		}
 		closers = append(closers, indexr)
 
 		chunkr, err := b.Chunks()
 		if err != nil {
-			return errors.Wrapf(err, "open chunk reader for block %+v", b.Meta())
+			return fmt.Errorf("open chunk reader for block %+v: %w", b.Meta(), err)
 		}
 		closers = append(closers, chunkr)
 
 		tombsr, err := b.Tombstones()
 		if err != nil {
-			return errors.Wrapf(err, "open tombstone reader for block %+v", b.Meta())
+			return fmt.Errorf("open tombstone reader for block %+v: %w", b.Meta(), err)
 		}
 		closers = append(closers, tombsr)
 
@@ -755,11 +755,11 @@ func (c DefaultBlockPopulator) PopulateBlock(ctx context.Context, metrics *Compa
 
 	for symbols.Next() {
 		if err := indexw.AddSymbol(symbols.At()); err != nil {
-			return errors.Wrap(err, "add symbol")
+			return fmt.Errorf("add symbol: %w", err)
 		}
 	}
-	if symbols.Err() != nil {
-		return errors.Wrap(symbols.Err(), "next symbol")
+	if err := symbols.Err(); err != nil {
+		return fmt.Errorf("next symbol: %w", err)
 	}
 
 	var (
@@ -791,8 +791,8 @@ func (c DefaultBlockPopulator) PopulateBlock(ctx context.Context, metrics *Compa
 			// chunk file purposes.
 			chks = append(chks, chksIter.At())
 		}
-		if chksIter.Err() != nil {
-			return errors.Wrap(chksIter.Err(), "chunk iter")
+		if err := chksIter.Err(); err != nil {
+			return fmt.Errorf("chunk iter: %w", err)
 		}
 
 		// Skip the series with all deleted chunks.
@@ -801,10 +801,10 @@ func (c DefaultBlockPopulator) PopulateBlock(ctx context.Context, metrics *Compa
 		}
 
 		if err := chunkw.WriteChunks(chks...); err != nil {
-			return errors.Wrap(err, "write chunks")
+			return fmt.Errorf("write chunks: %w", err)
 		}
 		if err := indexw.AddSeries(ref, s.Labels(), chks...); err != nil {
-			return errors.Wrap(err, "add series")
+			return fmt.Errorf("add series: %w", err)
 		}
 
 		meta.Stats.NumChunks += uint64(len(chks))
@@ -815,13 +815,13 @@ func (c DefaultBlockPopulator) PopulateBlock(ctx context.Context, metrics *Compa
 
 		for _, chk := range chks {
 			if err := chunkPool.Put(chk.Chunk); err != nil {
-				return errors.Wrap(err, "put chunk")
+				return fmt.Errorf("put chunk: %w", err)
 			}
 		}
 		ref++
 	}
-	if set.Err() != nil {
-		return errors.Wrap(set.Err(), "iterate compaction set")
+	if err := set.Err(); err != nil {
+		return fmt.Errorf("iterate compaction set: %w", err)
 	}
 
 	return nil
diff --git a/tsdb/compact_test.go b/tsdb/compact_test.go
index 75d564139..f33bb73c1 100644
--- a/tsdb/compact_test.go
+++ b/tsdb/compact_test.go
@@ -15,6 +15,7 @@ package tsdb
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"math"
 	"math/rand"
@@ -27,7 +28,6 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/oklog/ulid"
-	"github.com/pkg/errors"
 	prom_testutil "github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/require"
 
diff --git a/tsdb/db.go b/tsdb/db.go
index 2e3801a9e..6d283fdd8 100644
--- a/tsdb/db.go
+++ b/tsdb/db.go
@@ -16,6 +16,7 @@ package tsdb
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"io/fs"
@@ -30,7 +31,6 @@ import (
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/oklog/ulid"
-	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.uber.org/atomic"
 	"golang.org/x/exp/slices"
@@ -386,7 +386,7 @@ type DBReadOnly struct {
 // OpenDBReadOnly opens DB in the given directory for read only operations.
 func OpenDBReadOnly(dir string, l log.Logger) (*DBReadOnly, error) {
 	if _, err := os.Stat(dir); err != nil {
-		return nil, errors.Wrap(err, "opening the db dir")
+		return nil, fmt.Errorf("opening the db dir: %w", err)
 	}
 
 	if l == nil {
@@ -407,7 +407,7 @@ func OpenDBReadOnly(dir string, l log.Logger) (*DBReadOnly, error) {
 func (db *DBReadOnly) FlushWAL(dir string) (returnErr error) {
 	blockReaders, err := db.Blocks()
 	if err != nil {
-		return errors.Wrap(err, "read blocks")
+		return fmt.Errorf("read blocks: %w", err)
 	}
 	maxBlockTime := int64(math.MinInt64)
 	if len(blockReaders) > 0 {
@@ -432,15 +432,16 @@ func (db *DBReadOnly) FlushWAL(dir string) (returnErr error) {
 		return err
 	}
 	defer func() {
-		returnErr = tsdb_errors.NewMulti(
-			returnErr,
-			errors.Wrap(head.Close(), "closing Head"),
-		).Err()
+		errs := tsdb_errors.NewMulti(returnErr)
+		if err := head.Close(); err != nil {
+			errs.Add(fmt.Errorf("closing Head: %w", err))
+		}
+		returnErr = errs.Err()
 	}()
 	// Set the min valid time for the ingested wal samples
 	// to be no lower than the maxt of the last block.
 	if err := head.Init(maxBlockTime); err != nil {
-		return errors.Wrap(err, "read WAL")
+		return fmt.Errorf("read WAL: %w", err)
 	}
 	mint := head.MinTime()
 	maxt := head.MaxTime()
@@ -454,12 +455,15 @@ func (db *DBReadOnly) FlushWAL(dir string) (returnErr error) {
 		nil,
 	)
 	if err != nil {
-		return errors.Wrap(err, "create leveled compactor")
+		return fmt.Errorf("create leveled compactor: %w", err)
 	}
 	// Add +1 millisecond to block maxt because block intervals are half-open: [b.MinTime, b.MaxTime).
 	// Because of this block intervals are always +1 than the total samples it includes.
 	_, err = compactor.Write(dir, rh, mint, maxt+1, nil)
-	return errors.Wrap(err, "writing WAL")
+	if err != nil {
+		return fmt.Errorf("writing WAL: %w", err)
+	}
+	return nil
 }
 
 func (db *DBReadOnly) loadDataAsQueryable(maxt int64) (storage.SampleAndChunkQueryable, error) {
@@ -518,7 +522,7 @@ func (db *DBReadOnly) loadDataAsQueryable(maxt int64) (storage.SampleAndChunkQue
 		// Set the min valid time for the ingested wal samples
 		// to be no lower than the maxt of the last block.
 		if err := head.Init(maxBlockTime); err != nil {
-			return nil, errors.Wrap(err, "read WAL")
+			return nil, fmt.Errorf("read WAL: %w", err)
 		}
 		// Set the wal to nil to disable all wal operations.
 		// This is mainly to avoid blocking when closing the head.
@@ -580,7 +584,9 @@ func (db *DBReadOnly) Blocks() ([]BlockReader, error) {
 		}
 		errs := tsdb_errors.NewMulti()
 		for ulid, err := range corrupted {
-			errs.Add(errors.Wrapf(err, "corrupted block %s", ulid.String()))
+			if err != nil {
+				errs.Add(fmt.Errorf("corrupted block %s: %w", ulid.String(), err))
+			}
 		}
 		return nil, errs.Err()
 	}
@@ -761,7 +767,7 @@ func open(dir string, l log.Logger, r prometheus.Registerer, opts *Options, rngs
 
 	// Fixup bad format written by Prometheus 2.1.
 	if err := repairBadIndexVersion(l, dir); err != nil {
-		return nil, errors.Wrap(err, "repair bad index version")
+		return nil, fmt.Errorf("repair bad index version: %w", err)
 	}
 
 	walDir := filepath.Join(dir, "wal")
@@ -769,12 +775,12 @@ func open(dir string, l log.Logger, r prometheus.Registerer, opts *Options, rngs
 
 	// Migrate old WAL if one exists.
 	if err := MigrateWAL(l, walDir); err != nil {
-		return nil, errors.Wrap(err, "migrate WAL")
+		return nil, fmt.Errorf("migrate WAL: %w", err)
 	}
 	for _, tmpDir := range []string{walDir, dir} {
 		// Remove tmp dirs.
 		if err := removeBestEffortTmpDirs(l, tmpDir); err != nil {
-			return nil, errors.Wrap(err, "remove tmp dirs")
+			return nil, fmt.Errorf("remove tmp dirs: %w", err)
 		}
 	}
 
@@ -797,11 +803,11 @@ func open(dir string, l log.Logger, r prometheus.Registerer, opts *Options, rngs
 		}
 
 		close(db.donec) // DB is never run if it was an error, so close this channel here.
-
-		returnedErr = tsdb_errors.NewMulti(
-			returnedErr,
-			errors.Wrap(db.Close(), "close DB after failed startup"),
-		).Err()
+		errs := tsdb_errors.NewMulti(returnedErr)
+		if err := db.Close(); err != nil {
+			errs.Add(fmt.Errorf("close DB after failed startup: %w", err))
+		}
+		returnedErr = errs.Err()
 	}()
 
 	if db.blocksToDelete == nil {
@@ -823,7 +829,7 @@ func open(dir string, l log.Logger, r prometheus.Registerer, opts *Options, rngs
 	db.compactor, err = NewLeveledCompactorWithChunkSize(ctx, r, l, rngs, db.chunkPool, opts.MaxBlockChunkSegmentSize, nil)
 	if err != nil {
 		cancel()
-		return nil, errors.Wrap(err, "create leveled compactor")
+		return nil, fmt.Errorf("create leveled compactor: %w", err)
 	}
 	db.compactCancel = cancel
 
@@ -905,17 +911,17 @@ func open(dir string, l log.Logger, r prometheus.Registerer, opts *Options, rngs
 
 	if initErr := db.head.Init(minValidTime); initErr != nil {
 		db.head.metrics.walCorruptionsTotal.Inc()
-		e, ok := initErr.(*errLoadWbl)
-		if ok {
+		var e *errLoadWbl
+		if errors.As(initErr, &e) {
 			level.Warn(db.logger).Log("msg", "Encountered WBL read error, attempting repair", "err", initErr)
 			if err := wbl.Repair(e.err); err != nil {
-				return nil, errors.Wrap(err, "repair corrupted WBL")
+				return nil, fmt.Errorf("repair corrupted WBL: %w", err)
 			}
 			level.Info(db.logger).Log("msg", "Successfully repaired WBL")
 		} else {
 			level.Warn(db.logger).Log("msg", "Encountered WAL read error, attempting repair", "err", initErr)
 			if err := wal.Repair(initErr); err != nil {
-				return nil, errors.Wrap(err, "repair corrupted WAL")
+				return nil, fmt.Errorf("repair corrupted WAL: %w", err)
 			}
 			level.Info(db.logger).Log("msg", "Successfully repaired WAL")
 		}
@@ -1131,10 +1137,11 @@ func (db *DB) Compact(ctx context.Context) (returnErr error) {
 
 	lastBlockMaxt := int64(math.MinInt64)
 	defer func() {
-		returnErr = tsdb_errors.NewMulti(
-			returnErr,
-			errors.Wrap(db.head.truncateWAL(lastBlockMaxt), "WAL truncation in Compact defer"),
-		).Err()
+		errs := tsdb_errors.NewMulti(returnErr)
+		if err := db.head.truncateWAL(lastBlockMaxt); err != nil {
+			errs.Add(fmt.Errorf("WAL truncation in Compact defer: %w", err))
+		}
+		returnErr = errs.Err()
 	}()
 
 	start := time.Now()
@@ -1168,7 +1175,7 @@ func (db *DB) Compact(ctx context.Context) (returnErr error) {
 		db.head.WaitForAppendersOverlapping(rh.MaxTime())
 
 		if err := db.compactHead(rh); err != nil {
-			return errors.Wrap(err, "compact head")
+			return fmt.Errorf("compact head: %w", err)
 		}
 		// Consider only successful compactions for WAL truncation.
 		lastBlockMaxt = maxt
@@ -1177,7 +1184,7 @@ func (db *DB) Compact(ctx context.Context) (returnErr error) {
 	// Clear some disk space before compacting blocks, especially important
 	// when Head compaction happened over a long time range.
 	if err := db.head.truncateWAL(lastBlockMaxt); err != nil {
-		return errors.Wrap(err, "WAL truncation in Compact")
+		return fmt.Errorf("WAL truncation in Compact: %w", err)
 	}
 
 	compactionDuration := time.Since(start)
@@ -1192,7 +1199,7 @@ func (db *DB) Compact(ctx context.Context) (returnErr error) {
 	if lastBlockMaxt != math.MinInt64 {
 		// The head was compacted, so we compact OOO head as well.
 		if err := db.compactOOOHead(ctx); err != nil {
-			return errors.Wrap(err, "compact ooo head")
+			return fmt.Errorf("compact ooo head: %w", err)
 		}
 	}
 
@@ -1205,11 +1212,11 @@ func (db *DB) CompactHead(head *RangeHead) error {
 	defer db.cmtx.Unlock()
 
 	if err := db.compactHead(head); err != nil {
-		return errors.Wrap(err, "compact head")
+		return fmt.Errorf("compact head: %w", err)
 	}
 
 	if err := db.head.truncateWAL(head.BlockMaxTime()); err != nil {
-		return errors.Wrap(err, "WAL truncation")
+		return fmt.Errorf("WAL truncation: %w", err)
 	}
 	return nil
 }
@@ -1228,12 +1235,12 @@ func (db *DB) compactOOOHead(ctx context.Context) error {
 	}
 	oooHead, err := NewOOOCompactionHead(ctx, db.head)
 	if err != nil {
-		return errors.Wrap(err, "get ooo compaction head")
+		return fmt.Errorf("get ooo compaction head: %w", err)
 	}
 
 	ulids, err := db.compactOOO(db.dir, oooHead)
 	if err != nil {
-		return errors.Wrap(err, "compact ooo head")
+		return fmt.Errorf("compact ooo head: %w", err)
 	}
 	if err := db.reloadBlocks(); err != nil {
 		errs := tsdb_errors.NewMulti(err)
@@ -1242,7 +1249,7 @@ func (db *DB) compactOOOHead(ctx context.Context) error {
 				errs.Add(errRemoveAll)
 			}
 		}
-		return errors.Wrap(errs.Err(), "reloadBlocks blocks after failed compact ooo head")
+		return fmt.Errorf("reloadBlocks blocks after failed compact ooo head: %w", errs.Err())
 	}
 
 	lastWBLFile, minOOOMmapRef := oooHead.LastWBLFile(), oooHead.LastMmapRef()
@@ -1262,7 +1269,7 @@ func (db *DB) compactOOOHead(ctx context.Context) error {
 		}
 
 		if err := db.head.truncateOOO(lastWBLFile, minOOOMmapRef); err != nil {
-			return errors.Wrap(err, "truncate ooo wbl")
+			return fmt.Errorf("truncate ooo wbl: %w", err)
 		}
 	}
 
@@ -1298,12 +1305,12 @@ func (db *DB) compactOOO(dest string, oooHead *OOOCompactionHead) (_ []ulid.ULID
 			blockDir := filepath.Join(dest, uid.String())
 			meta, _, err := readMetaFile(blockDir)
 			if err != nil {
-				return ulids, errors.Wrap(err, "read meta")
+				return ulids, fmt.Errorf("read meta: %w", err)
 			}
 			meta.Compaction.SetOutOfOrder()
 			_, err = writeMetaFile(db.logger, blockDir, meta)
 			if err != nil {
-				return ulids, errors.Wrap(err, "write meta")
+				return ulids, fmt.Errorf("write meta: %w", err)
 			}
 		}
 	}
@@ -1329,20 +1336,20 @@ func (db *DB) compactOOO(dest string, oooHead *OOOCompactionHead) (_ []ulid.ULID
 func (db *DB) compactHead(head *RangeHead) error {
 	uid, err := db.compactor.Write(db.dir, head, head.MinTime(), head.BlockMaxTime(), nil)
 	if err != nil {
-		return errors.Wrap(err, "persist head block")
+		return fmt.Errorf("persist head block: %w", err)
 	}
 
 	if err := db.reloadBlocks(); err != nil {
 		if errRemoveAll := os.RemoveAll(filepath.Join(db.dir, uid.String())); errRemoveAll != nil {
 			return tsdb_errors.NewMulti(
-				errors.Wrap(err, "reloadBlocks blocks"),
-				errors.Wrapf(errRemoveAll, "delete persisted head block after failed db reloadBlocks:%s", uid),
+				fmt.Errorf("reloadBlocks blocks: %w", err),
+				fmt.Errorf("delete persisted head block after failed db reloadBlocks:%s: %w", uid, errRemoveAll),
 			).Err()
 		}
-		return errors.Wrap(err, "reloadBlocks blocks")
+		return fmt.Errorf("reloadBlocks blocks: %w", err)
 	}
 	if err = db.head.truncateMemory(head.BlockMaxTime()); err != nil {
-		return errors.Wrap(err, "head memory truncate")
+		return fmt.Errorf("head memory truncate: %w", err)
 	}
 	return nil
 }
@@ -1354,7 +1361,7 @@ func (db *DB) compactBlocks() (err error) {
 	for {
 		plan, err := db.compactor.Plan(db.dir)
 		if err != nil {
-			return errors.Wrap(err, "plan compaction")
+			return fmt.Errorf("plan compaction: %w", err)
 		}
 		if len(plan) == 0 {
 			break
@@ -1368,14 +1375,14 @@ func (db *DB) compactBlocks() (err error) {
 
 		uid, err := db.compactor.Compact(db.dir, plan, db.blocks)
 		if err != nil {
-			return errors.Wrapf(err, "compact %s", plan)
+			return fmt.Errorf("compact %s: %w", plan, err)
 		}
 
 		if err := db.reloadBlocks(); err != nil {
 			if err := os.RemoveAll(filepath.Join(db.dir, uid.String())); err != nil {
-				return errors.Wrapf(err, "delete compacted block after failed db reloadBlocks:%s", uid)
+				return fmt.Errorf("delete compacted block after failed db reloadBlocks:%s: %w", uid, err)
 			}
-			return errors.Wrap(err, "reloadBlocks blocks")
+			return fmt.Errorf("reloadBlocks blocks: %w", err)
 		}
 	}
 
@@ -1396,14 +1403,14 @@ func getBlock(allBlocks []*Block, id ulid.ULID) (*Block, bool) {
 // reload reloads blocks and truncates the head and its WAL.
 func (db *DB) reload() error {
 	if err := db.reloadBlocks(); err != nil {
-		return errors.Wrap(err, "reloadBlocks")
+		return fmt.Errorf("reloadBlocks: %w", err)
 	}
 	maxt, ok := db.inOrderBlocksMaxTime()
 	if !ok {
 		return nil
 	}
 	if err := db.head.Truncate(maxt); err != nil {
-		return errors.Wrap(err, "head truncate")
+		return fmt.Errorf("head truncate: %w", err)
 	}
 	return nil
 }
@@ -1457,7 +1464,9 @@ func (db *DB) reloadBlocks() (err error) {
 		}
 		errs := tsdb_errors.NewMulti()
 		for ulid, err := range corrupted {
-			errs.Add(errors.Wrapf(err, "corrupted block %s", ulid.String()))
+			if err != nil {
+				errs.Add(fmt.Errorf("corrupted block %s: %w", ulid.String(), err))
+			}
 		}
 		return errs.Err()
 	}
@@ -1509,7 +1518,7 @@ func (db *DB) reloadBlocks() (err error) {
 		}
 	}
 	if err := db.deleteBlocks(deletable); err != nil {
-		return errors.Wrapf(err, "delete %v blocks", len(deletable))
+		return fmt.Errorf("delete %v blocks: %w", len(deletable), err)
 	}
 	return nil
 }
@@ -1517,7 +1526,7 @@ func (db *DB) reloadBlocks() (err error) {
 func openBlocks(l log.Logger, dir string, loaded []*Block, chunkPool chunkenc.Pool) (blocks []*Block, corrupted map[ulid.ULID]error, err error) {
 	bDirs, err := blockDirs(dir)
 	if err != nil {
-		return nil, nil, errors.Wrap(err, "find blocks")
+		return nil, nil, fmt.Errorf("find blocks: %w", err)
 	}
 
 	corrupted = make(map[ulid.ULID]error)
@@ -1651,16 +1660,16 @@ func (db *DB) deleteBlocks(blocks map[ulid.ULID]*Block) error {
 			// Noop.
 			continue
 		case err != nil:
-			return errors.Wrapf(err, "stat dir %v", toDelete)
+			return fmt.Errorf("stat dir %v: %w", toDelete, err)
 		}
 
 		// Replace atomically to avoid partial block when process would crash during deletion.
 		tmpToDelete := filepath.Join(db.dir, fmt.Sprintf("%s%s", ulid, tmpForDeletionBlockDirSuffix))
 		if err := fileutil.Replace(toDelete, tmpToDelete); err != nil {
-			return errors.Wrapf(err, "replace of obsolete block for deletion %s", ulid)
+			return fmt.Errorf("replace of obsolete block for deletion %s: %w", ulid, err)
 		}
 		if err := os.RemoveAll(tmpToDelete); err != nil {
-			return errors.Wrapf(err, "delete obsolete block %s", ulid)
+			return fmt.Errorf("delete obsolete block %s: %w", ulid, err)
 		}
 		level.Info(db.logger).Log("msg", "Deleting obsolete block", "block", ulid)
 	}
@@ -1868,7 +1877,7 @@ func (db *DB) Snapshot(dir string, withHead bool) error {
 		level.Info(db.logger).Log("msg", "Snapshotting block", "block", b)
 
 		if err := b.Snapshot(dir); err != nil {
-			return errors.Wrapf(err, "error snapshotting block: %s", b.Dir())
+			return fmt.Errorf("error snapshotting block: %s: %w", b.Dir(), err)
 		}
 	}
 	if !withHead {
@@ -1881,7 +1890,7 @@ func (db *DB) Snapshot(dir string, withHead bool) error {
 	// Add +1 millisecond to block maxt because block intervals are half-open: [b.MinTime, b.MaxTime).
 	// Because of this block intervals are always +1 than the total samples it includes.
 	if _, err := db.compactor.Write(dir, head, mint, maxt+1, nil); err != nil {
-		return errors.Wrap(err, "snapshot head block")
+		return fmt.Errorf("snapshot head block: %w", err)
 	}
 	return nil
 }
@@ -1916,7 +1925,7 @@ func (db *DB) Querier(mint, maxt int64) (_ storage.Querier, err error) {
 		var err error
 		inOrderHeadQuerier, err := NewBlockQuerier(rh, mint, maxt)
 		if err != nil {
-			return nil, errors.Wrapf(err, "open block querier for head %s", rh)
+			return nil, fmt.Errorf("open block querier for head %s: %w", rh, err)
 		}
 
 		// Getting the querier above registers itself in the queue that the truncation waits on.
@@ -1925,7 +1934,7 @@ func (db *DB) Querier(mint, maxt int64) (_ storage.Querier, err error) {
 		shouldClose, getNew, newMint := db.head.IsQuerierCollidingWithTruncation(mint, maxt)
 		if shouldClose {
 			if err := inOrderHeadQuerier.Close(); err != nil {
-				return nil, errors.Wrapf(err, "closing head block querier %s", rh)
+				return nil, fmt.Errorf("closing head block querier %s: %w", rh, err)
 			}
 			inOrderHeadQuerier = nil
 		}
@@ -1933,7 +1942,7 @@ func (db *DB) Querier(mint, maxt int64) (_ storage.Querier, err error) {
 			rh := NewRangeHead(db.head, newMint, maxt)
 			inOrderHeadQuerier, err = NewBlockQuerier(rh, newMint, maxt)
 			if err != nil {
-				return nil, errors.Wrapf(err, "open block querier for head while getting new querier %s", rh)
+				return nil, fmt.Errorf("open block querier for head while getting new querier %s: %w", rh, err)
 			}
 		}
 
@@ -1950,7 +1959,7 @@ func (db *DB) Querier(mint, maxt int64) (_ storage.Querier, err error) {
 			// If NewBlockQuerier() failed, make sure to clean up the pending read created by NewOOORangeHead.
 			rh.isoState.Close()
 
-			return nil, errors.Wrapf(err, "open block querier for ooo head %s", rh)
+			return nil, fmt.Errorf("open block querier for ooo head %s: %w", rh, err)
 		}
 
 		blockQueriers = append(blockQueriers, outOfOrderHeadQuerier)
@@ -1959,7 +1968,7 @@ func (db *DB) Querier(mint, maxt int64) (_ storage.Querier, err error) {
 	for _, b := range blocks {
 		q, err := NewBlockQuerier(b, mint, maxt)
 		if err != nil {
-			return nil, errors.Wrapf(err, "open querier for block %s", b)
+			return nil, fmt.Errorf("open querier for block %s: %w", b, err)
 		}
 		blockQueriers = append(blockQueriers, q)
 	}
@@ -1997,7 +2006,7 @@ func (db *DB) blockChunkQuerierForRange(mint, maxt int64) (_ []storage.ChunkQuer
 		rh := NewRangeHead(db.head, mint, maxt)
 		inOrderHeadQuerier, err := NewBlockChunkQuerier(rh, mint, maxt)
 		if err != nil {
-			return nil, errors.Wrapf(err, "open querier for head %s", rh)
+			return nil, fmt.Errorf("open querier for head %s: %w", rh, err)
 		}
 
 		// Getting the querier above registers itself in the queue that the truncation waits on.
@@ -2006,7 +2015,7 @@ func (db *DB) blockChunkQuerierForRange(mint, maxt int64) (_ []storage.ChunkQuer
 		shouldClose, getNew, newMint := db.head.IsQuerierCollidingWithTruncation(mint, maxt)
 		if shouldClose {
 			if err := inOrderHeadQuerier.Close(); err != nil {
-				return nil, errors.Wrapf(err, "closing head querier %s", rh)
+				return nil, fmt.Errorf("closing head querier %s: %w", rh, err)
 			}
 			inOrderHeadQuerier = nil
 		}
@@ -2014,7 +2023,7 @@ func (db *DB) blockChunkQuerierForRange(mint, maxt int64) (_ []storage.ChunkQuer
 			rh := NewRangeHead(db.head, newMint, maxt)
 			inOrderHeadQuerier, err = NewBlockChunkQuerier(rh, newMint, maxt)
 			if err != nil {
-				return nil, errors.Wrapf(err, "open querier for head while getting new querier %s", rh)
+				return nil, fmt.Errorf("open querier for head while getting new querier %s: %w", rh, err)
 			}
 		}
 
@@ -2027,7 +2036,7 @@ func (db *DB) blockChunkQuerierForRange(mint, maxt int64) (_ []storage.ChunkQuer
 		rh := NewOOORangeHead(db.head, mint, maxt, db.lastGarbageCollectedMmapRef)
 		outOfOrderHeadQuerier, err := NewBlockChunkQuerier(rh, mint, maxt)
 		if err != nil {
-			return nil, errors.Wrapf(err, "open block chunk querier for ooo head %s", rh)
+			return nil, fmt.Errorf("open block chunk querier for ooo head %s: %w", rh, err)
 		}
 
 		blockQueriers = append(blockQueriers, outOfOrderHeadQuerier)
@@ -2036,7 +2045,7 @@ func (db *DB) blockChunkQuerierForRange(mint, maxt int64) (_ []storage.ChunkQuer
 	for _, b := range blocks {
 		q, err := NewBlockChunkQuerier(b, mint, maxt)
 		if err != nil {
-			return nil, errors.Wrapf(err, "open querier for block %s", b)
+			return nil, fmt.Errorf("open querier for block %s: %w", b, err)
 		}
 		blockQueriers = append(blockQueriers, q)
 	}
@@ -2105,7 +2114,7 @@ func (db *DB) CleanTombstones() (err error) {
 		for _, pb := range db.Blocks() {
 			uid, safeToDelete, cleanErr := pb.CleanTombstones(db.Dir(), db.compactor)
 			if cleanErr != nil {
-				return errors.Wrapf(cleanErr, "clean tombstones: %s", pb.Dir())
+				return fmt.Errorf("clean tombstones: %s: %w", pb.Dir(), cleanErr)
 			}
 			if !safeToDelete {
 				// There was nothing to clean.
@@ -2133,7 +2142,10 @@ func (db *DB) CleanTombstones() (err error) {
 					level.Error(db.logger).Log("msg", "failed to delete block after failed `CleanTombstones`", "dir", dir, "err", err)
 				}
 			}
-			return errors.Wrap(err, "reload blocks")
+			if err != nil {
+				return fmt.Errorf("reload blocks: %w", err)
+			}
+			return nil
 		}
 	}
 	return nil
diff --git a/tsdb/db_test.go b/tsdb/db_test.go
index 4dcdef858..cb12f8630 100644
--- a/tsdb/db_test.go
+++ b/tsdb/db_test.go
@@ -34,7 +34,6 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/oklog/ulid"
-	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	prom_testutil "github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/require"
@@ -355,7 +354,7 @@ func TestDBAppenderAddRef(t *testing.T) {
 
 	// Missing labels & invalid refs should fail.
 	_, err = app2.Append(9999999, labels.EmptyLabels(), 1, 1)
-	require.Equal(t, ErrInvalidSample, errors.Cause(err))
+	require.ErrorIs(t, err, ErrInvalidSample)
 
 	require.NoError(t, app2.Commit())
 
diff --git a/tsdb/exemplar.go b/tsdb/exemplar.go
index 8eaf42653..805de70da 100644
--- a/tsdb/exemplar.go
+++ b/tsdb/exemplar.go
@@ -15,6 +15,7 @@ package tsdb
 
 import (
 	"context"
+	"errors"
 	"sync"
 	"unicode/utf8"
 
@@ -363,7 +364,7 @@ func (ce *CircularExemplarStorage) AddExemplar(l labels.Labels, e exemplar.Exemp
 
 	err := ce.validateExemplar(seriesLabels, e, true)
 	if err != nil {
-		if err == storage.ErrDuplicateExemplar {
+		if errors.Is(err, storage.ErrDuplicateExemplar) {
 			// Duplicate exemplar, noop.
 			return nil
 		}
diff --git a/tsdb/head.go b/tsdb/head.go
index 848357359..63d8e9ea1 100644
--- a/tsdb/head.go
+++ b/tsdb/head.go
@@ -15,6 +15,7 @@ package tsdb
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"math"
@@ -27,7 +28,6 @@ import (
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/oklog/ulid"
-	"github.com/pkg/errors"
 	"go.uber.org/atomic"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -623,11 +623,11 @@ func (h *Head) Init(minValidTime int64) error {
 		if h.wal != nil {
 			_, endAt, err := wlog.Segments(h.wal.Dir())
 			if err != nil {
-				return errors.Wrap(err, "finding WAL segments")
+				return fmt.Errorf("finding WAL segments: %w", err)
 			}
 
 			_, idx, _, err := LastChunkSnapshot(h.opts.ChunkDirRoot)
-			if err != nil && err != record.ErrNotFound {
+			if err != nil && !errors.Is(err, record.ErrNotFound) {
 				level.Error(h.logger).Log("msg", "Could not find last snapshot", "err", err)
 			}
 
@@ -674,7 +674,8 @@ func (h *Head) Init(minValidTime int64) error {
 		if err != nil {
 			// TODO(codesome): clear out all m-map chunks here for refSeries.
 			level.Error(h.logger).Log("msg", "Loading on-disk chunks failed", "err", err)
-			if _, ok := errors.Cause(err).(*chunks.CorruptionErr); ok {
+			var cerr *chunks.CorruptionErr
+			if errors.As(err, &cerr) {
 				h.metrics.mmapChunkCorruptionTotal.Inc()
 			}
 
@@ -701,14 +702,14 @@ func (h *Head) Init(minValidTime int64) error {
 	checkpointReplayStart := time.Now()
 	// Backfill the checkpoint first if it exists.
 	dir, startFrom, err := wlog.LastCheckpoint(h.wal.Dir())
-	if err != nil && err != record.ErrNotFound {
-		return errors.Wrap(err, "find last checkpoint")
+	if err != nil && !errors.Is(err, record.ErrNotFound) {
+		return fmt.Errorf("find last checkpoint: %w", err)
 	}
 
 	// Find the last segment.
 	_, endAt, e := wlog.Segments(h.wal.Dir())
 	if e != nil {
-		return errors.Wrap(e, "finding WAL segments")
+		return fmt.Errorf("finding WAL segments: %w", e)
 	}
 
 	h.startWALReplayStatus(startFrom, endAt)
@@ -717,7 +718,7 @@ func (h *Head) Init(minValidTime int64) error {
 	if err == nil && startFrom >= snapIdx {
 		sr, err := wlog.NewSegmentsReader(dir)
 		if err != nil {
-			return errors.Wrap(err, "open checkpoint")
+			return fmt.Errorf("open checkpoint: %w", err)
 		}
 		defer func() {
 			if err := sr.Close(); err != nil {
@@ -728,7 +729,7 @@ func (h *Head) Init(minValidTime int64) error {
 		// A corrupted checkpoint is a hard error for now and requires user
 		// intervention. There's likely little data that can be recovered anyway.
 		if err := h.loadWAL(wlog.NewReader(sr), multiRef, mmappedChunks, oooMmappedChunks); err != nil {
-			return errors.Wrap(err, "backfill checkpoint")
+			return fmt.Errorf("backfill checkpoint: %w", err)
 		}
 		h.updateWALReplayStatusRead(startFrom)
 		startFrom++
@@ -745,7 +746,7 @@ func (h *Head) Init(minValidTime int64) error {
 	for i := startFrom; i <= endAt; i++ {
 		s, err := wlog.OpenReadSegment(wlog.SegmentName(h.wal.Dir(), i))
 		if err != nil {
-			return errors.Wrap(err, fmt.Sprintf("open WAL segment: %d", i))
+			return fmt.Errorf("open WAL segment: %d: %w", i, err)
 		}
 
 		offset := 0
@@ -758,7 +759,7 @@ func (h *Head) Init(minValidTime int64) error {
 			continue
 		}
 		if err != nil {
-			return errors.Wrapf(err, "segment reader (offset=%d)", offset)
+			return fmt.Errorf("segment reader (offset=%d): %w", offset, err)
 		}
 		err = h.loadWAL(wlog.NewReader(sr), multiRef, mmappedChunks, oooMmappedChunks)
 		if err := sr.Close(); err != nil {
@@ -777,14 +778,14 @@ func (h *Head) Init(minValidTime int64) error {
 		// Replay WBL.
 		startFrom, endAt, e = wlog.Segments(h.wbl.Dir())
 		if e != nil {
-			return &errLoadWbl{errors.Wrap(e, "finding WBL segments")}
+			return &errLoadWbl{fmt.Errorf("finding WBL segments: %w", e)}
 		}
 		h.startWALReplayStatus(startFrom, endAt)
 
 		for i := startFrom; i <= endAt; i++ {
 			s, err := wlog.OpenReadSegment(wlog.SegmentName(h.wbl.Dir(), i))
 			if err != nil {
-				return &errLoadWbl{errors.Wrap(err, fmt.Sprintf("open WBL segment: %d", i))}
+				return &errLoadWbl{fmt.Errorf("open WBL segment: %d: %w", i, err)}
 			}
 
 			sr := wlog.NewSegmentBufReader(s)
@@ -905,7 +906,7 @@ func (h *Head) loadMmappedChunks(refSeries map[chunks.HeadSeriesRef]*memSeries)
 		return nil
 	}); err != nil {
 		// secondLastRef because the lastRef caused an error.
-		return nil, nil, secondLastRef, errors.Wrap(err, "iterate on on-disk chunks")
+		return nil, nil, secondLastRef, fmt.Errorf("iterate on on-disk chunks: %w", err)
 	}
 	return mmappedChunks, oooMmappedChunks, lastRef, nil
 }
@@ -1224,12 +1225,12 @@ func (h *Head) truncateWAL(mint int64) error {
 
 	first, last, err := wlog.Segments(h.wal.Dir())
 	if err != nil {
-		return errors.Wrap(err, "get segment range")
+		return fmt.Errorf("get segment range: %w", err)
 	}
 	// Start a new segment, so low ingestion volume TSDB don't have more WAL than
 	// needed.
 	if _, err := h.wal.NextSegment(); err != nil {
-		return errors.Wrap(err, "next segment")
+		return fmt.Errorf("next segment: %w", err)
 	}
 	last-- // Never consider last segment for checkpoint.
 	if last < 0 {
@@ -1256,10 +1257,11 @@ func (h *Head) truncateWAL(mint int64) error {
 	h.metrics.checkpointCreationTotal.Inc()
 	if _, err = wlog.Checkpoint(h.logger, h.wal, first, last, keep, mint); err != nil {
 		h.metrics.checkpointCreationFail.Inc()
-		if _, ok := errors.Cause(err).(*wlog.CorruptionErr); ok {
+		var cerr *chunks.CorruptionErr
+		if errors.As(err, &cerr) {
 			h.metrics.walCorruptionsTotal.Inc()
 		}
-		return errors.Wrap(err, "create checkpoint")
+		return fmt.Errorf("create checkpoint: %w", err)
 	}
 	if err := h.wal.Truncate(last + 1); err != nil {
 		// If truncating fails, we'll just try again at the next checkpoint.
@@ -1352,7 +1354,7 @@ func (h *Head) truncateSeriesAndChunkDiskMapper(caller string) error {
 
 	// Truncate the chunk m-mapper.
 	if err := h.chunkDiskMapper.Truncate(uint32(minMmapFile)); err != nil {
-		return errors.Wrap(err, "truncate chunks.HeadReadWriter by file number")
+		return fmt.Errorf("truncate chunks.HeadReadWriter by file number: %w", err)
 	}
 	return nil
 }
@@ -1467,13 +1469,13 @@ func (h *Head) Delete(ctx context.Context, mint, maxt int64, ms ...*labels.Match
 
 	p, err := PostingsForMatchers(ctx, ir, ms...)
 	if err != nil {
-		return errors.Wrap(err, "select series")
+		return fmt.Errorf("select series: %w", err)
 	}
 
 	var stones []tombstones.Stone
 	for p.Next() {
 		if err := ctx.Err(); err != nil {
-			return errors.Wrap(err, "select series")
+			return fmt.Errorf("select series: %w", err)
 		}
 
 		series := h.series.getByID(chunks.HeadSeriesRef(p.At()))
@@ -1495,8 +1497,8 @@ func (h *Head) Delete(ctx context.Context, mint, maxt int64, ms ...*labels.Match
 	if p.Err() != nil {
 		return p.Err()
 	}
-	if ctx.Err() != nil {
-		return errors.Wrap(err, "select series")
+	if err := ctx.Err(); err != nil {
+		return fmt.Errorf("select series: %w", err)
 	}
 
 	if h.wal != nil {
diff --git a/tsdb/head_append.go b/tsdb/head_append.go
index afb461afe..f509317c8 100644
--- a/tsdb/head_append.go
+++ b/tsdb/head_append.go
@@ -15,11 +15,11 @@ package tsdb
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"math"
 
 	"github.com/go-kit/log/level"
-	"github.com/pkg/errors"
 
 	"github.com/prometheus/prometheus/model/exemplar"
 	"github.com/prometheus/prometheus/model/histogram"
@@ -358,10 +358,10 @@ func (a *headAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64
 		a.head.metrics.oooHistogram.Observe(float64(delta) / 1000)
 	}
 	if err != nil {
-		switch err {
-		case storage.ErrOutOfOrderSample:
+		switch {
+		case errors.Is(err, storage.ErrOutOfOrderSample):
 			a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat).Inc()
-		case storage.ErrTooOldSample:
+		case errors.Is(err, storage.ErrTooOldSample):
 			a.head.metrics.tooOldSamples.WithLabelValues(sampleMetricTypeFloat).Inc()
 		}
 		return 0, err
@@ -428,10 +428,10 @@ func (a *headAppender) getOrCreate(lset labels.Labels) (*memSeries, error) {
 	// Ensure no empty labels have gotten through.
 	lset = lset.WithoutEmpty()
 	if lset.IsEmpty() {
-		return nil, errors.Wrap(ErrInvalidSample, "empty labelset")
+		return nil, fmt.Errorf("empty labelset: %w", ErrInvalidSample)
 	}
 	if l, dup := lset.HasDuplicateLabelNames(); dup {
-		return nil, errors.Wrap(ErrInvalidSample, fmt.Sprintf(`label name "%s" is not unique`, l))
+		return nil, fmt.Errorf(`label name "%s" is not unique: %w`, l, ErrInvalidSample)
 	}
 	var created bool
 	var err error
@@ -557,7 +557,7 @@ func (a *headAppender) AppendExemplar(ref storage.SeriesRef, lset labels.Labels,
 
 	err := a.head.exemplars.ValidateExemplar(s.lset, e)
 	if err != nil {
-		if err == storage.ErrDuplicateExemplar || err == storage.ErrExemplarsDisabled {
+		if errors.Is(err, storage.ErrDuplicateExemplar) || errors.Is(err, storage.ErrExemplarsDisabled) {
 			// Duplicate, don't return an error but don't accept the exemplar.
 			return 0, nil
 		}
@@ -596,11 +596,11 @@ func (a *headAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels
 		// Ensure no empty labels have gotten through.
 		lset = lset.WithoutEmpty()
 		if lset.IsEmpty() {
-			return 0, errors.Wrap(ErrInvalidSample, "empty labelset")
+			return 0, fmt.Errorf("empty labelset: %w", ErrInvalidSample)
 		}
 
 		if l, dup := lset.HasDuplicateLabelNames(); dup {
-			return 0, errors.Wrap(ErrInvalidSample, fmt.Sprintf(`label name "%s" is not unique`, l))
+			return 0, fmt.Errorf(`label name "%s" is not unique: %w`, l, ErrInvalidSample)
 		}
 
 		var created bool
@@ -628,7 +628,7 @@ func (a *headAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels
 		s.Lock()
 		if err := s.appendableHistogram(t, h); err != nil {
 			s.Unlock()
-			if err == storage.ErrOutOfOrderSample {
+			if errors.Is(err, storage.ErrOutOfOrderSample) {
 				a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeHistogram).Inc()
 			}
 			return 0, err
@@ -645,7 +645,7 @@ func (a *headAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels
 		s.Lock()
 		if err := s.appendableFloatHistogram(t, fh); err != nil {
 			s.Unlock()
-			if err == storage.ErrOutOfOrderSample {
+			if errors.Is(err, storage.ErrOutOfOrderSample) {
 				a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeHistogram).Inc()
 			}
 			return 0, err
@@ -729,7 +729,7 @@ func (a *headAppender) log() error {
 		buf = rec[:0]
 
 		if err := a.head.wal.Log(rec); err != nil {
-			return errors.Wrap(err, "log series")
+			return fmt.Errorf("log series: %w", err)
 		}
 	}
 	if len(a.metadata) > 0 {
@@ -737,7 +737,7 @@ func (a *headAppender) log() error {
 		buf = rec[:0]
 
 		if err := a.head.wal.Log(rec); err != nil {
-			return errors.Wrap(err, "log metadata")
+			return fmt.Errorf("log metadata: %w", err)
 		}
 	}
 	if len(a.samples) > 0 {
@@ -745,21 +745,21 @@ func (a *headAppender) log() error {
 		buf = rec[:0]
 
 		if err := a.head.wal.Log(rec); err != nil {
-			return errors.Wrap(err, "log samples")
+			return fmt.Errorf("log samples: %w", err)
 		}
 	}
 	if len(a.histograms) > 0 {
 		rec = enc.HistogramSamples(a.histograms, buf)
 		buf = rec[:0]
 		if err := a.head.wal.Log(rec); err != nil {
-			return errors.Wrap(err, "log histograms")
+			return fmt.Errorf("log histograms: %w", err)
 		}
 	}
 	if len(a.floatHistograms) > 0 {
 		rec = enc.FloatHistogramSamples(a.floatHistograms, buf)
 		buf = rec[:0]
 		if err := a.head.wal.Log(rec); err != nil {
-			return errors.Wrap(err, "log float histograms")
+			return fmt.Errorf("log float histograms: %w", err)
 		}
 	}
 	// Exemplars should be logged after samples (float/native histogram/etc),
@@ -771,7 +771,7 @@ func (a *headAppender) log() error {
 		buf = rec[:0]
 
 		if err := a.head.wal.Log(rec); err != nil {
-			return errors.Wrap(err, "log exemplars")
+			return fmt.Errorf("log exemplars: %w", err)
 		}
 	}
 	return nil
@@ -800,7 +800,7 @@ func (a *headAppender) Commit() (err error) {
 
 	if err := a.log(); err != nil {
 		_ = a.Rollback() // Most likely the same error will happen again.
-		return errors.Wrap(err, "write to WAL")
+		return fmt.Errorf("write to WAL: %w", err)
 	}
 
 	if a.head.writeNotified != nil {
@@ -818,7 +818,7 @@ func (a *headAppender) Commit() (err error) {
 		}
 		// We don't instrument exemplar appends here, all is instrumented by storage.
 		if err := a.head.exemplars.AddExemplar(s.lset, e.exemplar); err != nil {
-			if err == storage.ErrOutOfOrderExemplar {
+			if errors.Is(err, storage.ErrOutOfOrderExemplar) {
 				continue
 			}
 			level.Debug(a.head.logger).Log("msg", "Unknown error while adding exemplar", "err", err)
@@ -898,16 +898,16 @@ func (a *headAppender) Commit() (err error) {
 		series.Lock()
 
 		oooSample, _, err := series.appendable(s.T, s.V, a.headMaxt, a.minValidTime, a.oooTimeWindow)
-		switch err {
-		case nil:
+		switch {
+		case err == nil:
 			// Do nothing.
-		case storage.ErrOutOfOrderSample:
+		case errors.Is(err, storage.ErrOutOfOrderSample):
 			samplesAppended--
 			oooRejected++
-		case storage.ErrOutOfBounds:
+		case errors.Is(err, storage.ErrOutOfBounds):
 			samplesAppended--
 			oobRejected++
-		case storage.ErrTooOldSample:
+		case errors.Is(err, storage.ErrTooOldSample):
 			samplesAppended--
 			tooOldRejected++
 		default:
@@ -1487,7 +1487,7 @@ func (s *memSeries) mmapChunks(chunkDiskMapper *chunks.ChunkDiskMapper) (count i
 }
 
 func handleChunkWriteError(err error) {
-	if err != nil && err != chunks.ErrChunkDiskMapperClosed {
+	if err != nil && !errors.Is(err, chunks.ErrChunkDiskMapperClosed) {
 		panic(err)
 	}
 }
diff --git a/tsdb/head_bench_test.go b/tsdb/head_bench_test.go
index 8fdf94db0..a03794810 100644
--- a/tsdb/head_bench_test.go
+++ b/tsdb/head_bench_test.go
@@ -14,10 +14,10 @@
 package tsdb
 
 import (
+	"errors"
 	"strconv"
 	"testing"
 
-	"github.com/pkg/errors"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/atomic"
 
diff --git a/tsdb/head_read.go b/tsdb/head_read.go
index 35ef26a58..362764480 100644
--- a/tsdb/head_read.go
+++ b/tsdb/head_read.go
@@ -15,11 +15,12 @@ package tsdb
 
 import (
 	"context"
+	"errors"
+	"fmt"
 	"math"
 	"sync"
 
 	"github.com/go-kit/log/level"
-	"github.com/pkg/errors"
 	"golang.org/x/exp/slices"
 
 	"github.com/prometheus/prometheus/model/labels"
@@ -133,7 +134,7 @@ func (h *headIndexReader) SortedPostings(p index.Postings) index.Postings {
 		}
 	}
 	if err := p.Err(); err != nil {
-		return index.ErrPostings(errors.Wrap(err, "expand postings"))
+		return index.ErrPostings(fmt.Errorf("expand postings: %w", err))
 	}
 
 	slices.SortFunc(series, func(a, b *memSeries) int {
@@ -388,7 +389,8 @@ func (s *memSeries) chunk(id chunks.HeadChunkID, chunkDiskMapper *chunks.ChunkDi
 	if ix < len(s.mmappedChunks) {
 		chk, err := chunkDiskMapper.Chunk(s.mmappedChunks[ix].ref)
 		if err != nil {
-			if _, ok := err.(*chunks.CorruptionErr); ok {
+			var cerr *chunks.CorruptionErr
+			if errors.As(err, &cerr) {
 				panic(err)
 			}
 			return nil, false, false, err
@@ -516,14 +518,15 @@ func (s *memSeries) oooMergedChunks(meta chunks.Meta, cdm *chunks.ChunkDiskMappe
 				xor, err = s.ooo.oooHeadChunk.chunk.ToXORBetweenTimestamps(meta.OOOLastMinTime, meta.OOOLastMaxTime)
 			}
 			if err != nil {
-				return nil, errors.Wrap(err, "failed to convert ooo head chunk to xor chunk")
+				return nil, fmt.Errorf("failed to convert ooo head chunk to xor chunk: %w", err)
 			}
 			iterable = xor
 		} else {
 			chk, err := cdm.Chunk(c.ref)
 			if err != nil {
-				if _, ok := err.(*chunks.CorruptionErr); ok {
-					return nil, errors.Wrap(err, "invalid ooo mmapped chunk")
+				var cerr *chunks.CorruptionErr
+				if errors.As(err, &cerr) {
+					return nil, fmt.Errorf("invalid ooo mmapped chunk: %w", err)
 				}
 				return nil, err
 			}
diff --git a/tsdb/head_test.go b/tsdb/head_test.go
index 535647d3a..5c2749bed 100644
--- a/tsdb/head_test.go
+++ b/tsdb/head_test.go
@@ -30,7 +30,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	prom_testutil "github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/prometheus/common/model"
@@ -2056,9 +2055,8 @@ func TestWalRepair_DecodingError(t *testing.T) {
 					require.Equal(t, 0.0, prom_testutil.ToFloat64(h.metrics.walCorruptionsTotal))
 					initErr := h.Init(math.MinInt64)
 
-					err = errors.Cause(initErr) // So that we can pick up errors even if wrapped.
-					_, corrErr := err.(*wlog.CorruptionErr)
-					require.True(t, corrErr, "reading the wal didn't return corruption error")
+					var cerr *wlog.CorruptionErr
+					require.ErrorAs(t, initErr, &cerr, "reading the wal didn't return corruption error")
 					require.NoError(t, h.Close()) // Head will close the wal as well.
 				}
 
@@ -2129,12 +2127,11 @@ func TestWblRepair_DecodingError(t *testing.T) {
 		require.Equal(t, 0.0, prom_testutil.ToFloat64(h.metrics.walCorruptionsTotal))
 		initErr := h.Init(math.MinInt64)
 
-		_, ok := initErr.(*errLoadWbl)
-		require.True(t, ok) // Wbl errors are wrapped into errLoadWbl, make sure we can unwrap it.
+		var elb *errLoadWbl
+		require.ErrorAs(t, initErr, &elb) // Wbl errors are wrapped into errLoadWbl, make sure we can unwrap it.
 
-		err = errors.Cause(initErr) // So that we can pick up errors even if wrapped.
-		_, corrErr := err.(*wlog.CorruptionErr)
-		require.True(t, corrErr, "reading the wal didn't return corruption error")
+		var cerr *wlog.CorruptionErr
+		require.ErrorAs(t, initErr, &cerr, "reading the wal didn't return corruption error")
 		require.NoError(t, h.Close()) // Head will close the wal as well.
 	}
 
diff --git a/tsdb/head_wal.go b/tsdb/head_wal.go
index 07fa8280c..a492a85a0 100644
--- a/tsdb/head_wal.go
+++ b/tsdb/head_wal.go
@@ -14,6 +14,7 @@
 package tsdb
 
 import (
+	"errors"
 	"fmt"
 	"math"
 	"os"
@@ -24,7 +25,6 @@ import (
 	"time"
 
 	"github.com/go-kit/log/level"
-	"github.com/pkg/errors"
 	"go.uber.org/atomic"
 
 	"github.com/prometheus/prometheus/model/exemplar"
@@ -128,7 +128,7 @@ func (h *Head) loadWAL(r *wlog.Reader, multiRef map[chunks.HeadSeriesRef]chunks.
 			// At the moment the only possible error here is out of order exemplars, which we shouldn't see when
 			// replaying the WAL, so lets just log the error if it's not that type.
 			err = h.exemplars.AddExemplar(ms.lset, exemplar.Exemplar{Ts: e.T, Value: e.V, Labels: e.Labels})
-			if err != nil && err == storage.ErrOutOfOrderExemplar {
+			if err != nil && errors.Is(err, storage.ErrOutOfOrderExemplar) {
 				level.Warn(h.logger).Log("msg", "Unexpected error when replaying WAL on exemplar record", "err", err)
 			}
 		}
@@ -145,7 +145,7 @@ func (h *Head) loadWAL(r *wlog.Reader, multiRef map[chunks.HeadSeriesRef]chunks.
 				series, err = dec.Series(rec, series)
 				if err != nil {
 					decodeErr = &wlog.CorruptionErr{
-						Err:     errors.Wrap(err, "decode series"),
+						Err:     fmt.Errorf("decode series: %w", err),
 						Segment: r.Segment(),
 						Offset:  r.Offset(),
 					}
@@ -157,7 +157,7 @@ func (h *Head) loadWAL(r *wlog.Reader, multiRef map[chunks.HeadSeriesRef]chunks.
 				samples, err = dec.Samples(rec, samples)
 				if err != nil {
 					decodeErr = &wlog.CorruptionErr{
-						Err:     errors.Wrap(err, "decode samples"),
+						Err:     fmt.Errorf("decode samples: %w", err),
 						Segment: r.Segment(),
 						Offset:  r.Offset(),
 					}
@@ -169,7 +169,7 @@ func (h *Head) loadWAL(r *wlog.Reader, multiRef map[chunks.HeadSeriesRef]chunks.
 				tstones, err = dec.Tombstones(rec, tstones)
 				if err != nil {
 					decodeErr = &wlog.CorruptionErr{
-						Err:     errors.Wrap(err, "decode tombstones"),
+						Err:     fmt.Errorf("decode tombstones: %w", err),
 						Segment: r.Segment(),
 						Offset:  r.Offset(),
 					}
@@ -181,7 +181,7 @@ func (h *Head) loadWAL(r *wlog.Reader, multiRef map[chunks.HeadSeriesRef]chunks.
 				exemplars, err = dec.Exemplars(rec, exemplars)
 				if err != nil {
 					decodeErr = &wlog.CorruptionErr{
-						Err:     errors.Wrap(err, "decode exemplars"),
+						Err:     fmt.Errorf("decode exemplars: %w", err),
 						Segment: r.Segment(),
 						Offset:  r.Offset(),
 					}
@@ -193,7 +193,7 @@ func (h *Head) loadWAL(r *wlog.Reader, multiRef map[chunks.HeadSeriesRef]chunks.
 				hists, err = dec.HistogramSamples(rec, hists)
 				if err != nil {
 					decodeErr = &wlog.CorruptionErr{
-						Err:     errors.Wrap(err, "decode histograms"),
+						Err:     fmt.Errorf("decode histograms: %w", err),
 						Segment: r.Segment(),
 						Offset:  r.Offset(),
 					}
@@ -205,7 +205,7 @@ func (h *Head) loadWAL(r *wlog.Reader, multiRef map[chunks.HeadSeriesRef]chunks.
 				hists, err = dec.FloatHistogramSamples(rec, hists)
 				if err != nil {
 					decodeErr = &wlog.CorruptionErr{
-						Err:     errors.Wrap(err, "decode float histograms"),
+						Err:     fmt.Errorf("decode float histograms: %w", err),
 						Segment: r.Segment(),
 						Offset:  r.Offset(),
 					}
@@ -217,7 +217,7 @@ func (h *Head) loadWAL(r *wlog.Reader, multiRef map[chunks.HeadSeriesRef]chunks.
 				meta, err := dec.Metadata(rec, meta)
 				if err != nil {
 					decodeErr = &wlog.CorruptionErr{
-						Err:     errors.Wrap(err, "decode metadata"),
+						Err:     fmt.Errorf("decode metadata: %w", err),
 						Segment: r.Segment(),
 						Offset:  r.Offset(),
 					}
@@ -416,8 +416,8 @@ Outer:
 	close(exemplarsInput)
 	wg.Wait()
 
-	if r.Err() != nil {
-		return errors.Wrap(r.Err(), "read records")
+	if err := r.Err(); err != nil {
+		return fmt.Errorf("read records: %w", err)
 	}
 
 	if unknownRefs.Load()+unknownExemplarRefs.Load()+unknownHistogramRefs.Load()+unknownMetadataRefs.Load() > 0 {
@@ -708,7 +708,7 @@ func (h *Head) loadWBL(r *wlog.Reader, multiRef map[chunks.HeadSeriesRef]chunks.
 				samples, err = dec.Samples(rec, samples)
 				if err != nil {
 					decodeErr = &wlog.CorruptionErr{
-						Err:     errors.Wrap(err, "decode samples"),
+						Err:     fmt.Errorf("decode samples: %w", err),
 						Segment: r.Segment(),
 						Offset:  r.Offset(),
 					}
@@ -720,7 +720,7 @@ func (h *Head) loadWBL(r *wlog.Reader, multiRef map[chunks.HeadSeriesRef]chunks.
 				markers, err = dec.MmapMarkers(rec, markers)
 				if err != nil {
 					decodeErr = &wlog.CorruptionErr{
-						Err:     errors.Wrap(err, "decode mmap markers"),
+						Err:     fmt.Errorf("decode mmap markers: %w", err),
 						Segment: r.Segment(),
 						Offset:  r.Offset(),
 					}
@@ -806,8 +806,8 @@ func (h *Head) loadWBL(r *wlog.Reader, multiRef map[chunks.HeadSeriesRef]chunks.
 	}
 	wg.Wait()
 
-	if r.Err() != nil {
-		return errors.Wrap(r.Err(), "read records")
+	if err := r.Err(); err != nil {
+		return fmt.Errorf("read records: %w", err)
 	}
 
 	if unknownRefs.Load() > 0 || mmapMarkerUnknownRefs.Load() > 0 {
@@ -995,7 +995,7 @@ func decodeSeriesFromChunkSnapshot(d *record.Decoder, b []byte) (csr chunkSnapsh
 
 	chk, err := chunkenc.FromData(enc, chunkBytesCopy)
 	if err != nil {
-		return csr, errors.Wrap(err, "chunk from data")
+		return csr, fmt.Errorf("chunk from data: %w", err)
 	}
 	csr.mc.chunk = chk
 
@@ -1030,7 +1030,7 @@ func encodeTombstonesToSnapshotRecord(tr tombstones.Reader) ([]byte, error) {
 	buf.PutByte(chunkSnapshotRecordTypeTombstones)
 	b, err := tombstones.Encode(tr)
 	if err != nil {
-		return nil, errors.Wrap(err, "encode tombstones")
+		return nil, fmt.Errorf("encode tombstones: %w", err)
 	}
 	buf.PutUvarintBytes(b)
 
@@ -1045,7 +1045,10 @@ func decodeTombstonesSnapshotRecord(b []byte) (tombstones.Reader, error) {
 	}
 
 	tr, err := tombstones.Decode(dec.UvarintBytes())
-	return tr, errors.Wrap(err, "decode tombstones")
+	if err != nil {
+		return tr, fmt.Errorf("decode tombstones: %w", err)
+	}
+	return tr, nil
 }
 
 const chunkSnapshotPrefix = "chunk_snapshot."
@@ -1072,13 +1075,13 @@ func (h *Head) ChunkSnapshot() (*ChunkSnapshotStats, error) {
 	stats := &ChunkSnapshotStats{}
 
 	wlast, woffset, err := h.wal.LastSegmentAndOffset()
-	if err != nil && err != record.ErrNotFound {
-		return stats, errors.Wrap(err, "get last wal segment and offset")
+	if err != nil && !errors.Is(err, record.ErrNotFound) {
+		return stats, fmt.Errorf("get last wal segment and offset: %w", err)
 	}
 
 	_, cslast, csoffset, err := LastChunkSnapshot(h.opts.ChunkDirRoot)
-	if err != nil && err != record.ErrNotFound {
-		return stats, errors.Wrap(err, "find last chunk snapshot")
+	if err != nil && !errors.Is(err, record.ErrNotFound) {
+		return stats, fmt.Errorf("find last chunk snapshot: %w", err)
 	}
 
 	if wlast == cslast && woffset == csoffset {
@@ -1093,11 +1096,11 @@ func (h *Head) ChunkSnapshot() (*ChunkSnapshotStats, error) {
 	stats.Dir = cpdir
 
 	if err := os.MkdirAll(cpdirtmp, 0o777); err != nil {
-		return stats, errors.Wrap(err, "create chunk snapshot dir")
+		return stats, fmt.Errorf("create chunk snapshot dir: %w", err)
 	}
 	cp, err := wlog.New(nil, nil, cpdirtmp, h.wal.CompressionType())
 	if err != nil {
-		return stats, errors.Wrap(err, "open chunk snapshot")
+		return stats, fmt.Errorf("open chunk snapshot: %w", err)
 	}
 
 	// Ensures that an early return caused by an error doesn't leave any tmp files.
@@ -1126,7 +1129,7 @@ func (h *Head) ChunkSnapshot() (*ChunkSnapshotStats, error) {
 			if len(buf) > 10*1024*1024 {
 				if err := cp.Log(recs...); err != nil {
 					h.series.locks[i].RUnlock()
-					return stats, errors.Wrap(err, "flush records")
+					return stats, fmt.Errorf("flush records: %w", err)
 				}
 				buf, recs = buf[:0], recs[:0]
 			}
@@ -1139,16 +1142,16 @@ func (h *Head) ChunkSnapshot() (*ChunkSnapshotStats, error) {
 	// Add tombstones to the snapshot.
 	tombstonesReader, err := h.Tombstones()
 	if err != nil {
-		return stats, errors.Wrap(err, "get tombstones")
+		return stats, fmt.Errorf("get tombstones: %w", err)
 	}
 	rec, err := encodeTombstonesToSnapshotRecord(tombstonesReader)
 	if err != nil {
-		return stats, errors.Wrap(err, "encode tombstones")
+		return stats, fmt.Errorf("encode tombstones: %w", err)
 	}
 	recs = append(recs, rec)
 	// Flush remaining series records and tombstones.
 	if err := cp.Log(recs...); err != nil {
-		return stats, errors.Wrap(err, "flush records")
+		return stats, fmt.Errorf("flush records: %w", err)
 	}
 	buf = buf[:0]
 
@@ -1167,7 +1170,7 @@ func (h *Head) ChunkSnapshot() (*ChunkSnapshotStats, error) {
 		encbuf.PutByte(chunkSnapshotRecordTypeExemplars)
 		enc.EncodeExemplarsIntoBuffer(batch, &encbuf)
 		if err := cp.Log(encbuf.Get()); err != nil {
-			return errors.Wrap(err, "log exemplars")
+			return fmt.Errorf("log exemplars: %w", err)
 		}
 		buf, batch = buf[:0], batch[:0]
 		return nil
@@ -1175,7 +1178,7 @@ func (h *Head) ChunkSnapshot() (*ChunkSnapshotStats, error) {
 	err = h.exemplars.IterateExemplars(func(seriesLabels labels.Labels, e exemplar.Exemplar) error {
 		if len(batch) >= maxExemplarsPerRecord {
 			if err := flushExemplars(); err != nil {
-				return errors.Wrap(err, "flush exemplars")
+				return fmt.Errorf("flush exemplars: %w", err)
 			}
 		}
 
@@ -1193,19 +1196,19 @@ func (h *Head) ChunkSnapshot() (*ChunkSnapshotStats, error) {
 		return nil
 	})
 	if err != nil {
-		return stats, errors.Wrap(err, "iterate exemplars")
+		return stats, fmt.Errorf("iterate exemplars: %w", err)
 	}
 
 	// Flush remaining exemplars.
 	if err := flushExemplars(); err != nil {
-		return stats, errors.Wrap(err, "flush exemplars at the end")
+		return stats, fmt.Errorf("flush exemplars at the end: %w", err)
 	}
 
 	if err := cp.Close(); err != nil {
-		return stats, errors.Wrap(err, "close chunk snapshot")
+		return stats, fmt.Errorf("close chunk snapshot: %w", err)
 	}
 	if err := fileutil.Replace(cpdirtmp, cpdir); err != nil {
-		return stats, errors.Wrap(err, "rename chunk snapshot directory")
+		return stats, fmt.Errorf("rename chunk snapshot directory: %w", err)
 	}
 
 	if err := DeleteChunkSnapshots(h.opts.ChunkDirRoot, wlast, woffset); err != nil {
@@ -1229,7 +1232,10 @@ func (h *Head) performChunkSnapshot() error {
 	if err == nil {
 		level.Info(h.logger).Log("msg", "chunk snapshot complete", "duration", elapsed.String(), "num_series", stats.TotalSeries, "dir", stats.Dir)
 	}
-	return errors.Wrap(err, "chunk snapshot")
+	if err != nil {
+		return fmt.Errorf("chunk snapshot: %w", err)
+	}
+	return nil
 }
 
 // ChunkSnapshotStats returns stats about a created chunk snapshot.
@@ -1327,16 +1333,16 @@ func DeleteChunkSnapshots(dir string, maxIndex, maxOffset int) error {
 func (h *Head) loadChunkSnapshot() (int, int, map[chunks.HeadSeriesRef]*memSeries, error) {
 	dir, snapIdx, snapOffset, err := LastChunkSnapshot(h.opts.ChunkDirRoot)
 	if err != nil {
-		if err == record.ErrNotFound {
+		if errors.Is(err, record.ErrNotFound) {
 			return snapIdx, snapOffset, nil, nil
 		}
-		return snapIdx, snapOffset, nil, errors.Wrap(err, "find last chunk snapshot")
+		return snapIdx, snapOffset, nil, fmt.Errorf("find last chunk snapshot: %w", err)
 	}
 
 	start := time.Now()
 	sr, err := wlog.NewSegmentsReader(dir)
 	if err != nil {
-		return snapIdx, snapOffset, nil, errors.Wrap(err, "open chunk snapshot")
+		return snapIdx, snapOffset, nil, fmt.Errorf("open chunk snapshot: %w", err)
 	}
 	defer func() {
 		if err := sr.Close(); err != nil {
@@ -1424,7 +1430,7 @@ Outer:
 			numSeries++
 			csr, err := decodeSeriesFromChunkSnapshot(&dec, rec)
 			if err != nil {
-				loopErr = errors.Wrap(err, "decode series record")
+				loopErr = fmt.Errorf("decode series record: %w", err)
 				break Outer
 			}
 			recordChan <- csr
@@ -1432,7 +1438,7 @@ Outer:
 		case chunkSnapshotRecordTypeTombstones:
 			tr, err := decodeTombstonesSnapshotRecord(rec)
 			if err != nil {
-				loopErr = errors.Wrap(err, "decode tombstones")
+				loopErr = fmt.Errorf("decode tombstones: %w", err)
 				break Outer
 			}
 
@@ -1440,7 +1446,7 @@ Outer:
 				h.tombstones.AddInterval(ref, ivs...)
 				return nil
 			}); err != nil {
-				loopErr = errors.Wrap(err, "iterate tombstones")
+				loopErr = fmt.Errorf("iterate tombstones: %w", err)
 				break Outer
 			}
 
@@ -1468,7 +1474,7 @@ Outer:
 			exemplarBuf = exemplarBuf[:0]
 			exemplarBuf, err = dec.ExemplarsFromBuffer(&decbuf, exemplarBuf)
 			if err != nil {
-				loopErr = errors.Wrap(err, "exemplars from buffer")
+				loopErr = fmt.Errorf("exemplars from buffer: %w", err)
 				break Outer
 			}
 
@@ -1484,7 +1490,7 @@ Outer:
 					Value:  e.V,
 					Ts:     e.T,
 				}); err != nil {
-					loopErr = errors.Wrap(err, "add exemplar")
+					loopErr = fmt.Errorf("add exemplar: %w", err)
 					break Outer
 				}
 			}
@@ -1502,16 +1508,19 @@ Outer:
 	}
 
 	close(errChan)
-	merr := tsdb_errors.NewMulti(errors.Wrap(loopErr, "decode loop"))
+	merr := tsdb_errors.NewMulti()
+	if loopErr != nil {
+		merr.Add(fmt.Errorf("decode loop: %w", loopErr))
+	}
 	for err := range errChan {
-		merr.Add(errors.Wrap(err, "record processing"))
+		merr.Add(fmt.Errorf("record processing: %w", err))
 	}
 	if err := merr.Err(); err != nil {
 		return -1, -1, nil, err
 	}
 
-	if r.Err() != nil {
-		return -1, -1, nil, errors.Wrap(r.Err(), "read records")
+	if err := r.Err(); err != nil {
+		return -1, -1, nil, fmt.Errorf("read records: %w", err)
 	}
 
 	if len(refSeries) == 0 {
diff --git a/tsdb/index/index.go b/tsdb/index/index.go
index 44ee66386..2b025a352 100644
--- a/tsdb/index/index.go
+++ b/tsdb/index/index.go
@@ -425,7 +425,7 @@ func (w *Writer) AddSeries(ref storage.SeriesRef, lset labels.Labels, chunks ...
 	// We add padding to 16 bytes to increase the addressable space we get through 4 byte
 	// series references.
 	if err := w.addPadding(16); err != nil {
-		return fmt.Errorf("failed to write padding bytes: %v", err)
+		return fmt.Errorf("failed to write padding bytes: %w", err)
 	}
 
 	if w.f.pos%16 != 0 {
@@ -442,7 +442,7 @@ func (w *Writer) AddSeries(ref storage.SeriesRef, lset labels.Labels, chunks ...
 		if !ok {
 			nameIndex, err = w.symbols.ReverseLookup(l.Name)
 			if err != nil {
-				return fmt.Errorf("symbol entry for %q does not exist, %v", l.Name, err)
+				return fmt.Errorf("symbol entry for %q does not exist, %w", l.Name, err)
 			}
 		}
 		w.labelNames[l.Name]++
@@ -452,7 +452,7 @@ func (w *Writer) AddSeries(ref storage.SeriesRef, lset labels.Labels, chunks ...
 		if !ok || cacheEntry.lastValue != l.Value {
 			valueIndex, err = w.symbols.ReverseLookup(l.Value)
 			if err != nil {
-				return fmt.Errorf("symbol entry for %q does not exist, %v", l.Value, err)
+				return fmt.Errorf("symbol entry for %q does not exist, %w", l.Value, err)
 			}
 			w.symbolCache[l.Name] = symbolCacheEntry{
 				index:          nameIndex,
diff --git a/tsdb/mocks_test.go b/tsdb/mocks_test.go
index 268017caa..d7c2b0a4f 100644
--- a/tsdb/mocks_test.go
+++ b/tsdb/mocks_test.go
@@ -14,7 +14,7 @@
 package tsdb
 
 import (
-	"github.com/pkg/errors"
+	"fmt"
 
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/storage"
@@ -41,7 +41,7 @@ func (m *mockIndexWriter) AddSeries(_ storage.SeriesRef, l labels.Labels, chks .
 	for i, chk := range chks {
 		c, err := copyChunk(chk.Chunk)
 		if err != nil {
-			return errors.Wrap(err, "mockIndexWriter: copy chunk")
+			return fmt.Errorf("mockIndexWriter: copy chunk: %w", err)
 		}
 		chksNew[i] = chunks.Meta{MaxTime: chk.MaxTime, MinTime: chk.MinTime, Chunk: c}
 	}
diff --git a/tsdb/querier.go b/tsdb/querier.go
index 6584d7da0..f88e4415e 100644
--- a/tsdb/querier.go
+++ b/tsdb/querier.go
@@ -15,13 +15,13 @@ package tsdb
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"math"
 	"strings"
 	"unicode/utf8"
 
 	"github.com/oklog/ulid"
-	"github.com/pkg/errors"
 	"golang.org/x/exp/slices"
 
 	"github.com/prometheus/prometheus/model/histogram"
@@ -63,18 +63,18 @@ type blockBaseQuerier struct {
 func newBlockBaseQuerier(b BlockReader, mint, maxt int64) (*blockBaseQuerier, error) {
 	indexr, err := b.Index()
 	if err != nil {
-		return nil, errors.Wrap(err, "open index reader")
+		return nil, fmt.Errorf("open index reader: %w", err)
 	}
 	chunkr, err := b.Chunks()
 	if err != nil {
 		indexr.Close()
-		return nil, errors.Wrap(err, "open chunk reader")
+		return nil, fmt.Errorf("open chunk reader: %w", err)
 	}
 	tombsr, err := b.Tombstones()
 	if err != nil {
 		indexr.Close()
 		chunkr.Close()
-		return nil, errors.Wrap(err, "open tombstone reader")
+		return nil, fmt.Errorf("open tombstone reader: %w", err)
 	}
 
 	if tombsr == nil {
@@ -442,12 +442,12 @@ func inversePostingsForMatcher(ctx context.Context, ix IndexReader, m *labels.Ma
 func labelValuesWithMatchers(ctx context.Context, r IndexReader, name string, matchers ...*labels.Matcher) ([]string, error) {
 	p, err := PostingsForMatchers(ctx, r, matchers...)
 	if err != nil {
-		return nil, errors.Wrap(err, "fetching postings for matchers")
+		return nil, fmt.Errorf("fetching postings for matchers: %w", err)
 	}
 
 	allValues, err := r.LabelValues(ctx, name)
 	if err != nil {
-		return nil, errors.Wrapf(err, "fetching values of label %s", name)
+		return nil, fmt.Errorf("fetching values of label %s: %w", name, err)
 	}
 
 	// If we have a matcher for the label name, we can filter out values that don't match
@@ -473,12 +473,12 @@ func labelValuesWithMatchers(ctx context.Context, r IndexReader, name string, ma
 	for i, value := range allValues {
 		valuesPostings[i], err = r.Postings(ctx, name, value)
 		if err != nil {
-			return nil, errors.Wrapf(err, "fetching postings for %s=%q", name, value)
+			return nil, fmt.Errorf("fetching postings for %s=%q: %w", name, value, err)
 		}
 	}
 	indexes, err := index.FindIntersectingPostings(p, valuesPostings)
 	if err != nil {
-		return nil, errors.Wrap(err, "intersecting postings")
+		return nil, fmt.Errorf("intersecting postings: %w", err)
 	}
 
 	values := make([]string, 0, len(indexes))
@@ -499,8 +499,8 @@ func labelNamesWithMatchers(ctx context.Context, r IndexReader, matchers ...*lab
 	for p.Next() {
 		postings = append(postings, p.At())
 	}
-	if p.Err() != nil {
-		return nil, errors.Wrapf(p.Err(), "postings for label names with matchers")
+	if err := p.Err(); err != nil {
+		return nil, fmt.Errorf("postings for label names with matchers: %w", err)
 	}
 
 	return r.LabelNamesFor(ctx, postings...)
@@ -539,10 +539,10 @@ func (b *blockBaseSeriesSet) Next() bool {
 	for b.p.Next() {
 		if err := b.index.Series(b.p.At(), &b.builder, &b.bufChks); err != nil {
 			// Postings may be stale. Skip if no underlying series exists.
-			if errors.Cause(err) == storage.ErrNotFound {
+			if errors.Is(err, storage.ErrNotFound) {
 				continue
 			}
-			b.err = errors.Wrapf(err, "get series %d", b.p.At())
+			b.err = fmt.Errorf("get series %d: %w", b.p.At(), err)
 			return false
 		}
 
@@ -552,7 +552,7 @@ func (b *blockBaseSeriesSet) Next() bool {
 
 		intervals, err := b.tombstones.Get(b.p.At())
 		if err != nil {
-			b.err = errors.Wrap(err, "get tombstones")
+			b.err = fmt.Errorf("get tombstones: %w", err)
 			return false
 		}
 
@@ -702,7 +702,7 @@ func (p *populateWithDelGenericSeriesIterator) next(copyHeadChunk bool) bool {
 	}
 
 	if p.err != nil {
-		p.err = errors.Wrapf(p.err, "cannot populate chunk %d from block %s", p.currMeta.Ref, p.blockID.String())
+		p.err = fmt.Errorf("cannot populate chunk %d from block %s: %w", p.currMeta.Ref, p.blockID.String(), p.err)
 		return false
 	}
 
@@ -900,7 +900,7 @@ func (p *populateWithDelChunkSeriesIterator) populateCurrForSingleChunk() bool {
 	valueType := p.currDelIter.Next()
 	if valueType == chunkenc.ValNone {
 		if err := p.currDelIter.Err(); err != nil {
-			p.err = errors.Wrap(err, "iterate chunk while re-encoding")
+			p.err = fmt.Errorf("iterate chunk while re-encoding: %w", err)
 		}
 		return false
 	}
@@ -968,11 +968,11 @@ func (p *populateWithDelChunkSeriesIterator) populateCurrForSingleChunk() bool {
 	}
 
 	if err != nil {
-		p.err = errors.Wrap(err, "iterate chunk while re-encoding")
+		p.err = fmt.Errorf("iterate chunk while re-encoding: %w", err)
 		return false
 	}
 	if err := p.currDelIter.Err(); err != nil {
-		p.err = errors.Wrap(err, "iterate chunk while re-encoding")
+		p.err = fmt.Errorf("iterate chunk while re-encoding: %w", err)
 		return false
 	}
 
@@ -991,7 +991,7 @@ func (p *populateWithDelChunkSeriesIterator) populateChunksFromIterable() bool {
 	firstValueType := p.currDelIter.Next()
 	if firstValueType == chunkenc.ValNone {
 		if err := p.currDelIter.Err(); err != nil {
-			p.err = errors.Wrap(err, "populateChunksFromIterable: no samples could be read")
+			p.err = fmt.Errorf("populateChunksFromIterable: no samples could be read: %w", err)
 			return false
 		}
 		return false
@@ -1075,11 +1075,11 @@ func (p *populateWithDelChunkSeriesIterator) populateChunksFromIterable() bool {
 	}
 
 	if err != nil {
-		p.err = errors.Wrap(err, "populateChunksFromIterable: error when writing new chunks")
+		p.err = fmt.Errorf("populateChunksFromIterable: error when writing new chunks: %w", err)
 		return false
 	}
 	if err = p.currDelIter.Err(); err != nil {
-		p.err = errors.Wrap(err, "populateChunksFromIterable: currDelIter error when writing new chunks")
+		p.err = fmt.Errorf("populateChunksFromIterable: currDelIter error when writing new chunks: %w", err)
 		return false
 	}
 
diff --git a/tsdb/querier_test.go b/tsdb/querier_test.go
index 64e16392d..18d81b85b 100644
--- a/tsdb/querier_test.go
+++ b/tsdb/querier_test.go
@@ -15,6 +15,7 @@ package tsdb
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"math"
 	"math/rand"
@@ -26,7 +27,6 @@ import (
 	"time"
 
 	"github.com/oklog/ulid"
-	"github.com/pkg/errors"
 	"github.com/stretchr/testify/require"
 
 	"github.com/prometheus/prometheus/model/histogram"
@@ -2317,7 +2317,7 @@ func (m mockIndex) Postings(ctx context.Context, name string, values ...string)
 func (m mockIndex) SortedPostings(p index.Postings) index.Postings {
 	ep, err := index.ExpandPostings(p)
 	if err != nil {
-		return index.ErrPostings(errors.Wrap(err, "expand postings"))
+		return index.ErrPostings(fmt.Errorf("expand postings: %w", err))
 	}
 
 	sort.Slice(ep, func(i, j int) bool {
diff --git a/tsdb/repair.go b/tsdb/repair.go
index 081116454..9d2c5738d 100644
--- a/tsdb/repair.go
+++ b/tsdb/repair.go
@@ -22,7 +22,6 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
-	"github.com/pkg/errors"
 
 	tsdb_errors "github.com/prometheus/prometheus/tsdb/errors"
 	"github.com/prometheus/prometheus/tsdb/fileutil"
@@ -35,7 +34,7 @@ func repairBadIndexVersion(logger log.Logger, dir string) error {
 	// We must actually set the index file version to 2 and revert the meta.json version back to 1.
 	dirs, err := blockDirs(dir)
 	if err != nil {
-		return errors.Wrapf(err, "list block dirs in %q", dir)
+		return fmt.Errorf("list block dirs in %q: %w", dir, err)
 	}
 
 	tmpFiles := make([]string, 0, len(dirs))
@@ -71,44 +70,54 @@ func repairBadIndexVersion(logger log.Logger, dir string) error {
 
 		repl, err := os.Create(filepath.Join(d, "index.repaired"))
 		if err != nil {
-			return errors.Wrapf(err, "create index.repaired for block dir: %v", d)
+			return fmt.Errorf("create index.repaired for block dir: %v: %w", d, err)
 		}
 		tmpFiles = append(tmpFiles, repl.Name())
 
 		broken, err := os.Open(filepath.Join(d, indexFilename))
 		if err != nil {
-			return errors.Wrapf(err, "open broken index for block dir: %v", d)
+			return fmt.Errorf("open broken index for block dir: %v: %w", d, err)
 		}
 		if _, err := io.Copy(repl, broken); err != nil {
-			return errors.Wrapf(err, "copy content of index to index.repaired for block dir: %v", d)
+			return fmt.Errorf("copy content of index to index.repaired for block dir: %v: %w", d, err)
 		}
 
 		// Set the 5th byte to 2 to indicate the correct file format version.
 		if _, err := repl.WriteAt([]byte{2}, 4); err != nil {
-			return tsdb_errors.NewMulti(
-				errors.Wrapf(err, "rewrite of index.repaired for block dir: %v", d),
-				errors.Wrap(repl.Close(), "close"),
-			).Err()
+			errs := tsdb_errors.NewMulti(
+				fmt.Errorf("rewrite of index.repaired for block dir: %v: %w", d, err))
+			if err := repl.Close(); err != nil {
+				errs.Add(fmt.Errorf("close: %w", err))
+			}
+			return errs.Err()
 		}
 		if err := repl.Sync(); err != nil {
-			return tsdb_errors.NewMulti(
-				errors.Wrapf(err, "sync of index.repaired for block dir: %v", d),
-				errors.Wrap(repl.Close(), "close"),
-			).Err()
+			errs := tsdb_errors.NewMulti(
+				fmt.Errorf("sync of index.repaired for block dir: %v: %w", d, err))
+			if err := repl.Close(); err != nil {
+				errs.Add(fmt.Errorf("close: %w", err))
+			}
+			return errs.Err()
 		}
 		if err := repl.Close(); err != nil {
-			return errors.Wrapf(repl.Close(), "close repaired index for block dir: %v", d)
+			return fmt.Errorf("close repaired index for block dir: %v: %w", d, err)
 		}
 		if err := broken.Close(); err != nil {
-			return errors.Wrapf(repl.Close(), "close broken index for block dir: %v", d)
+			if err := repl.Close(); err != nil {
+				return fmt.Errorf("close broken index for block dir: %v: %w", d, err)
+			}
 		}
 		if err := fileutil.Replace(repl.Name(), broken.Name()); err != nil {
-			return errors.Wrapf(repl.Close(), "replaced broken index with index.repaired for block dir: %v", d)
+			if err := repl.Close(); err != nil {
+				return fmt.Errorf("replaced broken index with index.repaired for block dir: %v: %w", d, err)
+			}
 		}
 		// Reset version of meta.json to 1.
 		meta.Version = metaVersion1
 		if _, err := writeMetaFile(logger, d, meta); err != nil {
-			return errors.Wrapf(repl.Close(), "write meta for block dir: %v", d)
+			if err := repl.Close(); err != nil {
+				return fmt.Errorf("write meta for block dir: %v: %w", d, err)
+			}
 		}
 	}
 	return nil
diff --git a/tsdb/wal.go b/tsdb/wal.go
index bc7db35bf..1509c9cd9 100644
--- a/tsdb/wal.go
+++ b/tsdb/wal.go
@@ -16,6 +16,7 @@ package tsdb
 import (
 	"bufio"
 	"encoding/binary"
+	"errors"
 	"fmt"
 	"hash"
 	"hash/crc32"
@@ -28,7 +29,6 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
-	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/prometheus/prometheus/storage"
@@ -210,7 +210,7 @@ func OpenSegmentWAL(dir string, logger log.Logger, flushInterval time.Duration,
 
 		for _, fn := range fns[i:] {
 			if err := os.Remove(fn); err != nil {
-				return w, errors.Wrap(err, "removing segment failed")
+				return w, fmt.Errorf("removing segment failed: %w", err)
 			}
 		}
 		break
@@ -237,8 +237,8 @@ func (r *repairingWALReader) Read(
 	if err == nil {
 		return nil
 	}
-	cerr, ok := errors.Cause(err).(walCorruptionErr)
-	if !ok {
+	var cerr *walCorruptionErr
+	if !errors.As(err, &cerr) {
 		return err
 	}
 	r.wal.metrics.corruptions.Inc()
@@ -309,7 +309,7 @@ func (w *SegmentWAL) Truncate(mint int64, keep func(chunks.HeadSeriesRef) bool)
 		// Past WAL files are closed. We have to reopen them for another read.
 		f, err := w.openSegmentFile(sf.Name())
 		if err != nil {
-			return errors.Wrap(err, "open old WAL segment for read")
+			return fmt.Errorf("open old WAL segment for read: %w", err)
 		}
 		candidates = append(candidates, &segmentFile{
 			File:      f,
@@ -326,7 +326,7 @@ func (w *SegmentWAL) Truncate(mint int64, keep func(chunks.HeadSeriesRef) bool)
 	// Create a new tmp file.
 	f, err := w.createSegmentFile(filepath.Join(w.dirFile.Name(), "compact.tmp"))
 	if err != nil {
-		return errors.Wrap(err, "create compaction segment")
+		return fmt.Errorf("create compaction segment: %w", err)
 	}
 	defer func() {
 		if err := os.RemoveAll(f.Name()); err != nil {
@@ -352,7 +352,7 @@ func (w *SegmentWAL) Truncate(mint int64, keep func(chunks.HeadSeriesRef) bool)
 
 		err := r.decodeSeries(flag, byt, &decSeries)
 		if err != nil {
-			return errors.Wrap(err, "decode samples while truncating")
+			return fmt.Errorf("decode samples while truncating: %w", err)
 		}
 		for _, s := range decSeries {
 			if keep(s.Ref) {
@@ -367,11 +367,11 @@ func (w *SegmentWAL) Truncate(mint int64, keep func(chunks.HeadSeriesRef) bool)
 		w.putBuffer(buf)
 
 		if err != nil {
-			return errors.Wrap(err, "write to compaction segment")
+			return fmt.Errorf("write to compaction segment: %w", err)
 		}
 	}
-	if r.Err() != nil {
-		return errors.Wrap(r.Err(), "read candidate WAL files")
+	if err := r.Err(); err != nil {
+		return fmt.Errorf("read candidate WAL files: %w", err)
 	}
 
 	off, err := csf.Seek(0, io.SeekCurrent)
@@ -390,12 +390,12 @@ func (w *SegmentWAL) Truncate(mint int64, keep func(chunks.HeadSeriesRef) bool)
 
 	_ = candidates[0].Close() // need close before remove on platform windows
 	if err := fileutil.Replace(csf.Name(), candidates[0].Name()); err != nil {
-		return errors.Wrap(err, "rename compaction segment")
+		return fmt.Errorf("rename compaction segment: %w", err)
 	}
 	for _, f := range candidates[1:] {
 		f.Close() // need close before remove on platform windows
 		if err := os.RemoveAll(f.Name()); err != nil {
-			return errors.Wrap(err, "delete WAL segment file")
+			return fmt.Errorf("delete WAL segment file: %w", err)
 		}
 	}
 	if err := w.dirFile.Sync(); err != nil {
@@ -435,7 +435,7 @@ func (w *SegmentWAL) LogSeries(series []record.RefSeries) error {
 	w.putBuffer(buf)
 
 	if err != nil {
-		return errors.Wrap(err, "log series")
+		return fmt.Errorf("log series: %w", err)
 	}
 
 	tf := w.head()
@@ -462,7 +462,7 @@ func (w *SegmentWAL) LogSamples(samples []record.RefSample) error {
 	w.putBuffer(buf)
 
 	if err != nil {
-		return errors.Wrap(err, "log series")
+		return fmt.Errorf("log series: %w", err)
 	}
 	tf := w.head()
 
@@ -488,7 +488,7 @@ func (w *SegmentWAL) LogDeletes(stones []tombstones.Stone) error {
 	w.putBuffer(buf)
 
 	if err != nil {
-		return errors.Wrap(err, "log series")
+		return fmt.Errorf("log series: %w", err)
 	}
 	tf := w.head()
 
@@ -523,7 +523,7 @@ func (w *SegmentWAL) openSegmentFile(name string) (*os.File, error) {
 
 	switch n, err := f.Read(metab); {
 	case err != nil:
-		return nil, errors.Wrapf(err, "validate meta %q", f.Name())
+		return nil, fmt.Errorf("validate meta %q: %w", f.Name(), err)
 	case n != 8:
 		return nil, fmt.Errorf("invalid header size %d in %q", n, f.Name())
 	}
@@ -573,16 +573,16 @@ func (w *SegmentWAL) cut() error {
 			w.actorc <- func() error {
 				off, err := hf.Seek(0, io.SeekCurrent)
 				if err != nil {
-					return errors.Wrapf(err, "finish old segment %s", hf.Name())
+					return fmt.Errorf("finish old segment %s: %w", hf.Name(), err)
 				}
 				if err := hf.Truncate(off); err != nil {
-					return errors.Wrapf(err, "finish old segment %s", hf.Name())
+					return fmt.Errorf("finish old segment %s: %w", hf.Name(), err)
 				}
 				if err := hf.Sync(); err != nil {
-					return errors.Wrapf(err, "finish old segment %s", hf.Name())
+					return fmt.Errorf("finish old segment %s: %w", hf.Name(), err)
 				}
 				if err := hf.Close(); err != nil {
-					return errors.Wrapf(err, "finish old segment %s", hf.Name())
+					return fmt.Errorf("finish old segment %s: %w", hf.Name(), err)
 				}
 				return nil
 			}
@@ -600,7 +600,10 @@ func (w *SegmentWAL) cut() error {
 
 	go func() {
 		w.actorc <- func() error {
-			return errors.Wrap(w.dirFile.Sync(), "sync WAL directory")
+			if err := w.dirFile.Sync(); err != nil {
+				return fmt.Errorf("sync WAL directory: %w", err)
+			}
+			return nil
 		}
 	}()
 
@@ -635,7 +638,7 @@ func (w *SegmentWAL) Sync() error {
 		head = w.head()
 	}()
 	if err != nil {
-		return errors.Wrap(err, "flush buffer")
+		return fmt.Errorf("flush buffer: %w", err)
 	}
 	if head != nil {
 		// But only fsync the head segment after releasing the mutex as it will block on disk I/O.
@@ -726,11 +729,13 @@ func (w *SegmentWAL) Close() error {
 	// only the current segment will still be open.
 	if hf := w.head(); hf != nil {
 		if err := hf.Close(); err != nil {
-			return errors.Wrapf(err, "closing WAL head %s", hf.Name())
+			return fmt.Errorf("closing WAL head %s: %w", hf.Name(), err)
 		}
 	}
-
-	return errors.Wrapf(w.dirFile.Close(), "closing WAL dir %s", w.dirFile.Name())
+	if err := w.dirFile.Close(); err != nil {
+		return fmt.Errorf("closing WAL dir %s: %w", w.dirFile.Name(), err)
+	}
+	return nil
 }
 
 func (w *SegmentWAL) write(t WALEntryType, flag uint8, buf []byte) error {
@@ -921,7 +926,7 @@ func (r *walReader) Read(
 
 			err = r.decodeSeries(flag, b, &series)
 			if err != nil {
-				err = errors.Wrap(err, "decode series entry")
+				err = fmt.Errorf("decode series entry: %w", err)
 				break
 			}
 			datac <- series
@@ -940,7 +945,7 @@ func (r *walReader) Read(
 
 			err = r.decodeSamples(flag, b, &samples)
 			if err != nil {
-				err = errors.Wrap(err, "decode samples entry")
+				err = fmt.Errorf("decode samples entry: %w", err)
 				break
 			}
 			datac <- samples
@@ -960,7 +965,7 @@ func (r *walReader) Read(
 
 			err = r.decodeDeletes(flag, b, &deletes)
 			if err != nil {
-				err = errors.Wrap(err, "decode delete entry")
+				err = fmt.Errorf("decode delete entry: %w", err)
 				break
 			}
 			datac <- deletes
@@ -982,8 +987,8 @@ func (r *walReader) Read(
 	if err != nil {
 		return err
 	}
-	if r.Err() != nil {
-		return errors.Wrap(r.Err(), "read entry")
+	if err := r.Err(); err != nil {
+		return fmt.Errorf("read entry: %w", err)
 	}
 	return nil
 }
@@ -1046,12 +1051,16 @@ type walCorruptionErr struct {
 	lastOffset int64
 }
 
-func (e walCorruptionErr) Error() string {
+func (e *walCorruptionErr) Error() string {
 	return fmt.Sprintf("%s <file: %d, lastOffset: %d>", e.err, e.file, e.lastOffset)
 }
 
+func (e *walCorruptionErr) Unwrap() error {
+	return e.err
+}
+
 func (r *walReader) corruptionErr(s string, args ...interface{}) error {
-	return walCorruptionErr{
+	return &walCorruptionErr{
 		err:        fmt.Errorf(s, args...),
 		file:       r.cur,
 		lastOffset: r.lastOffset,
@@ -1152,8 +1161,8 @@ func (r *walReader) decodeSamples(flag byte, b []byte, res *[]record.RefSample)
 		})
 	}
 
-	if dec.Err() != nil {
-		return errors.Wrapf(dec.Err(), "decode error after %d samples", len(*res))
+	if err := dec.Err(); err != nil {
+		return fmt.Errorf("decode error after %d samples: %w", len(*res), err)
 	}
 	if len(dec.B) > 0 {
 		return fmt.Errorf("unexpected %d bytes left in entry", len(dec.B))
@@ -1185,7 +1194,7 @@ func deprecatedWALExists(logger log.Logger, dir string) (bool, error) {
 	// Detect whether we still have the old WAL.
 	fns, err := sequenceFiles(dir)
 	if err != nil && !os.IsNotExist(err) {
-		return false, errors.Wrap(err, "list sequence files")
+		return false, fmt.Errorf("list sequence files: %w", err)
 	}
 	if len(fns) == 0 {
 		return false, nil // No WAL at all yet.
@@ -1194,13 +1203,13 @@ func deprecatedWALExists(logger log.Logger, dir string) (bool, error) {
 	// old WAL.
 	f, err := os.Open(fns[0])
 	if err != nil {
-		return false, errors.Wrap(err, "check first existing segment")
+		return false, fmt.Errorf("check first existing segment: %w", err)
 	}
 	defer f.Close()
 
 	var hdr [4]byte
-	if _, err := f.Read(hdr[:]); err != nil && err != io.EOF {
-		return false, errors.Wrap(err, "read header from first segment")
+	if _, err := f.Read(hdr[:]); err != nil && !errors.Is(err, io.EOF) {
+		return false, fmt.Errorf("read header from first segment: %w", err)
 	}
 	// If we cannot read the magic header for segments of the old WAL, abort.
 	// Either it's migrated already or there's a corruption issue with which
@@ -1223,11 +1232,11 @@ func MigrateWAL(logger log.Logger, dir string) (err error) {
 
 	tmpdir := dir + ".tmp"
 	if err := os.RemoveAll(tmpdir); err != nil {
-		return errors.Wrap(err, "cleanup replacement dir")
+		return fmt.Errorf("cleanup replacement dir: %w", err)
 	}
 	repl, err := wlog.New(logger, nil, tmpdir, wlog.CompressionNone)
 	if err != nil {
-		return errors.Wrap(err, "open new WAL")
+		return fmt.Errorf("open new WAL: %w", err)
 	}
 
 	// It should've already been closed as part of the previous finalization.
@@ -1240,7 +1249,7 @@ func MigrateWAL(logger log.Logger, dir string) (err error) {
 
 	w, err := OpenSegmentWAL(dir, logger, time.Minute, nil)
 	if err != nil {
-		return errors.Wrap(err, "open old WAL")
+		return fmt.Errorf("open old WAL: %w", err)
 	}
 	defer w.Close()
 
@@ -1271,22 +1280,22 @@ func MigrateWAL(logger log.Logger, dir string) (err error) {
 		},
 	)
 	if decErr != nil {
-		return errors.Wrap(err, "decode old entries")
+		return fmt.Errorf("decode old entries: %w", err)
 	}
 	if err != nil {
-		return errors.Wrap(err, "write new entries")
+		return fmt.Errorf("write new entries: %w", err)
 	}
 	// We explicitly close even when there is a defer for Windows to be
 	// able to delete it. The defer is in place to close it in-case there
 	// are errors above.
 	if err := w.Close(); err != nil {
-		return errors.Wrap(err, "close old WAL")
+		return fmt.Errorf("close old WAL: %w", err)
 	}
 	if err := repl.Close(); err != nil {
-		return errors.Wrap(err, "close new WAL")
+		return fmt.Errorf("close new WAL: %w", err)
 	}
 	if err := fileutil.Replace(tmpdir, dir); err != nil {
-		return errors.Wrap(err, "replace old WAL")
+		return fmt.Errorf("replace old WAL: %w", err)
 	}
 	return nil
 }
diff --git a/tsdb/wlog/wlog.go b/tsdb/wlog/wlog.go
index c3ae001d9..fdea75694 100644
--- a/tsdb/wlog/wlog.go
+++ b/tsdb/wlog/wlog.go
@@ -116,6 +116,10 @@ func (e *CorruptionErr) Error() string {
 	return fmt.Sprintf("corruption in segment %s at %d: %s", SegmentName(e.Dir, e.Segment), e.Offset, e.Err)
 }
 
+func (e *CorruptionErr) Unwrap() error {
+	return e.Err
+}
+
 // OpenWriteSegment opens segment k in dir. The returned segment is ready for new appends.
 func OpenWriteSegment(logger log.Logger, dir string, k int) (*Segment, error) {
 	segName := SegmentName(dir, k)

From 108a749a459e346de0d76c109f9cded21d4b0932 Mon Sep 17 00:00:00 2001
From: Paulin Todev <paulin.todev@gmail.com>
Date: Mon, 11 Dec 2023 13:12:43 +0000
Subject: [PATCH 022/115] Set up labels for counters in advance

Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
---
 discovery/kubernetes/endpoints.go     | 21 +++++++++++++++------
 discovery/kubernetes/endpointslice.go | 21 +++++++++++++++------
 discovery/kubernetes/ingress.go       | 19 +++++++++++++++----
 discovery/kubernetes/node.go          | 19 +++++++++++++++----
 discovery/kubernetes/pod.go           | 10 +++++++---
 discovery/kubernetes/service.go       | 19 +++++++++++++++----
 6 files changed, 82 insertions(+), 27 deletions(-)

diff --git a/discovery/kubernetes/endpoints.go b/discovery/kubernetes/endpoints.go
index 801a45f7c..f97f1ee56 100644
--- a/discovery/kubernetes/endpoints.go
+++ b/discovery/kubernetes/endpoints.go
@@ -53,6 +53,15 @@ func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node ca
 	if l == nil {
 		l = log.NewNopLogger()
 	}
+
+	epAddCount := eventCount.WithLabelValues("endpoints", "add")
+	epUpdateCount := eventCount.WithLabelValues("endpoints", "update")
+	epDeleteCount := eventCount.WithLabelValues("endpoints", "delete")
+
+	svcAddCount := eventCount.WithLabelValues("service", "add")
+	svcUpdateCount := eventCount.WithLabelValues("service", "update")
+	svcDeleteCount := eventCount.WithLabelValues("service", "delete")
+
 	e := &Endpoints{
 		logger:           l,
 		endpointsInf:     eps,
@@ -68,15 +77,15 @@ func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node ca
 
 	_, err := e.endpointsInf.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(o interface{}) {
-			eventCount.WithLabelValues("endpoints", "add").Inc()
+			epAddCount.Inc()
 			e.enqueue(o)
 		},
 		UpdateFunc: func(_, o interface{}) {
-			eventCount.WithLabelValues("endpoints", "update").Inc()
+			epUpdateCount.Inc()
 			e.enqueue(o)
 		},
 		DeleteFunc: func(o interface{}) {
-			eventCount.WithLabelValues("endpoints", "delete").Inc()
+			epDeleteCount.Inc()
 			e.enqueue(o)
 		},
 	})
@@ -107,15 +116,15 @@ func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node ca
 		// TODO(fabxc): potentially remove add and delete event handlers. Those should
 		// be triggered via the endpoint handlers already.
 		AddFunc: func(o interface{}) {
-			eventCount.WithLabelValues("service", "add").Inc()
+			svcAddCount.Inc()
 			serviceUpdate(o)
 		},
 		UpdateFunc: func(_, o interface{}) {
-			eventCount.WithLabelValues("service", "update").Inc()
+			svcUpdateCount.Inc()
 			serviceUpdate(o)
 		},
 		DeleteFunc: func(o interface{}) {
-			eventCount.WithLabelValues("service", "delete").Inc()
+			svcDeleteCount.Inc()
 			serviceUpdate(o)
 		},
 	})
diff --git a/discovery/kubernetes/endpointslice.go b/discovery/kubernetes/endpointslice.go
index e2ac1de42..a7e0ea98c 100644
--- a/discovery/kubernetes/endpointslice.go
+++ b/discovery/kubernetes/endpointslice.go
@@ -56,6 +56,15 @@ func NewEndpointSlice(l log.Logger, eps cache.SharedIndexInformer, svc, pod, nod
 	if l == nil {
 		l = log.NewNopLogger()
 	}
+
+	epslAddCount := eventCount.WithLabelValues("endpointslice", "add")
+	epslUpdateCount := eventCount.WithLabelValues("endpointslice", "update")
+	epslDeleteCount := eventCount.WithLabelValues("endpointslice", "delete")
+
+	svcAddCount := eventCount.WithLabelValues("service", "add")
+	svcUpdateCount := eventCount.WithLabelValues("service", "update")
+	svcDeleteCount := eventCount.WithLabelValues("service", "delete")
+
 	e := &EndpointSlice{
 		logger:             l,
 		endpointSliceInf:   eps,
@@ -71,15 +80,15 @@ func NewEndpointSlice(l log.Logger, eps cache.SharedIndexInformer, svc, pod, nod
 
 	_, err := e.endpointSliceInf.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(o interface{}) {
-			eventCount.WithLabelValues("endpointslice", "add").Inc()
+			epslAddCount.Inc()
 			e.enqueue(o)
 		},
 		UpdateFunc: func(_, o interface{}) {
-			eventCount.WithLabelValues("endpointslice", "update").Inc()
+			epslUpdateCount.Inc()
 			e.enqueue(o)
 		},
 		DeleteFunc: func(o interface{}) {
-			eventCount.WithLabelValues("endpointslice", "delete").Inc()
+			epslDeleteCount.Inc()
 			e.enqueue(o)
 		},
 	})
@@ -110,15 +119,15 @@ func NewEndpointSlice(l log.Logger, eps cache.SharedIndexInformer, svc, pod, nod
 	}
 	_, err = e.serviceInf.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(o interface{}) {
-			eventCount.WithLabelValues("service", "add").Inc()
+			svcAddCount.Inc()
 			serviceUpdate(o)
 		},
 		UpdateFunc: func(_, o interface{}) {
-			eventCount.WithLabelValues("service", "update").Inc()
+			svcUpdateCount.Inc()
 			serviceUpdate(o)
 		},
 		DeleteFunc: func(o interface{}) {
-			eventCount.WithLabelValues("service", "delete").Inc()
+			svcDeleteCount.Inc()
 			serviceUpdate(o)
 		},
 	})
diff --git a/discovery/kubernetes/ingress.go b/discovery/kubernetes/ingress.go
index 91b1fb0ee..72712245a 100644
--- a/discovery/kubernetes/ingress.go
+++ b/discovery/kubernetes/ingress.go
@@ -41,18 +41,29 @@ type Ingress struct {
 
 // NewIngress returns a new ingress discovery.
 func NewIngress(l log.Logger, inf cache.SharedInformer, eventCount *prometheus.CounterVec) *Ingress {
-	s := &Ingress{logger: l, informer: inf, store: inf.GetStore(), queue: workqueue.NewNamed("ingress")}
+
+	ingressAddCount := eventCount.WithLabelValues("ingress", "add")
+	ingressUpdateCount := eventCount.WithLabelValues("ingress", "update")
+	ingressDeleteCount := eventCount.WithLabelValues("ingress", "delete")
+
+	s := &Ingress{
+		logger:   l,
+		informer: inf,
+		store:    inf.GetStore(),
+		queue:    workqueue.NewNamed("ingress"),
+	}
+
 	_, err := s.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(o interface{}) {
-			eventCount.WithLabelValues("ingress", "add").Inc()
+			ingressAddCount.Inc()
 			s.enqueue(o)
 		},
 		DeleteFunc: func(o interface{}) {
-			eventCount.WithLabelValues("ingress", "delete").Inc()
+			ingressDeleteCount.Inc()
 			s.enqueue(o)
 		},
 		UpdateFunc: func(_, o interface{}) {
-			eventCount.WithLabelValues("ingress", "update").Inc()
+			ingressUpdateCount.Inc()
 			s.enqueue(o)
 		},
 	})
diff --git a/discovery/kubernetes/node.go b/discovery/kubernetes/node.go
index f1e37e6fa..f9a318336 100644
--- a/discovery/kubernetes/node.go
+++ b/discovery/kubernetes/node.go
@@ -49,18 +49,29 @@ func NewNode(l log.Logger, inf cache.SharedInformer, eventCount *prometheus.Coun
 	if l == nil {
 		l = log.NewNopLogger()
 	}
-	n := &Node{logger: l, informer: inf, store: inf.GetStore(), queue: workqueue.NewNamed("node")}
+
+	nodeAddCount := eventCount.WithLabelValues("node", "add")
+	nodeUpdateCount := eventCount.WithLabelValues("node", "update")
+	nodeDeleteCount := eventCount.WithLabelValues("node", "delete")
+
+	n := &Node{
+		logger:   l,
+		informer: inf,
+		store:    inf.GetStore(),
+		queue:    workqueue.NewNamed("node"),
+	}
+
 	_, err := n.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(o interface{}) {
-			eventCount.WithLabelValues("node", "add").Inc()
+			nodeAddCount.Inc()
 			n.enqueue(o)
 		},
 		DeleteFunc: func(o interface{}) {
-			eventCount.WithLabelValues("node", "delete").Inc()
+			nodeDeleteCount.Inc()
 			n.enqueue(o)
 		},
 		UpdateFunc: func(_, o interface{}) {
-			eventCount.WithLabelValues("node", "update").Inc()
+			nodeUpdateCount.Inc()
 			n.enqueue(o)
 		},
 	})
diff --git a/discovery/kubernetes/pod.go b/discovery/kubernetes/pod.go
index cc809b29c..31936f9c0 100644
--- a/discovery/kubernetes/pod.go
+++ b/discovery/kubernetes/pod.go
@@ -51,6 +51,10 @@ func NewPod(l log.Logger, pods cache.SharedIndexInformer, nodes cache.SharedInfo
 		l = log.NewNopLogger()
 	}
 
+	podAddCount := eventCount.WithLabelValues("pod", "add")
+	podDeleteCount := eventCount.WithLabelValues("pod", "delete")
+	podUpdateCount := eventCount.WithLabelValues("pod", "update")
+
 	p := &Pod{
 		podInf:           pods,
 		nodeInf:          nodes,
@@ -61,15 +65,15 @@ func NewPod(l log.Logger, pods cache.SharedIndexInformer, nodes cache.SharedInfo
 	}
 	_, err := p.podInf.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(o interface{}) {
-			eventCount.WithLabelValues("pod", "add").Inc()
+			podAddCount.Inc()
 			p.enqueue(o)
 		},
 		DeleteFunc: func(o interface{}) {
-			eventCount.WithLabelValues("pod", "delete").Inc()
+			podDeleteCount.Inc()
 			p.enqueue(o)
 		},
 		UpdateFunc: func(_, o interface{}) {
-			eventCount.WithLabelValues("pod", "update").Inc()
+			podUpdateCount.Inc()
 			p.enqueue(o)
 		},
 	})
diff --git a/discovery/kubernetes/service.go b/discovery/kubernetes/service.go
index a680ebee8..02158b124 100644
--- a/discovery/kubernetes/service.go
+++ b/discovery/kubernetes/service.go
@@ -44,18 +44,29 @@ func NewService(l log.Logger, inf cache.SharedInformer, eventCount *prometheus.C
 	if l == nil {
 		l = log.NewNopLogger()
 	}
-	s := &Service{logger: l, informer: inf, store: inf.GetStore(), queue: workqueue.NewNamed("service")}
+
+	svcAddCount := eventCount.WithLabelValues("service", "add")
+	svcUpdateCount := eventCount.WithLabelValues("service", "update")
+	svcDeleteCount := eventCount.WithLabelValues("service", "delete")
+
+	s := &Service{
+		logger:   l,
+		informer: inf,
+		store:    inf.GetStore(),
+		queue:    workqueue.NewNamed("service"),
+	}
+
 	_, err := s.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(o interface{}) {
-			eventCount.WithLabelValues("service", "add").Inc()
+			svcAddCount.Inc()
 			s.enqueue(o)
 		},
 		DeleteFunc: func(o interface{}) {
-			eventCount.WithLabelValues("service", "delete").Inc()
+			svcDeleteCount.Inc()
 			s.enqueue(o)
 		},
 		UpdateFunc: func(_, o interface{}) {
-			eventCount.WithLabelValues("service", "update").Inc()
+			svcUpdateCount.Inc()
 			s.enqueue(o)
 		},
 	})

From 27bb57a37bd92523f932f17b202c98985888e21f Mon Sep 17 00:00:00 2001
From: Paulin Todev <paulin.todev@gmail.com>
Date: Mon, 11 Dec 2023 13:17:47 +0000
Subject: [PATCH 023/115] Define metric label values in one place

Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
---
 discovery/kubernetes/endpoints.go     | 14 +++++++-------
 discovery/kubernetes/endpointslice.go | 14 +++++++-------
 discovery/kubernetes/ingress.go       |  8 ++++----
 discovery/kubernetes/kubernetes.go    | 24 ++++++++++++++++++++++--
 discovery/kubernetes/node.go          |  8 ++++----
 discovery/kubernetes/pod.go           |  8 ++++----
 discovery/kubernetes/service.go       |  8 ++++----
 7 files changed, 52 insertions(+), 32 deletions(-)

diff --git a/discovery/kubernetes/endpoints.go b/discovery/kubernetes/endpoints.go
index f97f1ee56..512d77552 100644
--- a/discovery/kubernetes/endpoints.go
+++ b/discovery/kubernetes/endpoints.go
@@ -54,13 +54,13 @@ func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node ca
 		l = log.NewNopLogger()
 	}
 
-	epAddCount := eventCount.WithLabelValues("endpoints", "add")
-	epUpdateCount := eventCount.WithLabelValues("endpoints", "update")
-	epDeleteCount := eventCount.WithLabelValues("endpoints", "delete")
+	epAddCount := eventCount.WithLabelValues(RoleEndpoint.String(), MetricLabelRoleAdd)
+	epUpdateCount := eventCount.WithLabelValues(RoleEndpoint.String(), MetricLabelRoleUpdate)
+	epDeleteCount := eventCount.WithLabelValues(RoleEndpoint.String(), MetricLabelRoleDelete)
 
-	svcAddCount := eventCount.WithLabelValues("service", "add")
-	svcUpdateCount := eventCount.WithLabelValues("service", "update")
-	svcDeleteCount := eventCount.WithLabelValues("service", "delete")
+	svcAddCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleAdd)
+	svcUpdateCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleUpdate)
+	svcDeleteCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleDelete)
 
 	e := &Endpoints{
 		logger:           l,
@@ -72,7 +72,7 @@ func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node ca
 		podStore:         pod.GetStore(),
 		nodeInf:          node,
 		withNodeMetadata: node != nil,
-		queue:            workqueue.NewNamed("endpoints"),
+		queue:            workqueue.NewNamed(RoleEndpoint.String()),
 	}
 
 	_, err := e.endpointsInf.AddEventHandler(cache.ResourceEventHandlerFuncs{
diff --git a/discovery/kubernetes/endpointslice.go b/discovery/kubernetes/endpointslice.go
index a7e0ea98c..21095df4a 100644
--- a/discovery/kubernetes/endpointslice.go
+++ b/discovery/kubernetes/endpointslice.go
@@ -57,13 +57,13 @@ func NewEndpointSlice(l log.Logger, eps cache.SharedIndexInformer, svc, pod, nod
 		l = log.NewNopLogger()
 	}
 
-	epslAddCount := eventCount.WithLabelValues("endpointslice", "add")
-	epslUpdateCount := eventCount.WithLabelValues("endpointslice", "update")
-	epslDeleteCount := eventCount.WithLabelValues("endpointslice", "delete")
+	epslAddCount := eventCount.WithLabelValues(RoleEndpointSlice.String(), MetricLabelRoleAdd)
+	epslUpdateCount := eventCount.WithLabelValues(RoleEndpointSlice.String(), MetricLabelRoleUpdate)
+	epslDeleteCount := eventCount.WithLabelValues(RoleEndpointSlice.String(), MetricLabelRoleDelete)
 
-	svcAddCount := eventCount.WithLabelValues("service", "add")
-	svcUpdateCount := eventCount.WithLabelValues("service", "update")
-	svcDeleteCount := eventCount.WithLabelValues("service", "delete")
+	svcAddCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleAdd)
+	svcUpdateCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleUpdate)
+	svcDeleteCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleDelete)
 
 	e := &EndpointSlice{
 		logger:             l,
@@ -75,7 +75,7 @@ func NewEndpointSlice(l log.Logger, eps cache.SharedIndexInformer, svc, pod, nod
 		podStore:           pod.GetStore(),
 		nodeInf:            node,
 		withNodeMetadata:   node != nil,
-		queue:              workqueue.NewNamed("endpointSlice"),
+		queue:              workqueue.NewNamed(RoleEndpointSlice.String()),
 	}
 
 	_, err := e.endpointSliceInf.AddEventHandler(cache.ResourceEventHandlerFuncs{
diff --git a/discovery/kubernetes/ingress.go b/discovery/kubernetes/ingress.go
index 72712245a..80f508a64 100644
--- a/discovery/kubernetes/ingress.go
+++ b/discovery/kubernetes/ingress.go
@@ -42,15 +42,15 @@ type Ingress struct {
 // NewIngress returns a new ingress discovery.
 func NewIngress(l log.Logger, inf cache.SharedInformer, eventCount *prometheus.CounterVec) *Ingress {
 
-	ingressAddCount := eventCount.WithLabelValues("ingress", "add")
-	ingressUpdateCount := eventCount.WithLabelValues("ingress", "update")
-	ingressDeleteCount := eventCount.WithLabelValues("ingress", "delete")
+	ingressAddCount := eventCount.WithLabelValues(RoleIngress.String(), MetricLabelRoleAdd)
+	ingressUpdateCount := eventCount.WithLabelValues(RoleIngress.String(), MetricLabelRoleUpdate)
+	ingressDeleteCount := eventCount.WithLabelValues(RoleIngress.String(), MetricLabelRoleDelete)
 
 	s := &Ingress{
 		logger:   l,
 		informer: inf,
 		store:    inf.GetStore(),
-		queue:    workqueue.NewNamed("ingress"),
+		queue:    workqueue.NewNamed(RoleIngress.String()),
 	}
 
 	_, err := s.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
diff --git a/discovery/kubernetes/kubernetes.go b/discovery/kubernetes/kubernetes.go
index 4deaf3f68..1e59c0db6 100644
--- a/discovery/kubernetes/kubernetes.go
+++ b/discovery/kubernetes/kubernetes.go
@@ -102,6 +102,16 @@ func (c *Role) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	}
 }
 
+func (c Role) String() string {
+	return string(c)
+}
+
+const (
+	MetricLabelRoleAdd    = "add"
+	MetricLabelRoleDelete = "delete"
+	MetricLabelRoleUpdate = "update"
+)
+
 // SDConfig is the configuration for Kubernetes service discovery.
 type SDConfig struct {
 	APIServer          config.URL              `yaml:"api_server,omitempty"`
@@ -351,8 +361,18 @@ func New(l log.Logger, reg prometheus.Registerer, conf *SDConfig) (*Discovery, e
 	d.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{d.eventCount})
 
 	// Initialize metric vectors.
-	for _, role := range []string{"endpointslice", "endpoints", "node", "pod", "service", "ingress"} {
-		for _, evt := range []string{"add", "delete", "update"} {
+	for _, role := range []string{
+		RoleEndpointSlice.String(),
+		RoleEndpoint.String(),
+		RoleNode.String(),
+		RolePod.String(),
+		RoleService.String(),
+		RoleIngress.String()} {
+		for _, evt := range []string{
+			MetricLabelRoleAdd,
+			MetricLabelRoleDelete,
+			MetricLabelRoleUpdate,
+		} {
 			d.eventCount.WithLabelValues(role, evt)
 		}
 	}
diff --git a/discovery/kubernetes/node.go b/discovery/kubernetes/node.go
index f9a318336..74d87e22c 100644
--- a/discovery/kubernetes/node.go
+++ b/discovery/kubernetes/node.go
@@ -50,15 +50,15 @@ func NewNode(l log.Logger, inf cache.SharedInformer, eventCount *prometheus.Coun
 		l = log.NewNopLogger()
 	}
 
-	nodeAddCount := eventCount.WithLabelValues("node", "add")
-	nodeUpdateCount := eventCount.WithLabelValues("node", "update")
-	nodeDeleteCount := eventCount.WithLabelValues("node", "delete")
+	nodeAddCount := eventCount.WithLabelValues(RoleNode.String(), MetricLabelRoleAdd)
+	nodeUpdateCount := eventCount.WithLabelValues(RoleNode.String(), MetricLabelRoleUpdate)
+	nodeDeleteCount := eventCount.WithLabelValues(RoleNode.String(), MetricLabelRoleDelete)
 
 	n := &Node{
 		logger:   l,
 		informer: inf,
 		store:    inf.GetStore(),
-		queue:    workqueue.NewNamed("node"),
+		queue:    workqueue.NewNamed(RoleNode.String()),
 	}
 
 	_, err := n.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
diff --git a/discovery/kubernetes/pod.go b/discovery/kubernetes/pod.go
index 31936f9c0..615717c13 100644
--- a/discovery/kubernetes/pod.go
+++ b/discovery/kubernetes/pod.go
@@ -51,9 +51,9 @@ func NewPod(l log.Logger, pods cache.SharedIndexInformer, nodes cache.SharedInfo
 		l = log.NewNopLogger()
 	}
 
-	podAddCount := eventCount.WithLabelValues("pod", "add")
-	podDeleteCount := eventCount.WithLabelValues("pod", "delete")
-	podUpdateCount := eventCount.WithLabelValues("pod", "update")
+	podAddCount := eventCount.WithLabelValues(RolePod.String(), MetricLabelRoleAdd)
+	podDeleteCount := eventCount.WithLabelValues(RolePod.String(), MetricLabelRoleDelete)
+	podUpdateCount := eventCount.WithLabelValues(RolePod.String(), MetricLabelRoleUpdate)
 
 	p := &Pod{
 		podInf:           pods,
@@ -61,7 +61,7 @@ func NewPod(l log.Logger, pods cache.SharedIndexInformer, nodes cache.SharedInfo
 		withNodeMetadata: nodes != nil,
 		store:            pods.GetStore(),
 		logger:           l,
-		queue:            workqueue.NewNamed("pod"),
+		queue:            workqueue.NewNamed(RolePod.String()),
 	}
 	_, err := p.podInf.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(o interface{}) {
diff --git a/discovery/kubernetes/service.go b/discovery/kubernetes/service.go
index 02158b124..51204a5a1 100644
--- a/discovery/kubernetes/service.go
+++ b/discovery/kubernetes/service.go
@@ -45,15 +45,15 @@ func NewService(l log.Logger, inf cache.SharedInformer, eventCount *prometheus.C
 		l = log.NewNopLogger()
 	}
 
-	svcAddCount := eventCount.WithLabelValues("service", "add")
-	svcUpdateCount := eventCount.WithLabelValues("service", "update")
-	svcDeleteCount := eventCount.WithLabelValues("service", "delete")
+	svcAddCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleAdd)
+	svcUpdateCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleUpdate)
+	svcDeleteCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleDelete)
 
 	s := &Service{
 		logger:   l,
 		informer: inf,
 		store:    inf.GetStore(),
-		queue:    workqueue.NewNamed("service"),
+		queue:    workqueue.NewNamed(RoleService.String()),
 	}
 
 	_, err := s.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{

From d2e997030ee2cbb4ee9a5c32d77ae2bdf644ec77 Mon Sep 17 00:00:00 2001
From: Paulin Todev <paulin.todev@gmail.com>
Date: Mon, 11 Dec 2023 14:05:15 +0000
Subject: [PATCH 024/115] Fix linter issues

Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
---
 discovery/kubernetes/ingress.go    | 1 -
 discovery/kubernetes/kubernetes.go | 3 ++-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/discovery/kubernetes/ingress.go b/discovery/kubernetes/ingress.go
index 80f508a64..7b6366b25 100644
--- a/discovery/kubernetes/ingress.go
+++ b/discovery/kubernetes/ingress.go
@@ -41,7 +41,6 @@ type Ingress struct {
 
 // NewIngress returns a new ingress discovery.
 func NewIngress(l log.Logger, inf cache.SharedInformer, eventCount *prometheus.CounterVec) *Ingress {
-
 	ingressAddCount := eventCount.WithLabelValues(RoleIngress.String(), MetricLabelRoleAdd)
 	ingressUpdateCount := eventCount.WithLabelValues(RoleIngress.String(), MetricLabelRoleUpdate)
 	ingressDeleteCount := eventCount.WithLabelValues(RoleIngress.String(), MetricLabelRoleDelete)
diff --git a/discovery/kubernetes/kubernetes.go b/discovery/kubernetes/kubernetes.go
index 1e59c0db6..5c5f3dfb6 100644
--- a/discovery/kubernetes/kubernetes.go
+++ b/discovery/kubernetes/kubernetes.go
@@ -367,7 +367,8 @@ func New(l log.Logger, reg prometheus.Registerer, conf *SDConfig) (*Discovery, e
 		RoleNode.String(),
 		RolePod.String(),
 		RoleService.String(),
-		RoleIngress.String()} {
+		RoleIngress.String(),
+	} {
 		for _, evt := range []string{
 			MetricLabelRoleAdd,
 			MetricLabelRoleDelete,

From d12ccf9fa29dbc2d0f40013f04dbe7adb8a1113e Mon Sep 17 00:00:00 2001
From: Marcin Skalski <skalskimarcin33@gmail.com>
Date: Mon, 11 Dec 2023 16:33:42 +0100
Subject: [PATCH 025/115] kuma_sd: Extend Kuma SD configuration to allow users
 to specify ClientId

Signed-off-by: Marcin Skalski <skalskimarcin33@gmail.com>
---
 config/config_test.go               |  1 +
 config/testdata/conf.good.yml       |  1 +
 config/testdata/roundtrip.good.yml  |  1 +
 discovery/xds/kuma.go               | 18 ++++++++++++++----
 discovery/xds/kuma_test.go          |  2 +-
 discovery/xds/xds.go                |  1 +
 discovery/xds/xds_test.go           |  1 +
 docs/configuration/configuration.md |  5 +++++
 8 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/config/config_test.go b/config/config_test.go
index 5d753a0f7..7c061fd54 100644
--- a/config/config_test.go
+++ b/config/config_test.go
@@ -568,6 +568,7 @@ var expectedConf = &Config{
 			ServiceDiscoveryConfigs: discovery.Configs{
 				&xds.KumaSDConfig{
 					Server:           "http://kuma-control-plane.kuma-system.svc:5676",
+					ClientId:         "main-prometheus",
 					HTTPClientConfig: config.DefaultHTTPClientConfig,
 					RefreshInterval:  model.Duration(15 * time.Second),
 					FetchTimeout:     model.Duration(2 * time.Minute),
diff --git a/config/testdata/conf.good.yml b/config/testdata/conf.good.yml
index e034eff43..972099800 100644
--- a/config/testdata/conf.good.yml
+++ b/config/testdata/conf.good.yml
@@ -221,6 +221,7 @@ scrape_configs:
 
     kuma_sd_configs:
       - server: http://kuma-control-plane.kuma-system.svc:5676
+        clientId: main-prometheus
 
   - job_name: service-marathon
     marathon_sd_configs:
diff --git a/config/testdata/roundtrip.good.yml b/config/testdata/roundtrip.good.yml
index f2634d257..26589ad1b 100644
--- a/config/testdata/roundtrip.good.yml
+++ b/config/testdata/roundtrip.good.yml
@@ -108,6 +108,7 @@ scrape_configs:
 
     kuma_sd_configs:
       - server: http://kuma-control-plane.kuma-system.svc:5676
+        clientId: main-prometheus
 
     marathon_sd_configs:
       - servers:
diff --git a/discovery/xds/kuma.go b/discovery/xds/kuma.go
index bc88ba554..1cec053e4 100644
--- a/discovery/xds/kuma.go
+++ b/discovery/xds/kuma.go
@@ -178,10 +178,11 @@ func kumaMadsV1ResourceParser(resources []*anypb.Any, typeURL string) ([]model.L
 
 func NewKumaHTTPDiscovery(conf *KumaSDConfig, logger log.Logger) (discovery.Discoverer, error) {
 	// Default to "prometheus" if hostname is unavailable.
-	clientID, err := osutil.GetFQDN()
-	if err != nil {
-		level.Debug(logger).Log("msg", "error getting FQDN", "err", err)
-		clientID = "prometheus"
+	var clientID string
+	if conf.ClientId == "" {
+		clientID = defaultClientId(logger)
+	} else {
+		clientID = conf.ClientId
 	}
 
 	clientConfig := &HTTPResourceClientConfig{
@@ -215,3 +216,12 @@ func NewKumaHTTPDiscovery(conf *KumaSDConfig, logger log.Logger) (discovery.Disc
 
 	return d, nil
 }
+
+func defaultClientId(logger log.Logger) string {
+	clientID, err := osutil.GetFQDN()
+	if err != nil {
+		level.Debug(logger).Log("msg", "error getting FQDN", "err", err)
+		clientID = "prometheus"
+	}
+	return clientID
+}
diff --git a/discovery/xds/kuma_test.go b/discovery/xds/kuma_test.go
index 581be9fb1..7f2b0ce3b 100644
--- a/discovery/xds/kuma_test.go
+++ b/discovery/xds/kuma_test.go
@@ -204,7 +204,7 @@ func TestNewKumaHTTPDiscovery(t *testing.T) {
 	require.True(t, ok)
 	require.Equal(t, kumaConf.Server, resClient.Server())
 	require.Equal(t, KumaMadsV1ResourceTypeURL, resClient.ResourceTypeURL())
-	require.NotEmpty(t, resClient.ID())
+	require.Equal(t, kumaConf.ClientId, resClient.ID())
 	require.Equal(t, KumaMadsV1ResourceType, resClient.config.ResourceType)
 }
 
diff --git a/discovery/xds/xds.go b/discovery/xds/xds.go
index 48bdbab02..47baece78 100644
--- a/discovery/xds/xds.go
+++ b/discovery/xds/xds.go
@@ -55,6 +55,7 @@ type SDConfig struct {
 	RefreshInterval  model.Duration          `yaml:"refresh_interval,omitempty"`
 	FetchTimeout     model.Duration          `yaml:"fetch_timeout,omitempty"`
 	Server           string                  `yaml:"server,omitempty"`
+	ClientId         string                  `yaml:"clientId,omitempty"`
 }
 
 // mustRegisterMessage registers the provided message type in the typeRegistry, and panics
diff --git a/discovery/xds/xds_test.go b/discovery/xds/xds_test.go
index 974a47342..2e0f24e19 100644
--- a/discovery/xds/xds_test.go
+++ b/discovery/xds/xds_test.go
@@ -36,6 +36,7 @@ var (
 	sdConf = SDConfig{
 		Server:          "http://127.0.0.1",
 		RefreshInterval: model.Duration(10 * time.Second),
+		ClientId:        "test-id",
 	}
 
 	testFetchFailuresCount = prometheus.NewCounter(
diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md
index f05925d2b..d3ee459be 100644
--- a/docs/configuration/configuration.md
+++ b/docs/configuration/configuration.md
@@ -2230,6 +2230,11 @@ See below for the configuration options for Kuma MonitoringAssignment discovery:
 # Address of the Kuma Control Plane's MADS xDS server.
 server: <string>
 
+# Client id is used by Kuma Control Plane to compute Monitoring Assignment for specific Prometheus backend. 
+# This is useful when migrating between multiple Prometheus backends, or having separate backend for each Mesh
+# When not specified, system hostname/fqdn will be used if available, if not `prometheus` will be used.
+clientId: <string>
+
 # The time to wait between polling update requests.
 [ refresh_interval: <duration> | default = 30s ]
 

From 0af810aa718a9cb9a9466785715103ad6117e959 Mon Sep 17 00:00:00 2001
From: Marcin Skalski <skalskimarcin33@gmail.com>
Date: Mon, 11 Dec 2023 17:01:52 +0100
Subject: [PATCH 026/115] fix go lint

Signed-off-by: Marcin Skalski <skalskimarcin33@gmail.com>
---
 config/testdata/conf.good.yml       | 2 +-
 config/testdata/roundtrip.good.yml  | 2 +-
 discovery/xds/kuma.go               | 8 ++++----
 discovery/xds/kuma_test.go          | 2 +-
 discovery/xds/xds.go                | 2 +-
 discovery/xds/xds_test.go           | 2 +-
 docs/configuration/configuration.md | 2 +-
 7 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/config/testdata/conf.good.yml b/config/testdata/conf.good.yml
index 972099800..b58430164 100644
--- a/config/testdata/conf.good.yml
+++ b/config/testdata/conf.good.yml
@@ -221,7 +221,7 @@ scrape_configs:
 
     kuma_sd_configs:
       - server: http://kuma-control-plane.kuma-system.svc:5676
-        clientId: main-prometheus
+        client_id: main-prometheus
 
   - job_name: service-marathon
     marathon_sd_configs:
diff --git a/config/testdata/roundtrip.good.yml b/config/testdata/roundtrip.good.yml
index 26589ad1b..24ab7d259 100644
--- a/config/testdata/roundtrip.good.yml
+++ b/config/testdata/roundtrip.good.yml
@@ -108,7 +108,7 @@ scrape_configs:
 
     kuma_sd_configs:
       - server: http://kuma-control-plane.kuma-system.svc:5676
-        clientId: main-prometheus
+        client_id: main-prometheus
 
     marathon_sd_configs:
       - servers:
diff --git a/discovery/xds/kuma.go b/discovery/xds/kuma.go
index 1cec053e4..e79195bb2 100644
--- a/discovery/xds/kuma.go
+++ b/discovery/xds/kuma.go
@@ -179,10 +179,10 @@ func kumaMadsV1ResourceParser(resources []*anypb.Any, typeURL string) ([]model.L
 func NewKumaHTTPDiscovery(conf *KumaSDConfig, logger log.Logger) (discovery.Discoverer, error) {
 	// Default to "prometheus" if hostname is unavailable.
 	var clientID string
-	if conf.ClientId == "" {
-		clientID = defaultClientId(logger)
+	if conf.ClientID == "" {
+		clientID = defaultClientID(logger)
 	} else {
-		clientID = conf.ClientId
+		clientID = conf.ClientID
 	}
 
 	clientConfig := &HTTPResourceClientConfig{
@@ -217,7 +217,7 @@ func NewKumaHTTPDiscovery(conf *KumaSDConfig, logger log.Logger) (discovery.Disc
 	return d, nil
 }
 
-func defaultClientId(logger log.Logger) string {
+func defaultClientID(logger log.Logger) string {
 	clientID, err := osutil.GetFQDN()
 	if err != nil {
 		level.Debug(logger).Log("msg", "error getting FQDN", "err", err)
diff --git a/discovery/xds/kuma_test.go b/discovery/xds/kuma_test.go
index 7f2b0ce3b..6b4bb8784 100644
--- a/discovery/xds/kuma_test.go
+++ b/discovery/xds/kuma_test.go
@@ -204,7 +204,7 @@ func TestNewKumaHTTPDiscovery(t *testing.T) {
 	require.True(t, ok)
 	require.Equal(t, kumaConf.Server, resClient.Server())
 	require.Equal(t, KumaMadsV1ResourceTypeURL, resClient.ResourceTypeURL())
-	require.Equal(t, kumaConf.ClientId, resClient.ID())
+	require.Equal(t, kumaConf.ClientID, resClient.ID())
 	require.Equal(t, KumaMadsV1ResourceType, resClient.config.ResourceType)
 }
 
diff --git a/discovery/xds/xds.go b/discovery/xds/xds.go
index 47baece78..16aa3f148 100644
--- a/discovery/xds/xds.go
+++ b/discovery/xds/xds.go
@@ -55,7 +55,7 @@ type SDConfig struct {
 	RefreshInterval  model.Duration          `yaml:"refresh_interval,omitempty"`
 	FetchTimeout     model.Duration          `yaml:"fetch_timeout,omitempty"`
 	Server           string                  `yaml:"server,omitempty"`
-	ClientId         string                  `yaml:"clientId,omitempty"`
+	ClientID         string                  `yaml:"client_id,omitempty"`
 }
 
 // mustRegisterMessage registers the provided message type in the typeRegistry, and panics
diff --git a/discovery/xds/xds_test.go b/discovery/xds/xds_test.go
index 2e0f24e19..f57fff996 100644
--- a/discovery/xds/xds_test.go
+++ b/discovery/xds/xds_test.go
@@ -36,7 +36,7 @@ var (
 	sdConf = SDConfig{
 		Server:          "http://127.0.0.1",
 		RefreshInterval: model.Duration(10 * time.Second),
-		ClientId:        "test-id",
+		ClientID:        "test-id",
 	}
 
 	testFetchFailuresCount = prometheus.NewCounter(
diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md
index d3ee459be..41f54dbd2 100644
--- a/docs/configuration/configuration.md
+++ b/docs/configuration/configuration.md
@@ -2233,7 +2233,7 @@ server: <string>
 # Client id is used by Kuma Control Plane to compute Monitoring Assignment for specific Prometheus backend. 
 # This is useful when migrating between multiple Prometheus backends, or having separate backend for each Mesh
 # When not specified, system hostname/fqdn will be used if available, if not `prometheus` will be used.
-clientId: <string>
+client_id: <string>
 
 # The time to wait between polling update requests.
 [ refresh_interval: <duration> | default = 30s ]

From 48934aaef3c8e18b91d232a436c00c802c99a77a Mon Sep 17 00:00:00 2001
From: Marcin Skalski <skalskimarcin33@gmail.com>
Date: Mon, 11 Dec 2023 17:04:56 +0100
Subject: [PATCH 027/115] fix go lint

Signed-off-by: Marcin Skalski <skalskimarcin33@gmail.com>
---
 config/config_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/config_test.go b/config/config_test.go
index 7c061fd54..e614a4463 100644
--- a/config/config_test.go
+++ b/config/config_test.go
@@ -568,7 +568,7 @@ var expectedConf = &Config{
 			ServiceDiscoveryConfigs: discovery.Configs{
 				&xds.KumaSDConfig{
 					Server:           "http://kuma-control-plane.kuma-system.svc:5676",
-					ClientId:         "main-prometheus",
+					ClientID:         "main-prometheus",
 					HTTPClientConfig: config.DefaultHTTPClientConfig,
 					RefreshInterval:  model.Duration(15 * time.Second),
 					FetchTimeout:     model.Duration(2 * time.Minute),

From e27232614a37edbc40f7778c659806a23a3dd61b Mon Sep 17 00:00:00 2001
From: Marcin Skalski <skalskimarcin33@gmail.com>
Date: Tue, 12 Dec 2023 08:32:46 +0100
Subject: [PATCH 028/115] code review

Signed-off-by: Marcin Skalski <skalskimarcin33@gmail.com>
---
 discovery/xds/kuma.go | 22 ++++++++--------------
 1 file changed, 8 insertions(+), 14 deletions(-)

diff --git a/discovery/xds/kuma.go b/discovery/xds/kuma.go
index e79195bb2..567e5ab7c 100644
--- a/discovery/xds/kuma.go
+++ b/discovery/xds/kuma.go
@@ -178,11 +178,14 @@ func kumaMadsV1ResourceParser(resources []*anypb.Any, typeURL string) ([]model.L
 
 func NewKumaHTTPDiscovery(conf *KumaSDConfig, logger log.Logger) (discovery.Discoverer, error) {
 	// Default to "prometheus" if hostname is unavailable.
-	var clientID string
-	if conf.ClientID == "" {
-		clientID = defaultClientID(logger)
-	} else {
-		clientID = conf.ClientID
+	clientID := conf.ClientID
+	if clientID == "" {
+		var err error
+		clientID, err = osutil.GetFQDN()
+		if err != nil {
+			level.Debug(logger).Log("msg", "error getting FQDN", "err", err)
+			clientID = "prometheus"
+		}
 	}
 
 	clientConfig := &HTTPResourceClientConfig{
@@ -216,12 +219,3 @@ func NewKumaHTTPDiscovery(conf *KumaSDConfig, logger log.Logger) (discovery.Disc
 
 	return d, nil
 }
-
-func defaultClientID(logger log.Logger) string {
-	clientID, err := osutil.GetFQDN()
-	if err != nil {
-		level.Debug(logger).Log("msg", "error getting FQDN", "err", err)
-		clientID = "prometheus"
-	}
-	return clientID
-}

From f36b56a62ce01e9852d73bd25b310985b486a17c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?=
 <giedrius.statkevicius@vinted.com>
Date: Tue, 12 Dec 2023 11:58:54 +0200
Subject: [PATCH 029/115] tsdb: remove unused option (#13282)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Digging around the TSDB code and I've found that this flag is unused so
let's remove it.

Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
---
 cmd/prometheus/main.go |  1 -
 tsdb/db.go             | 35 +++++++++++++----------------------
 tsdb/db_test.go        |  3 ---
 3 files changed, 13 insertions(+), 26 deletions(-)

diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go
index 106f9d05c..7e8e23444 100644
--- a/cmd/prometheus/main.go
+++ b/cmd/prometheus/main.go
@@ -1597,7 +1597,6 @@ func (opts tsdbOptions) ToTSDBOptions() tsdb.Options {
 		RetentionDuration:              int64(time.Duration(opts.RetentionDuration) / time.Millisecond),
 		MaxBytes:                       int64(opts.MaxBytes),
 		NoLockfile:                     opts.NoLockfile,
-		AllowOverlappingCompaction:     true,
 		WALCompression:                 wlog.ParseCompressionType(opts.WALCompression, opts.WALCompressionType),
 		HeadChunksWriteQueueSize:       opts.HeadChunksWriteQueueSize,
 		SamplesPerChunk:                opts.SamplesPerChunk,
diff --git a/tsdb/db.go b/tsdb/db.go
index 6d283fdd8..b2cc37a19 100644
--- a/tsdb/db.go
+++ b/tsdb/db.go
@@ -70,20 +70,19 @@ var ErrNotReady = errors.New("TSDB not ready")
 // millisecond precision timestamps.
 func DefaultOptions() *Options {
 	return &Options{
-		WALSegmentSize:             wlog.DefaultSegmentSize,
-		MaxBlockChunkSegmentSize:   chunks.DefaultChunkSegmentSize,
-		RetentionDuration:          int64(15 * 24 * time.Hour / time.Millisecond),
-		MinBlockDuration:           DefaultBlockDuration,
-		MaxBlockDuration:           DefaultBlockDuration,
-		NoLockfile:                 false,
-		AllowOverlappingCompaction: true,
-		SamplesPerChunk:            DefaultSamplesPerChunk,
-		WALCompression:             wlog.CompressionNone,
-		StripeSize:                 DefaultStripeSize,
-		HeadChunksWriteBufferSize:  chunks.DefaultWriteBufferSize,
-		IsolationDisabled:          defaultIsolationDisabled,
-		HeadChunksWriteQueueSize:   chunks.DefaultWriteQueueSize,
-		OutOfOrderCapMax:           DefaultOutOfOrderCapMax,
+		WALSegmentSize:            wlog.DefaultSegmentSize,
+		MaxBlockChunkSegmentSize:  chunks.DefaultChunkSegmentSize,
+		RetentionDuration:         int64(15 * 24 * time.Hour / time.Millisecond),
+		MinBlockDuration:          DefaultBlockDuration,
+		MaxBlockDuration:          DefaultBlockDuration,
+		NoLockfile:                false,
+		SamplesPerChunk:           DefaultSamplesPerChunk,
+		WALCompression:            wlog.CompressionNone,
+		StripeSize:                DefaultStripeSize,
+		HeadChunksWriteBufferSize: chunks.DefaultWriteBufferSize,
+		IsolationDisabled:         defaultIsolationDisabled,
+		HeadChunksWriteQueueSize:  chunks.DefaultWriteQueueSize,
+		OutOfOrderCapMax:          DefaultOutOfOrderCapMax,
 	}
 }
 
@@ -115,14 +114,6 @@ type Options struct {
 	// NoLockfile disables creation and consideration of a lock file.
 	NoLockfile bool
 
-	// Compaction of overlapping blocks are allowed if AllowOverlappingCompaction is true.
-	// This is an optional flag for overlapping blocks.
-	// The reason why this flag exists is because there are various users of the TSDB
-	// that do not want vertical compaction happening on ingest time. Instead,
-	// they'd rather keep overlapping blocks and let another component do the overlapping compaction later.
-	// For Prometheus, this will always be true.
-	AllowOverlappingCompaction bool
-
 	// WALCompression configures the compression type to use on records in the WAL.
 	WALCompression wlog.CompressionType
 
diff --git a/tsdb/db_test.go b/tsdb/db_test.go
index cb12f8630..3bc094a3d 100644
--- a/tsdb/db_test.go
+++ b/tsdb/db_test.go
@@ -4877,7 +4877,6 @@ func Test_Querier_OOOQuery(t *testing.T) {
 	opts := DefaultOptions()
 	opts.OutOfOrderCapMax = 30
 	opts.OutOfOrderTimeWindow = 24 * time.Hour.Milliseconds()
-	opts.AllowOverlappingCompaction = false
 
 	series1 := labels.FromStrings("foo", "bar1")
 
@@ -4962,7 +4961,6 @@ func Test_ChunkQuerier_OOOQuery(t *testing.T) {
 	opts := DefaultOptions()
 	opts.OutOfOrderCapMax = 30
 	opts.OutOfOrderTimeWindow = 24 * time.Hour.Milliseconds()
-	opts.AllowOverlappingCompaction = false
 
 	series1 := labels.FromStrings("foo", "bar1")
 
@@ -6646,7 +6644,6 @@ func TestQueryHistogramFromBlocksWithCompaction(t *testing.T) {
 		t.Helper()
 
 		opts := DefaultOptions()
-		opts.AllowOverlappingCompaction = true // TODO(jesusvazquez): This replaced AllowOverlappingBlocks, make sure that works.
 		db := openTestDB(t, opts, nil)
 		t.Cleanup(func() {
 			require.NoError(t, db.Close())

From 5d5acf57446c23f42630c179bb5cac10ee2b6d17 Mon Sep 17 00:00:00 2001
From: Arianna Vespri <arianna.vespri@yahoo.it>
Date: Tue, 12 Dec 2023 12:40:08 +0100
Subject: [PATCH 030/115] Add unit protobuf parser

Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
---
 model/textparse/openmetricsparse.go | 1 -
 model/textparse/protobufparse.go    | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/model/textparse/openmetricsparse.go b/model/textparse/openmetricsparse.go
index bb5075544..72d4e8a4d 100644
--- a/model/textparse/openmetricsparse.go
+++ b/model/textparse/openmetricsparse.go
@@ -136,7 +136,6 @@ func (p *OpenMetricsParser) Type() ([]byte, MetricType) {
 // Must only be called after Next returned a unit entry.
 // The returned byte slices become invalid after the next call to Next.
 func (p *OpenMetricsParser) Unit() ([]byte, []byte) {
-	// The Prometheus format does not have units.
 	return p.l.b[p.offsets[0]:p.offsets[1]], p.text
 }
 
diff --git a/model/textparse/protobufparse.go b/model/textparse/protobufparse.go
index 23afb5c59..9dea097a2 100644
--- a/model/textparse/protobufparse.go
+++ b/model/textparse/protobufparse.go
@@ -272,7 +272,7 @@ func (p *ProtobufParser) Type() ([]byte, MetricType) {
 // Unit always returns (nil, nil) because units aren't supported by the protobuf
 // format.
 func (p *ProtobufParser) Unit() ([]byte, []byte) {
-	return nil, nil
+	return p.metricBytes.Bytes(), []byte(p.mf.GetUnit())
 }
 
 // Comment always returns nil because comments aren't supported by the protobuf

From 19709f75d05deb8f24c69788fa7b5a4939394734 Mon Sep 17 00:00:00 2001
From: Marcin Skalski <skalskimarcin33@gmail.com>
Date: Tue, 12 Dec 2023 14:49:43 +0100
Subject: [PATCH 031/115] fix kuma_sd docs

Signed-off-by: Marcin Skalski <skalskimarcin33@gmail.com>
---
 docs/configuration/configuration.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md
index 41f54dbd2..5e2f31c1c 100644
--- a/docs/configuration/configuration.md
+++ b/docs/configuration/configuration.md
@@ -2231,9 +2231,9 @@ See below for the configuration options for Kuma MonitoringAssignment discovery:
 server: <string>
 
 # Client id is used by Kuma Control Plane to compute Monitoring Assignment for specific Prometheus backend. 
-# This is useful when migrating between multiple Prometheus backends, or having separate backend for each Mesh
+# This is useful when migrating between multiple Prometheus backends, or having separate backend for each Mesh.
 # When not specified, system hostname/fqdn will be used if available, if not `prometheus` will be used.
-client_id: <string>
+[ client_id: <string> ]
 
 # The time to wait between polling update requests.
 [ refresh_interval: <duration> | default = 30s ]

From bb8363dbb30fc3864de8e837db11713022713958 Mon Sep 17 00:00:00 2001
From: Filip Petkovski <filip.petkovsky@gmail.com>
Date: Wed, 13 Dec 2023 08:30:02 +0100
Subject: [PATCH 032/115] Add comment on SampleRingIterator

Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
---
 promql/engine.go  |  7 +------
 storage/buffer.go | 10 ++--------
 2 files changed, 3 insertions(+), 14 deletions(-)

diff --git a/promql/engine.go b/promql/engine.go
index 8b4987921..16b8ee500 100644
--- a/promql/engine.go
+++ b/promql/engine.go
@@ -2114,12 +2114,7 @@ loop:
 				if floats == nil {
 					floats = getFPointSlice(16)
 				}
-				if n := len(floats); n < cap(floats) {
-					floats = floats[:n+1]
-					floats[n].T, floats[n].F = t, f
-				} else {
-					floats = append(floats, FPoint{T: t, F: f})
-				}
+				floats = append(floats, FPoint{T: t, F: f})
 			}
 		}
 	}
diff --git a/storage/buffer.go b/storage/buffer.go
index 75a935f9d..234771a77 100644
--- a/storage/buffer.go
+++ b/storage/buffer.go
@@ -310,6 +310,8 @@ func (r *sampleRing) iterator() *SampleRingIterator {
 	return &r.it
 }
 
+// SampleRingIterator is returned by BufferedSeriesIterator.Buffer() and can be
+// used to iterate samples buffered in the lookback window.
 type SampleRingIterator struct {
 	r  *sampleRing
 	i  int
@@ -358,14 +360,6 @@ func (it *SampleRingIterator) Next() chunkenc.ValueType {
 	}
 }
 
-func (it *SampleRingIterator) Seek(int64) chunkenc.ValueType {
-	return chunkenc.ValNone
-}
-
-func (it *SampleRingIterator) Err() error {
-	return nil
-}
-
 func (it *SampleRingIterator) At() (int64, float64) {
 	return it.t, it.f
 }

From ea356c472e9926d707f873ff1bef2f8aa5bd19f8 Mon Sep 17 00:00:00 2001
From: Filip Petkovski <filip.petkovsky@gmail.com>
Date: Wed, 13 Dec 2023 08:35:02 +0100
Subject: [PATCH 033/115] Add comment on SampleRingIterator methods

Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
---
 storage/buffer.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/storage/buffer.go b/storage/buffer.go
index 234771a77..d19f841d4 100644
--- a/storage/buffer.go
+++ b/storage/buffer.go
@@ -360,14 +360,20 @@ func (it *SampleRingIterator) Next() chunkenc.ValueType {
 	}
 }
 
+// At returns the current float element of the iterator.
 func (it *SampleRingIterator) At() (int64, float64) {
 	return it.t, it.f
 }
 
+// AtHistogram returns the current histogram element of the iterator.
 func (it *SampleRingIterator) AtHistogram() (int64, *histogram.Histogram) {
 	return it.t, it.h
 }
 
+// AtFloatHistogram returns the current histogram element of the iterator. If the
+// current sample is an integer histogram, it will be converted to a float histogram.
+// An optional histogram.FloatHistogram can be provided to avoid allocating a new
+// object for the conversion.
 func (it *SampleRingIterator) AtFloatHistogram(fh *histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
 	if it.fh == nil {
 		return it.t, it.h.ToFloat(fh)

From 103133124a55ad902b15bf02abe7eae5aecf4cc6 Mon Sep 17 00:00:00 2001
From: daniel-resdiary <109083091+daniel-resdiary@users.noreply.github.com>
Date: Wed, 13 Dec 2023 12:38:16 +0000
Subject: [PATCH 034/115] Get VM Scale Set NIC (#13283)

Calling `*armnetwork.InterfacesClient.Get()` doesn't work for Scale Set
VM NIC, because these use a different Resource ID format.

Use `*armnetwork.InterfacesClient.GetVirtualMachineScaleSetNetworkInterface()`
instead.  This needs both the scale set name and the instance ID, so
add an `InstanceID` field to the `virtualMachine` struct.  `InstanceID`
is empty for a VM that isn't a ScaleSetVM.

Signed-off-by: Daniel Nicholls <daniel.nicholls@resdiary.com>
---
 discovery/azure/azure.go      | 62 +++++++++++++++++++++++++++--------
 discovery/azure/azure_test.go |  6 ++++
 2 files changed, 55 insertions(+), 13 deletions(-)

diff --git a/discovery/azure/azure.go b/discovery/azure/azure.go
index 3b67a8102..4a85db232 100644
--- a/discovery/azure/azure.go
+++ b/discovery/azure/azure.go
@@ -95,7 +95,7 @@ func CloudConfigurationFromName(name string) (cloud.Configuration, error) {
 	name = strings.ToUpper(name)
 	env, ok := environments[name]
 	if !ok {
-		return env, fmt.Errorf("There is no cloud configuration matching the name %q", name)
+		return env, fmt.Errorf("there is no cloud configuration matching the name %q", name)
 	}
 
 	return env, nil
@@ -308,6 +308,7 @@ type virtualMachine struct {
 	Location          string
 	OsType            string
 	ScaleSet          string
+	InstanceID        string
 	Tags              map[string]*string
 	NetworkInterfaces []string
 	Size              string
@@ -408,17 +409,31 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 					networkInterface = v
 					d.cacheHitCount.Add(1)
 				} else {
-					networkInterface, err = client.getNetworkInterfaceByID(ctx, nicID)
-					if err != nil {
-						if errors.Is(err, errorNotFound) {
-							level.Warn(d.logger).Log("msg", "Network interface does not exist", "name", nicID, "err", err)
-						} else {
-							ch <- target{labelSet: nil, err: err}
+					if vm.ScaleSet == "" {
+						networkInterface, err = client.getVMNetworkInterfaceByID(ctx, nicID)
+						if err != nil {
+							if errors.Is(err, errorNotFound) {
+								level.Warn(d.logger).Log("msg", "Network interface does not exist", "name", nicID, "err", err)
+							} else {
+								ch <- target{labelSet: nil, err: err}
+							}
+							// Get out of this routine because we cannot continue without a network interface.
+							return
 						}
-						// Get out of this routine because we cannot continue without a network interface.
-						return
+						d.addToCache(nicID, networkInterface)
+					} else {
+						networkInterface, err = client.getVMScaleSetVMNetworkInterfaceByID(ctx, nicID, vm.ScaleSet, vm.InstanceID)
+						if err != nil {
+							if errors.Is(err, errorNotFound) {
+								level.Warn(d.logger).Log("msg", "Network interface does not exist", "name", nicID, "err", err)
+							} else {
+								ch <- target{labelSet: nil, err: err}
+							}
+							// Get out of this routine because we cannot continue without a network interface.
+							return
+						}
+						d.addToCache(nicID, networkInterface)
 					}
-					d.addToCache(nicID, networkInterface)
 				}
 
 				if networkInterface.Properties == nil {
@@ -626,6 +641,7 @@ func mapFromVMScaleSetVM(vm armcompute.VirtualMachineScaleSetVM, scaleSetName st
 		Location:          *(vm.Location),
 		OsType:            osType,
 		ScaleSet:          scaleSetName,
+		InstanceID:        *(vm.InstanceID),
 		Tags:              tags,
 		NetworkInterfaces: networkInterfaces,
 		Size:              size,
@@ -634,9 +650,9 @@ func mapFromVMScaleSetVM(vm armcompute.VirtualMachineScaleSetVM, scaleSetName st
 
 var errorNotFound = errors.New("network interface does not exist")
 
-// getNetworkInterfaceByID gets the network interface.
+// getVMNetworkInterfaceByID gets the network interface.
 // If a 404 is returned from the Azure API, `errorNotFound` is returned.
-func (client *azureClient) getNetworkInterfaceByID(ctx context.Context, networkInterfaceID string) (*armnetwork.Interface, error) {
+func (client *azureClient) getVMNetworkInterfaceByID(ctx context.Context, networkInterfaceID string) (*armnetwork.Interface, error) {
 	r, err := newAzureResourceFromID(networkInterfaceID, client.logger)
 	if err != nil {
 		return nil, fmt.Errorf("could not parse network interface ID: %w", err)
@@ -648,7 +664,27 @@ func (client *azureClient) getNetworkInterfaceByID(ctx context.Context, networkI
 		if errors.As(err, &responseError) && responseError.StatusCode == http.StatusNotFound {
 			return nil, errorNotFound
 		}
-		return nil, fmt.Errorf("Failed to retrieve Interface %v with error: %w", networkInterfaceID, err)
+		return nil, fmt.Errorf("failed to retrieve Interface %v with error: %w", networkInterfaceID, err)
+	}
+
+	return &resp.Interface, nil
+}
+
+// getVMScaleSetVMNetworkInterfaceByID gets the network interface.
+// If a 404 is returned from the Azure API, `errorNotFound` is returned.
+func (client *azureClient) getVMScaleSetVMNetworkInterfaceByID(ctx context.Context, networkInterfaceID, scaleSetName, instanceID string) (*armnetwork.Interface, error) {
+	r, err := newAzureResourceFromID(networkInterfaceID, client.logger)
+	if err != nil {
+		return nil, fmt.Errorf("could not parse network interface ID: %w", err)
+	}
+
+	resp, err := client.nic.GetVirtualMachineScaleSetNetworkInterface(ctx, r.ResourceGroupName, scaleSetName, instanceID, r.Name, &armnetwork.InterfacesClientGetVirtualMachineScaleSetNetworkInterfaceOptions{Expand: to.Ptr("IPConfigurations/PublicIPAddress")})
+	if err != nil {
+		var responseError *azcore.ResponseError
+		if errors.As(err, &responseError) && responseError.StatusCode == http.StatusNotFound {
+			return nil, errorNotFound
+		}
+		return nil, fmt.Errorf("failed to retrieve Interface %v with error: %w", networkInterfaceID, err)
 	}
 
 	return &resp.Interface, nil
diff --git a/discovery/azure/azure_test.go b/discovery/azure/azure_test.go
index 48f5b076c..024cf7591 100644
--- a/discovery/azure/azure_test.go
+++ b/discovery/azure/azure_test.go
@@ -142,6 +142,7 @@ func TestMapFromVMScaleSetVMWithEmptyTags(t *testing.T) {
 	vmSize := armcompute.VirtualMachineSizeTypes(size)
 	osType := armcompute.OperatingSystemTypesLinux
 	vmType := "type"
+	instanceID := "123"
 	location := "westeurope"
 	computerName := "computer_name"
 	networkProfile := armcompute.NetworkProfile{
@@ -166,6 +167,7 @@ func TestMapFromVMScaleSetVMWithEmptyTags(t *testing.T) {
 		ID:         &id,
 		Name:       &name,
 		Type:       &vmType,
+		InstanceID: &instanceID,
 		Location:   &location,
 		Tags:       nil,
 		Properties: properties,
@@ -182,6 +184,7 @@ func TestMapFromVMScaleSetVMWithEmptyTags(t *testing.T) {
 		Tags:              map[string]*string{},
 		NetworkInterfaces: []string{},
 		ScaleSet:          scaleSet,
+		InstanceID:        instanceID,
 		Size:              size,
 	}
 
@@ -197,6 +200,7 @@ func TestMapFromVMScaleSetVMWithTags(t *testing.T) {
 	vmSize := armcompute.VirtualMachineSizeTypes(size)
 	osType := armcompute.OperatingSystemTypesLinux
 	vmType := "type"
+	instanceID := "123"
 	location := "westeurope"
 	computerName := "computer_name"
 	tags := map[string]*string{
@@ -224,6 +228,7 @@ func TestMapFromVMScaleSetVMWithTags(t *testing.T) {
 		ID:         &id,
 		Name:       &name,
 		Type:       &vmType,
+		InstanceID: &instanceID,
 		Location:   &location,
 		Tags:       tags,
 		Properties: properties,
@@ -240,6 +245,7 @@ func TestMapFromVMScaleSetVMWithTags(t *testing.T) {
 		Tags:              tags,
 		NetworkInterfaces: []string{},
 		ScaleSet:          scaleSet,
+		InstanceID:        instanceID,
 		Size:              size,
 	}
 

From 0704c7254876fefbdde85fecb8ad389106312af2 Mon Sep 17 00:00:00 2001
From: Daniel Nicholls <daniel.nicholls@resdiary.com>
Date: Wed, 13 Dec 2023 14:14:01 +0000
Subject: [PATCH 035/115] Dedup code handling getting network interface

Signed-off-by: Daniel Nicholls <daniel.nicholls@resdiary.com>
---
 discovery/azure/azure.go | 30 ++++++++++--------------------
 1 file changed, 10 insertions(+), 20 deletions(-)

diff --git a/discovery/azure/azure.go b/discovery/azure/azure.go
index 4a85db232..2a0f8380b 100644
--- a/discovery/azure/azure.go
+++ b/discovery/azure/azure.go
@@ -411,29 +411,19 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 				} else {
 					if vm.ScaleSet == "" {
 						networkInterface, err = client.getVMNetworkInterfaceByID(ctx, nicID)
-						if err != nil {
-							if errors.Is(err, errorNotFound) {
-								level.Warn(d.logger).Log("msg", "Network interface does not exist", "name", nicID, "err", err)
-							} else {
-								ch <- target{labelSet: nil, err: err}
-							}
-							// Get out of this routine because we cannot continue without a network interface.
-							return
-						}
-						d.addToCache(nicID, networkInterface)
 					} else {
 						networkInterface, err = client.getVMScaleSetVMNetworkInterfaceByID(ctx, nicID, vm.ScaleSet, vm.InstanceID)
-						if err != nil {
-							if errors.Is(err, errorNotFound) {
-								level.Warn(d.logger).Log("msg", "Network interface does not exist", "name", nicID, "err", err)
-							} else {
-								ch <- target{labelSet: nil, err: err}
-							}
-							// Get out of this routine because we cannot continue without a network interface.
-							return
-						}
-						d.addToCache(nicID, networkInterface)
 					}
+					if err != nil {
+						if errors.Is(err, errorNotFound) {
+							level.Warn(d.logger).Log("msg", "Network interface does not exist", "name", nicID, "err", err)
+						} else {
+							ch <- target{labelSet: nil, err: err}
+						}
+						// Get out of this routine because we cannot continue without a network interface.
+						return
+					}
+					d.addToCache(nicID, networkInterface)
 				}
 
 				if networkInterface.Properties == nil {

From 69abd6d9f65a6358025461510237366f1d939c70 Mon Sep 17 00:00:00 2001
From: Arthur Silva Sens <arthur.sens@coralogix.com>
Date: Wed, 13 Dec 2023 12:12:50 -0300
Subject: [PATCH 036/115] Document feature flag
 'created-timestamp-zero-ingestion' (#13279)

Signed-off-by: Arthur Silva Sens <arthur.sens@coralogix.com>
---
 docs/feature_flags.md | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/docs/feature_flags.md b/docs/feature_flags.md
index 0e585f13c..bcf8309b5 100644
--- a/docs/feature_flags.md
+++ b/docs/feature_flags.md
@@ -194,3 +194,13 @@ won't work when you push OTLP metrics.
 
 Enables PromQL functions that are considered experimental and whose name or
 semantics could change.
+
+## Created Timestamps Zero Injection
+
+`--enable-feature=created-timestamp-zero-ingestion`
+
+Enables ingestion of created timestamp. Created timestamps are injected as 0 valued samples when appropriate. See [PromCon talk](https://youtu.be/nWf0BfQ5EEA) for details.
+
+Currently Prometheus supports created timestamps only on the traditional Prometheus Protobuf protocol (WIP for other protocols). As a result, when enabling this feature, the Prometheus protobuf scrape protocol will be prioritized (See `scrape_config.scrape_protocols` settings for more details).
+
+Besides enabling this feature in Prometheus, created timestamps need to be exposed by the application being scraped.

From 1a8381a5011e17328d059ab80d3963cac3e507b4 Mon Sep 17 00:00:00 2001
From: Diogo Teles Sant'Anna <diogoteles@google.com>
Date: Wed, 13 Dec 2023 16:30:06 -0300
Subject: [PATCH 037/115] Add minimal permissions to script golangci-lint.yml

Signed-off-by: Diogo Teles Sant'Anna <diogoteles@google.com>
---
 scripts/golangci-lint.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/scripts/golangci-lint.yml b/scripts/golangci-lint.yml
index 805c59fb7..4b292229d 100644
--- a/scripts/golangci-lint.yml
+++ b/scripts/golangci-lint.yml
@@ -12,8 +12,14 @@ on:
       - ".golangci.yml"
   pull_request:
 
+permissions:  # added using https://github.com/step-security/secure-repo
+  contents: read
+
 jobs:
   golangci:
+    permissions:
+      contents: read  # for actions/checkout to fetch code
+      pull-requests: read  # for golangci/golangci-lint-action to fetch pull requests
     name: lint
     runs-on: ubuntu-latest
     steps:

From 1f69dcfa6bfb5c53dacbe33b8aca45a6b7540cc8 Mon Sep 17 00:00:00 2001
From: Filip Petkovski <filip.petkovsky@gmail.com>
Date: Thu, 14 Dec 2023 11:28:15 +0100
Subject: [PATCH 038/115] Fix reusing float histograms

In https://github.com/prometheus/prometheus/pull/13276 we started reusing float histogram objects to reduce allocations in PromQL.
That PR introduces a bug where histogram pointers gets copied to the beginning of the histograms slice,
but are still kept in the end of the slice. When a new histogram is read into the last element,
it can overwrite a previous element because the pointer is the same.

This commit fixes the issue by moving outdated points to the end of the slice
so that we don't end up with duplicate pointers in the same buffer. In other words,
the slice gets rotated so that old objects can get reused.

Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
---
 promql/engine.go           |  5 +++
 promql/engine_test.go      | 91 +++++++++++++++++++++++++++++---------
 tsdb/tsdbutil/histogram.go |  8 ++++
 3 files changed, 82 insertions(+), 22 deletions(-)

diff --git a/promql/engine.go b/promql/engine.go
index 16b8ee500..12755663d 100644
--- a/promql/engine.go
+++ b/promql/engine.go
@@ -2052,7 +2052,12 @@ func (ev *evaluator) matrixIterSlice(
 		var drop int
 		for drop = 0; histograms[drop].T < mint; drop++ {
 		}
+		// Rotate the buffer around the drop index so that points before mint can be
+		// reused to store new histograms.
+		tail := make([]HPoint, drop)
+		copy(tail, histograms[:drop])
 		copy(histograms, histograms[drop:])
+		copy(histograms[len(histograms)-drop:], tail)
 		histograms = histograms[:len(histograms)-drop]
 		ev.currentSamples -= totalHPointSize(histograms)
 		// Only append points with timestamps after the last timestamp we have.
diff --git a/promql/engine_test.go b/promql/engine_test.go
index 9ab54dd16..105cdc10d 100644
--- a/promql/engine_test.go
+++ b/promql/engine_test.go
@@ -3169,28 +3169,75 @@ func TestNativeHistogramRate(t *testing.T) {
 	}
 	require.NoError(t, app.Commit())
 
-	queryString := fmt.Sprintf("rate(%s[1m])", seriesName)
-	qry, err := engine.NewInstantQuery(context.Background(), storage, nil, queryString, timestamp.Time(int64(5*time.Minute/time.Millisecond)))
-	require.NoError(t, err)
-	res := qry.Exec(context.Background())
-	require.NoError(t, res.Err)
-	vector, err := res.Vector()
-	require.NoError(t, err)
-	require.Len(t, vector, 1)
-	actualHistogram := vector[0].H
-	expectedHistogram := &histogram.FloatHistogram{
-		CounterResetHint: histogram.GaugeType,
-		Schema:           1,
-		ZeroThreshold:    0.001,
-		ZeroCount:        1. / 15.,
-		Count:            9. / 15.,
-		Sum:              1.226666666666667,
-		PositiveSpans:    []histogram.Span{{Offset: 0, Length: 2}, {Offset: 1, Length: 2}},
-		PositiveBuckets:  []float64{1. / 15., 1. / 15., 1. / 15., 1. / 15.},
-		NegativeSpans:    []histogram.Span{{Offset: 0, Length: 2}, {Offset: 1, Length: 2}},
-		NegativeBuckets:  []float64{1. / 15., 1. / 15., 1. / 15., 1. / 15.},
-	}
-	require.Equal(t, expectedHistogram, actualHistogram)
+	queryString := fmt.Sprintf("rate(%s[45s])", seriesName)
+	t.Run("instant_query", func(t *testing.T) {
+		qry, err := engine.NewInstantQuery(context.Background(), storage, nil, queryString, timestamp.Time(int64(5*time.Minute/time.Millisecond)))
+		require.NoError(t, err)
+		res := qry.Exec(context.Background())
+		require.NoError(t, res.Err)
+		vector, err := res.Vector()
+		require.NoError(t, err)
+		require.Len(t, vector, 1)
+		actualHistogram := vector[0].H
+		expectedHistogram := &histogram.FloatHistogram{
+			CounterResetHint: histogram.GaugeType,
+			Schema:           1,
+			ZeroThreshold:    0.001,
+			ZeroCount:        1. / 15.,
+			Count:            9. / 15.,
+			Sum:              1.2266666666666663,
+			PositiveSpans:    []histogram.Span{{Offset: 0, Length: 2}, {Offset: 1, Length: 2}},
+			PositiveBuckets:  []float64{1. / 15., 1. / 15., 1. / 15., 1. / 15.},
+			NegativeSpans:    []histogram.Span{{Offset: 0, Length: 2}, {Offset: 1, Length: 2}},
+			NegativeBuckets:  []float64{1. / 15., 1. / 15., 1. / 15., 1. / 15.},
+		}
+		require.Equal(t, expectedHistogram, actualHistogram)
+	})
+
+	t.Run("range_query", func(t *testing.T) {
+		step := 30 * time.Second
+		start := timestamp.Time(int64(5 * time.Minute / time.Millisecond))
+		end := start.Add(step)
+		qry, err := engine.NewRangeQuery(context.Background(), storage, nil, queryString, start, end, step)
+		require.NoError(t, err)
+		res := qry.Exec(context.Background())
+		require.NoError(t, res.Err)
+		matrix, err := res.Matrix()
+		require.NoError(t, err)
+		require.Len(t, matrix, 1)
+		require.Len(t, matrix[0].Histograms, 2)
+		actualHistograms := matrix[0].Histograms
+		expectedHistograms := []HPoint{{
+			T: 300000,
+			H: &histogram.FloatHistogram{
+				CounterResetHint: histogram.GaugeType,
+				Schema:           1,
+				ZeroThreshold:    0.001,
+				ZeroCount:        1. / 15.,
+				Count:            9. / 15.,
+				Sum:              1.2266666666666663,
+				PositiveSpans:    []histogram.Span{{Offset: 0, Length: 2}, {Offset: 1, Length: 2}},
+				PositiveBuckets:  []float64{1. / 15., 1. / 15., 1. / 15., 1. / 15.},
+				NegativeSpans:    []histogram.Span{{Offset: 0, Length: 2}, {Offset: 1, Length: 2}},
+				NegativeBuckets:  []float64{1. / 15., 1. / 15., 1. / 15., 1. / 15.},
+			},
+		}, {
+			T: 330000,
+			H: &histogram.FloatHistogram{
+				CounterResetHint: histogram.GaugeType,
+				Schema:           1,
+				ZeroThreshold:    0.001,
+				ZeroCount:        1. / 15.,
+				Count:            9. / 15.,
+				Sum:              1.2266666666666663,
+				PositiveSpans:    []histogram.Span{{Offset: 0, Length: 2}, {Offset: 1, Length: 2}},
+				PositiveBuckets:  []float64{1. / 15., 1. / 15., 1. / 15., 1. / 15.},
+				NegativeSpans:    []histogram.Span{{Offset: 0, Length: 2}, {Offset: 1, Length: 2}},
+				NegativeBuckets:  []float64{1. / 15., 1. / 15., 1. / 15., 1. / 15.},
+			},
+		}}
+		require.Equal(t, expectedHistograms, actualHistograms)
+	})
 }
 
 func TestNativeFloatHistogramRate(t *testing.T) {
diff --git a/tsdb/tsdbutil/histogram.go b/tsdb/tsdbutil/histogram.go
index 0847f81a8..bb8d49b20 100644
--- a/tsdb/tsdbutil/histogram.go
+++ b/tsdb/tsdbutil/histogram.go
@@ -30,6 +30,14 @@ func GenerateTestHistograms(n int) (r []*histogram.Histogram) {
 	return r
 }
 
+func GenerateTestHistogramsWithUnknownResetHint(n int) []*histogram.Histogram {
+	hs := GenerateTestHistograms(n)
+	for i := range hs {
+		hs[i].CounterResetHint = histogram.UnknownCounterReset
+	}
+	return hs
+}
+
 // GenerateTestHistogram but it is up to the user to set any known counter reset hint.
 func GenerateTestHistogram(i int) *histogram.Histogram {
 	return &histogram.Histogram{

From 9fb1e9715ce9871ab95a0c0ed806e9e41a9ac7c9 Mon Sep 17 00:00:00 2001
From: Arianna Vespri <arianna.vespri@yahoo.it>
Date: Thu, 14 Dec 2023 17:00:52 +0100
Subject: [PATCH 039/115] Go on adding protobuf parsing for unit

Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
---
 model/textparse/protobufparse.go      | 14 ++++++++++++--
 model/textparse/protobufparse_test.go |  2 ++
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/model/textparse/protobufparse.go b/model/textparse/protobufparse.go
index 9dea097a2..21329e2a0 100644
--- a/model/textparse/protobufparse.go
+++ b/model/textparse/protobufparse.go
@@ -269,8 +269,9 @@ func (p *ProtobufParser) Type() ([]byte, MetricType) {
 	return n, MetricTypeUnknown
 }
 
-// Unit always returns (nil, nil) because units aren't supported by the protobuf
-// format.
+// Unit returns the metric unit in the current entry.
+// Must only be called after Next returned a unit entry.
+// The returned byte slices become invalid after the next call to Next.
 func (p *ProtobufParser) Unit() ([]byte, []byte) {
 	return p.metricBytes.Bytes(), []byte(p.mf.GetUnit())
 }
@@ -418,6 +419,13 @@ func (p *ProtobufParser) Next() (Entry, error) {
 		default:
 			return EntryInvalid, fmt.Errorf("unknown metric type for metric %q: %s", name, p.mf.GetType())
 		}
+		unit := p.mf.GetUnit()
+		fmt.Println("This is the unit:", unit)
+		if len(unit) > 0 {
+			if !strings.HasSuffix(name, fmt.Sprintf("_%s_total", unit)) || !strings.HasSuffix(name, fmt.Sprintf("_%s", unit)) || len(name) < len(unit)+1 {
+				return EntryInvalid, fmt.Errorf("invalid unit for metric %q: %s", name, unit)
+			}
+		}
 		p.metricBytes.Reset()
 		p.metricBytes.WriteString(name)
 
@@ -435,6 +443,8 @@ func (p *ProtobufParser) Next() (Entry, error) {
 		if err := p.updateMetricBytes(); err != nil {
 			return EntryInvalid, err
 		}
+	// case EntryUnit:
+	//	p.state = EntryType // what is the right state and place for this?
 	case EntryHistogram, EntrySeries:
 		if p.redoClassic {
 			p.redoClassic = false
diff --git a/model/textparse/protobufparse_test.go b/model/textparse/protobufparse_test.go
index e062e64dd..a5b22e360 100644
--- a/model/textparse/protobufparse_test.go
+++ b/model/textparse/protobufparse_test.go
@@ -58,6 +58,7 @@ metric: <
 		`name: "go_memstats_alloc_bytes_total"
 help: "Total number of bytes allocated, even if freed."
 type: COUNTER
+unit: "bytes"
 metric: <
   counter: <
     value: 1.546544e+06
@@ -665,6 +666,7 @@ func TestProtobufParse(t *testing.T) {
 				{
 					m:    "go_memstats_alloc_bytes_total",
 					help: "Total number of bytes allocated, even if freed.",
+					unit: "bytes",
 				},
 				{
 					m:   "go_memstats_alloc_bytes_total",

From 952cb41373c00a46c060ade00e8ecc85603ef758 Mon Sep 17 00:00:00 2001
From: Matthieu MOREL <matthieu.morel35@gmail.com>
Date: Fri, 15 Dec 2023 10:21:18 +0000
Subject: [PATCH 040/115] build(deps): bump
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager

Signed-off-by: Matthieu MOREL <matthieu.morel35@gmail.com>
---
 discovery/azure/azure.go      |  4 ++--
 discovery/azure/azure_test.go |  2 +-
 go.mod                        |  7 +++----
 go.sum                        | 14 ++++++--------
 4 files changed, 12 insertions(+), 15 deletions(-)

diff --git a/discovery/azure/azure.go b/discovery/azure/azure.go
index 4a85db232..6637e9800 100644
--- a/discovery/azure/azure.go
+++ b/discovery/azure/azure.go
@@ -30,8 +30,8 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
 	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
-	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4"
-	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4"
 	cache "github.com/Code-Hex/go-generics-cache"
 	"github.com/Code-Hex/go-generics-cache/policy/lru"
 	"github.com/go-kit/log"
diff --git a/discovery/azure/azure_test.go b/discovery/azure/azure_test.go
index 024cf7591..4ff937e0b 100644
--- a/discovery/azure/azure_test.go
+++ b/discovery/azure/azure_test.go
@@ -17,7 +17,7 @@ import (
 	"testing"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/arm"
-	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
 )
diff --git a/go.mod b/go.mod
index 770607e9c..9ea078083 100644
--- a/go.mod
+++ b/go.mod
@@ -5,8 +5,9 @@ go 1.20
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0
-	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4 v4.2.1
-	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.3.0
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0
+	github.com/Code-Hex/go-generics-cache v1.3.1
 	github.com/alecthomas/kingpin/v2 v2.4.0
 	github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9
 	github.com/aws/aws-sdk-go v1.48.14
@@ -93,9 +94,7 @@ require (
 	cloud.google.com/go/compute v1.23.3 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect
-	github.com/Code-Hex/go-generics-cache v1.3.1
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
diff --git a/go.sum b/go.sum
index ae367bb90..a6f535dd7 100644
--- a/go.sum
+++ b/go.sum
@@ -40,14 +40,12 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZM
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG+S3q8UoJcmyU6nUeunJcMDHcRYHhs=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 h1:d81/ng9rET2YqdVkVwkb6EXeRrLJIwyGnJcAlAWKwhs=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4 v4.2.1 h1:UPeCRD+XY7QlaGQte2EVI2iOcWvUYA2XY8w5T/8v0NQ=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4 v4.2.1/go.mod h1:oGV6NlB0cvi1ZbYRR2UN44QHxWFyGk+iylgD0qaMXjA=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.1.2 h1:mLY+pNLjCUeKhgnAJWAKhEUQM+RJQo2H1fuGSw1Ky1E=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0 h1:QM6sE5k2ZT/vI5BEe0r7mqjsUSnhVBFbOsVkEuaEfiA=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0/go.mod h1:243D9iHbcQXoFUtgHJwL7gl2zx1aDuDMjvBZVGr2uW0=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1 h1:bWh0Z2rOEDfB/ywv/l0iHN1JgyazE6kW/aIA89+CEK0=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1/go.mod h1:Bzf34hhAE9NSxailk8xVeLEZbUjOXcC+GnU1mMKdhLw=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.0.0 h1:ECsQtyERDVz3NP3kvDOTLvbQhqWp/x9EsGKtb4ogUr8=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.3.0 h1:qgs/VAMSR+9qFhwTw4OwF2NbVuw+2m83pVZJjqkKQMw=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.3.0/go.mod h1:uYt4CfhkJA9o0FN7jfE5minm/i4nUE4MjGUJkzB6Zs8=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0 h1:bXwSugBiSbgtz7rOtbfGf+woewp4f06orW9OP5BjHLA=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0/go.mod h1:Y/HgrePTmGy9HjdSGTqZNa+apUpTVIEVKXJyARP2lrk=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 h1:7CBQ+Ei8SP2c6ydQTGCCrS35bDxgTMfoP2miAwK++OU=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 h1:WpB/QDNLpMw72xHJc34BNNykqSOeEJDAWkhf0u12/Jk=

From ea9708648434513cb543e00b745a2f4cff9570f5 Mon Sep 17 00:00:00 2001
From: machine424 <ayoubmrini424@gmail.com>
Date: Fri, 15 Dec 2023 18:03:50 +0100
Subject: [PATCH 041/115] ui: create a reproduction for
 https://github.com/prometheus/prometheus/issues/13292

Signed-off-by: machine424 <ayoubmrini424@gmail.com>
---
 .../pages/graph/GraphHeatmapHelpers.test.ts   | 27 +++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 web/ui/react-app/src/pages/graph/GraphHeatmapHelpers.test.ts

diff --git a/web/ui/react-app/src/pages/graph/GraphHeatmapHelpers.test.ts b/web/ui/react-app/src/pages/graph/GraphHeatmapHelpers.test.ts
new file mode 100644
index 000000000..dfc718587
--- /dev/null
+++ b/web/ui/react-app/src/pages/graph/GraphHeatmapHelpers.test.ts
@@ -0,0 +1,27 @@
+import { isHeatmapData } from './GraphHeatmapHelpers';
+
+describe('GraphHeatmapHelpers', () => {
+  it('isHeatmapData should return false for a vector', () => {
+    const data = {
+      resultType: 'vector',
+      result: [
+        {
+          metric: {
+            __name__: 'my_gauge',
+            job: 'target',
+          },
+          value: [1703091180.683, '6'],
+        },
+      ],
+    };
+    expect(isHeatmapData(data)).toBe(false);
+  });
+
+  it('isHeatmapData should return false for scalar resultType', () => {
+    const data = {
+      resultType: 'scalar',
+      result: [1703091180.125, '1703091180.125'],
+    };
+    expect(isHeatmapData(data)).toBe(false);
+  });
+});

From b65021d7a590810164d99afd99a13915e6ea38ec Mon Sep 17 00:00:00 2001
From: Arianna Vespri <arianna.vespri@yahoo.it>
Date: Sun, 17 Dec 2023 14:09:14 +0100
Subject: [PATCH 042/115] Get conditional right

Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
---
 model/textparse/protobufparse.go | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/model/textparse/protobufparse.go b/model/textparse/protobufparse.go
index 21329e2a0..685ad08f1 100644
--- a/model/textparse/protobufparse.go
+++ b/model/textparse/protobufparse.go
@@ -421,8 +421,13 @@ func (p *ProtobufParser) Next() (Entry, error) {
 		}
 		unit := p.mf.GetUnit()
 		fmt.Println("This is the unit:", unit)
+		fmt.Println("This is the name:", name)
 		if len(unit) > 0 {
-			if !strings.HasSuffix(name, fmt.Sprintf("_%s_total", unit)) || !strings.HasSuffix(name, fmt.Sprintf("_%s", unit)) || len(name) < len(unit)+1 {
+			sfx := fmt.Sprintf("_%s", unit)
+			if p.mf.GetType() == dto.MetricType_COUNTER {
+				sfx = fmt.Sprintf("_%s_total", unit)
+			}
+			if !strings.HasSuffix(name, sfx) {
 				return EntryInvalid, fmt.Errorf("invalid unit for metric %q: %s", name, unit)
 			}
 		}
@@ -443,8 +448,6 @@ func (p *ProtobufParser) Next() (Entry, error) {
 		if err := p.updateMetricBytes(); err != nil {
 			return EntryInvalid, err
 		}
-	// case EntryUnit:
-	//	p.state = EntryType // what is the right state and place for this?
 	case EntryHistogram, EntrySeries:
 		if p.redoClassic {
 			p.redoClassic = false

From 9ab7e3b3dec4b82870487154dce53baa3266a614 Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Mon, 18 Dec 2023 14:52:42 +0000
Subject: [PATCH 043/115] relabel: refactor: extract config.Validate method

And add a test for it, which fails because validation is not strong enough.

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
---
 model/relabel/relabel.go      |  4 ++
 model/relabel/relabel_test.go | 72 +++++++++++++++++++++++++++++++++++
 2 files changed, 76 insertions(+)

diff --git a/model/relabel/relabel.go b/model/relabel/relabel.go
index fadf35b86..fa0d809de 100644
--- a/model/relabel/relabel.go
+++ b/model/relabel/relabel.go
@@ -108,6 +108,10 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	if c.Regex.Regexp == nil {
 		c.Regex = MustNewRegexp("")
 	}
+	return c.Validate()
+}
+
+func (c *Config) Validate() error {
 	if c.Action == "" {
 		return fmt.Errorf("relabel action cannot be empty")
 	}
diff --git a/model/relabel/relabel_test.go b/model/relabel/relabel_test.go
index b50ff4010..fe040be3a 100644
--- a/model/relabel/relabel_test.go
+++ b/model/relabel/relabel_test.go
@@ -14,6 +14,7 @@
 package relabel
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/prometheus/common/model"
@@ -575,6 +576,77 @@ func TestRelabel(t *testing.T) {
 	}
 }
 
+func TestRelabelValidate(t *testing.T) {
+	tests := []struct {
+		config   Config
+		expected string
+	}{
+		{
+			config:   Config{},
+			expected: `relabel action cannot be empty`,
+		},
+		{
+			config: Config{
+				Action: Replace,
+			},
+			expected: `requires 'target_label' value`,
+		},
+		{
+			config: Config{
+				Action: Lowercase,
+			},
+			expected: `requires 'target_label' value`,
+		},
+		{
+			config: Config{
+				Action:      Lowercase,
+				Replacement: DefaultRelabelConfig.Replacement,
+				TargetLabel: "${3}",
+			},
+			expected: `"${3}" is invalid 'target_label'`,
+		},
+		{
+			config: Config{
+				SourceLabels: model.LabelNames{"a"},
+				Regex:        MustNewRegexp("some-([^-]+)-([^,]+)"),
+				Action:       Replace,
+				Replacement:  "${1}",
+				TargetLabel:  "${3}",
+			},
+		},
+		{
+			config: Config{
+				SourceLabels: model.LabelNames{"a"},
+				Regex:        MustNewRegexp("some-([^-]+)-([^,]+)"),
+				Action:       Replace,
+				Replacement:  "${1}",
+				TargetLabel:  "0${3}",
+			},
+			expected: `"0${3}" is invalid 'target_label'`,
+		},
+		{
+			config: Config{
+				SourceLabels: model.LabelNames{"a"},
+				Regex:        MustNewRegexp("some-([^-]+)-([^,]+)"),
+				Action:       Replace,
+				Replacement:  "${1}",
+				TargetLabel:  "-${3}",
+			},
+			expected: `"-${3}" is invalid 'target_label' for replace action`,
+		},
+	}
+	for i, test := range tests {
+		t.Run(fmt.Sprint(i), func(t *testing.T) {
+			err := test.config.Validate()
+			if test.expected == "" {
+				require.NoError(t, err)
+			} else {
+				require.ErrorContains(t, err, test.expected)
+			}
+		})
+	}
+}
+
 func TestTargetLabelValidity(t *testing.T) {
 	tests := []struct {
 		str   string

From 2d4c367d87cdd99eac1b4786cf1621c58d43735d Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Mon, 18 Dec 2023 14:58:56 +0000
Subject: [PATCH 044/115] relabel: stricter check that target labels are valid

For `Lowercase`, `KeepEqual`, etc., we do not expand a regexp, so
the target label name must not contain anything like `${1}`.

Also for the common case that the `Replace` target does not require any
template expansion, check that the entire string passes label name
validity rules.

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
---
 model/relabel/relabel.go | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/model/relabel/relabel.go b/model/relabel/relabel.go
index fa0d809de..1947e6273 100644
--- a/model/relabel/relabel.go
+++ b/model/relabel/relabel.go
@@ -121,7 +121,13 @@ func (c *Config) Validate() error {
 	if (c.Action == Replace || c.Action == HashMod || c.Action == Lowercase || c.Action == Uppercase || c.Action == KeepEqual || c.Action == DropEqual) && c.TargetLabel == "" {
 		return fmt.Errorf("relabel configuration for %s action requires 'target_label' value", c.Action)
 	}
-	if (c.Action == Replace || c.Action == Lowercase || c.Action == Uppercase || c.Action == KeepEqual || c.Action == DropEqual) && !relabelTarget.MatchString(c.TargetLabel) {
+	if c.Action == Replace && !strings.Contains(c.TargetLabel, "$") && !model.LabelName(c.TargetLabel).IsValid() {
+		return fmt.Errorf("%q is invalid 'target_label' for %s action", c.TargetLabel, c.Action)
+	}
+	if c.Action == Replace && strings.Contains(c.TargetLabel, "$") && !relabelTarget.MatchString(c.TargetLabel) {
+		return fmt.Errorf("%q is invalid 'target_label' for %s action", c.TargetLabel, c.Action)
+	}
+	if (c.Action == Lowercase || c.Action == Uppercase || c.Action == KeepEqual || c.Action == DropEqual) && !model.LabelName(c.TargetLabel).IsValid() {
 		return fmt.Errorf("%q is invalid 'target_label' for %s action", c.TargetLabel, c.Action)
 	}
 	if (c.Action == Lowercase || c.Action == Uppercase || c.Action == KeepEqual || c.Action == DropEqual) && c.Replacement != DefaultRelabelConfig.Replacement {

From 000182e4b8638bfe781a583c27a1e8a366ac82bf Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Mon, 18 Dec 2023 15:03:21 +0000
Subject: [PATCH 045/115] relabel: check validity of all test cases

Thought this would be a nice check on the `Validate()` function, but
some of the test cases needed tweaking to pass.

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
---
 model/relabel/relabel_test.go | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/model/relabel/relabel_test.go b/model/relabel/relabel_test.go
index fe040be3a..2b8fc911d 100644
--- a/model/relabel/relabel_test.go
+++ b/model/relabel/relabel_test.go
@@ -335,7 +335,7 @@ func TestRelabel(t *testing.T) {
 		},
 		{ // invalid target_labels
 			input: labels.FromMap(map[string]string{
-				"a": "some-name-value",
+				"a": "some-name-0",
 			}),
 			relabel: []*Config{
 				{
@@ -350,18 +350,18 @@ func TestRelabel(t *testing.T) {
 					Regex:        MustNewRegexp("some-([^-]+)-([^,]+)"),
 					Action:       Replace,
 					Replacement:  "${1}",
-					TargetLabel:  "0${3}",
+					TargetLabel:  "${3}",
 				},
 				{
 					SourceLabels: model.LabelNames{"a"},
-					Regex:        MustNewRegexp("some-([^-]+)-([^,]+)"),
+					Regex:        MustNewRegexp("some-([^-]+)(-[^,]+)"),
 					Action:       Replace,
 					Replacement:  "${1}",
-					TargetLabel:  "-${3}",
+					TargetLabel:  "${3}",
 				},
 			},
 			output: labels.FromMap(map[string]string{
-				"a": "some-name-value",
+				"a": "some-name-0",
 			}),
 		},
 		{ // more complex real-life like usecase
@@ -566,6 +566,7 @@ func TestRelabel(t *testing.T) {
 			if cfg.Replacement == "" {
 				cfg.Replacement = DefaultRelabelConfig.Replacement
 			}
+			require.NoError(t, cfg.Validate())
 		}
 
 		res, keep := Process(test.input, test.relabel...)

From 0289dd61571c7a812a235ecec5bc3f74fd7ccf50 Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Mon, 18 Dec 2023 16:38:59 +0000
Subject: [PATCH 046/115] relabel: blank replacement deletes label post-regexp

If `cfg.TargetLabel` is a template like `$1`, it won't match any labels,
so no point calling `lb.Del` with it.

Similarly if `target` is not a valid label name, it won't match any
labels, so don't call with that either.

The intention seems to have been that a blank _value_ would delete the
target, so change that code to use `target` instead of `cfg.TargetLabel`.

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
---
 model/relabel/relabel.go      |  3 +--
 model/relabel/relabel_test.go | 19 +++++++++++++++++++
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/model/relabel/relabel.go b/model/relabel/relabel.go
index 1947e6273..d29c3d07a 100644
--- a/model/relabel/relabel.go
+++ b/model/relabel/relabel.go
@@ -274,12 +274,11 @@ func relabel(cfg *Config, lb *labels.Builder) (keep bool) {
 		}
 		target := model.LabelName(cfg.Regex.ExpandString([]byte{}, cfg.TargetLabel, val, indexes))
 		if !target.IsValid() {
-			lb.Del(cfg.TargetLabel)
 			break
 		}
 		res := cfg.Regex.ExpandString([]byte{}, cfg.Replacement, val, indexes)
 		if len(res) == 0 {
-			lb.Del(cfg.TargetLabel)
+			lb.Del(string(target))
 			break
 		}
 		lb.Set(string(target), string(res))
diff --git a/model/relabel/relabel_test.go b/model/relabel/relabel_test.go
index 2b8fc911d..517b9b822 100644
--- a/model/relabel/relabel_test.go
+++ b/model/relabel/relabel_test.go
@@ -214,6 +214,25 @@ func TestRelabel(t *testing.T) {
 				"a": "boo",
 			}),
 		},
+		{
+			// Blank replacement should delete the label.
+			input: labels.FromMap(map[string]string{
+				"a": "foo",
+				"f": "baz",
+			}),
+			relabel: []*Config{
+				{
+					SourceLabels: model.LabelNames{"a"},
+					Regex:        MustNewRegexp("(f).*"),
+					TargetLabel:  "$1",
+					Replacement:  "$2",
+					Action:       Replace,
+				},
+			},
+			output: labels.FromMap(map[string]string{
+				"a": "foo",
+			}),
+		},
 		{
 			input: labels.FromMap(map[string]string{
 				"a": "foo",

From 8065bef172e8d88e22399504b175a8c9115e9da3 Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Wed, 22 Nov 2023 14:39:21 +0000
Subject: [PATCH 047/115] Move metric type definitions to common/model

They are used in multiple repos, so common is a better place for them.
Several packages now don't depend on `model/textparse`, e.g.
`storage/remote`.

Also remove `metadata` struct from `api.go`, since it was identical to
a struct in the `metadata` package.

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
---
 go.mod                                   |   2 +-
 go.sum                                   |   4 +-
 model/metadata/metadata.go               |   4 +-
 model/textparse/interface.go             |  16 +---
 model/textparse/openmetricsparse.go      |  20 +++--
 model/textparse/openmetricsparse_test.go |  25 +++---
 model/textparse/promparse.go             |  12 +--
 model/textparse/promparse_test.go        |   6 +-
 model/textparse/protobufparse.go         |  14 +--
 model/textparse/protobufparse_test.go    |  69 +++++++--------
 scrape/scrape.go                         |   6 +-
 scrape/scrape_test.go                    |   6 +-
 storage/remote/codec.go                  |   3 +-
 storage/remote/codec_test.go             |   8 +-
 storage/remote/metadata_watcher_test.go  |   7 +-
 storage/remote/queue_manager_test.go     |   3 +-
 tsdb/head_append.go                      |   2 +-
 tsdb/head_wal.go                         |   2 +-
 tsdb/record/record.go                    |  37 ++++----
 web/api/v1/api.go                        |  32 +++----
 web/api/v1/api_test.go                   | 104 +++++++++++------------
 21 files changed, 185 insertions(+), 197 deletions(-)

diff --git a/go.mod b/go.mod
index 9ea078083..0ecfd4374 100644
--- a/go.mod
+++ b/go.mod
@@ -48,7 +48,7 @@ require (
 	github.com/prometheus/alertmanager v0.26.0
 	github.com/prometheus/client_golang v1.17.0
 	github.com/prometheus/client_model v0.5.0
-	github.com/prometheus/common v0.45.0
+	github.com/prometheus/common v0.45.1-0.20231122191551-832cd6e99f99
 	github.com/prometheus/common/assets v0.2.0
 	github.com/prometheus/common/sigv4 v0.1.0
 	github.com/prometheus/exporter-toolkit v0.10.0
diff --git a/go.sum b/go.sum
index a6f535dd7..fb31c7a84 100644
--- a/go.sum
+++ b/go.sum
@@ -653,8 +653,8 @@ github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8b
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
 github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
-github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM=
-github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY=
+github.com/prometheus/common v0.45.1-0.20231122191551-832cd6e99f99 h1:V5ajRiLiCQGO+ggTr+07gMUcTqlIMMkDBfrJe5zKLmc=
+github.com/prometheus/common v0.45.1-0.20231122191551-832cd6e99f99/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY=
 github.com/prometheus/common/assets v0.2.0 h1:0P5OrzoHrYBOSM1OigWL3mY8ZvV2N4zIE/5AahrSrfM=
 github.com/prometheus/common/assets v0.2.0/go.mod h1:D17UVUE12bHbim7HzwUvtqm6gwBEaDQ0F+hIGbFbccI=
 github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4=
diff --git a/model/metadata/metadata.go b/model/metadata/metadata.go
index f227af0b9..f6f2827a4 100644
--- a/model/metadata/metadata.go
+++ b/model/metadata/metadata.go
@@ -13,11 +13,11 @@
 
 package metadata
 
-import "github.com/prometheus/prometheus/model/textparse"
+import "github.com/prometheus/common/model"
 
 // Metadata stores a series' metadata information.
 type Metadata struct {
-	Type textparse.MetricType
+	Type model.MetricType
 	Unit string
 	Help string
 }
diff --git a/model/textparse/interface.go b/model/textparse/interface.go
index df4259c85..f6d93f063 100644
--- a/model/textparse/interface.go
+++ b/model/textparse/interface.go
@@ -16,6 +16,8 @@ package textparse
 import (
 	"mime"
 
+	"github.com/prometheus/common/model"
+
 	"github.com/prometheus/prometheus/model/exemplar"
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
@@ -110,16 +112,4 @@ const (
 	EntryHistogram Entry = 5 // A series with a native histogram as a value.
 )
 
-// MetricType represents metric type values.
-type MetricType string
-
-const (
-	MetricTypeCounter        = MetricType("counter")
-	MetricTypeGauge          = MetricType("gauge")
-	MetricTypeHistogram      = MetricType("histogram")
-	MetricTypeGaugeHistogram = MetricType("gaugehistogram")
-	MetricTypeSummary        = MetricType("summary")
-	MetricTypeInfo           = MetricType("info")
-	MetricTypeStateset       = MetricType("stateset")
-	MetricTypeUnknown        = MetricType("unknown")
-)
+type MetricType = model.MetricType
diff --git a/model/textparse/openmetricsparse.go b/model/textparse/openmetricsparse.go
index f0c383723..d6b01eb8e 100644
--- a/model/textparse/openmetricsparse.go
+++ b/model/textparse/openmetricsparse.go
@@ -24,6 +24,8 @@ import (
 	"strings"
 	"unicode/utf8"
 
+	"github.com/prometheus/common/model"
+
 	"github.com/prometheus/prometheus/model/exemplar"
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
@@ -74,7 +76,7 @@ type OpenMetricsParser struct {
 	builder labels.ScratchBuilder
 	series  []byte
 	text    []byte
-	mtype   MetricType
+	mtype   model.MetricType
 	val     float64
 	ts      int64
 	hasTS   bool
@@ -272,21 +274,21 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
 		case tType:
 			switch s := yoloString(p.text); s {
 			case "counter":
-				p.mtype = MetricTypeCounter
+				p.mtype = model.MetricTypeCounter
 			case "gauge":
-				p.mtype = MetricTypeGauge
+				p.mtype = model.MetricTypeGauge
 			case "histogram":
-				p.mtype = MetricTypeHistogram
+				p.mtype = model.MetricTypeHistogram
 			case "gaugehistogram":
-				p.mtype = MetricTypeGaugeHistogram
+				p.mtype = model.MetricTypeGaugeHistogram
 			case "summary":
-				p.mtype = MetricTypeSummary
+				p.mtype = model.MetricTypeSummary
 			case "info":
-				p.mtype = MetricTypeInfo
+				p.mtype = model.MetricTypeInfo
 			case "stateset":
-				p.mtype = MetricTypeStateset
+				p.mtype = model.MetricTypeStateset
 			case "unknown":
-				p.mtype = MetricTypeUnknown
+				p.mtype = model.MetricTypeUnknown
 			default:
 				return EntryInvalid, fmt.Errorf("invalid metric type %q", s)
 			}
diff --git a/model/textparse/openmetricsparse_test.go b/model/textparse/openmetricsparse_test.go
index eed30364c..2b1d909f3 100644
--- a/model/textparse/openmetricsparse_test.go
+++ b/model/textparse/openmetricsparse_test.go
@@ -18,6 +18,7 @@ import (
 	"io"
 	"testing"
 
+	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 
 	"github.com/prometheus/prometheus/model/exemplar"
@@ -77,7 +78,7 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5`
 		m       string
 		t       *int64
 		v       float64
-		typ     MetricType
+		typ     model.MetricType
 		help    string
 		unit    string
 		comment string
@@ -88,7 +89,7 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5`
 			help: "A summary of the GC invocation durations.",
 		}, {
 			m:   "go_gc_duration_seconds",
-			typ: MetricTypeSummary,
+			typ: model.MetricTypeSummary,
 		}, {
 			m:    "go_gc_duration_seconds",
 			unit: "seconds",
@@ -130,7 +131,7 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5`
 			help: "Number of goroutines that currently exist.",
 		}, {
 			m:   "go_goroutines",
-			typ: MetricTypeGauge,
+			typ: model.MetricTypeGauge,
 		}, {
 			m:    `go_goroutines`,
 			v:    33,
@@ -138,21 +139,21 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5`
 			lset: labels.FromStrings("__name__", "go_goroutines"),
 		}, {
 			m:   "hh",
-			typ: MetricTypeHistogram,
+			typ: model.MetricTypeHistogram,
 		}, {
 			m:    `hh_bucket{le="+Inf"}`,
 			v:    1,
 			lset: labels.FromStrings("__name__", "hh_bucket", "le", "+Inf"),
 		}, {
 			m:   "gh",
-			typ: MetricTypeGaugeHistogram,
+			typ: model.MetricTypeGaugeHistogram,
 		}, {
 			m:    `gh_bucket{le="+Inf"}`,
 			v:    1,
 			lset: labels.FromStrings("__name__", "gh_bucket", "le", "+Inf"),
 		}, {
 			m:   "hhh",
-			typ: MetricTypeHistogram,
+			typ: model.MetricTypeHistogram,
 		}, {
 			m:    `hhh_bucket{le="+Inf"}`,
 			v:    1,
@@ -165,7 +166,7 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5`
 			e:    &exemplar.Exemplar{Labels: labels.FromStrings("id", "histogram-count-test"), Value: 4},
 		}, {
 			m:   "ggh",
-			typ: MetricTypeGaugeHistogram,
+			typ: model.MetricTypeGaugeHistogram,
 		}, {
 			m:    `ggh_bucket{le="+Inf"}`,
 			v:    1,
@@ -178,7 +179,7 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5`
 			e:    &exemplar.Exemplar{Labels: labels.FromStrings("id", "gaugehistogram-count-test", "xx", "yy"), Value: 4, HasTs: true, Ts: 123123},
 		}, {
 			m:   "smr_seconds",
-			typ: MetricTypeSummary,
+			typ: model.MetricTypeSummary,
 		}, {
 			m:    `smr_seconds_count`,
 			v:    2,
@@ -191,14 +192,14 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5`
 			e:    &exemplar.Exemplar{Labels: labels.FromStrings("id", "summary-sum-test"), Value: 1, HasTs: true, Ts: 123321},
 		}, {
 			m:   "ii",
-			typ: MetricTypeInfo,
+			typ: model.MetricTypeInfo,
 		}, {
 			m:    `ii{foo="bar"}`,
 			v:    1,
 			lset: labels.FromStrings("__name__", "ii", "foo", "bar"),
 		}, {
 			m:   "ss",
-			typ: MetricTypeStateset,
+			typ: model.MetricTypeStateset,
 		}, {
 			m:    `ss{ss="foo"}`,
 			v:    1,
@@ -213,7 +214,7 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5`
 			lset: labels.FromStrings("A", "a", "__name__", "ss"),
 		}, {
 			m:   "un",
-			typ: MetricTypeUnknown,
+			typ: model.MetricTypeUnknown,
 		}, {
 			m:    "_metric_starting_with_underscore",
 			v:    1,
@@ -228,7 +229,7 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5`
 			lset: labels.FromStrings("__name__", "testmetric", "label", `"bar"`),
 		}, {
 			m:   "foo",
-			typ: MetricTypeCounter,
+			typ: model.MetricTypeCounter,
 		}, {
 			m:    "foo_total",
 			v:    17,
diff --git a/model/textparse/promparse.go b/model/textparse/promparse.go
index 935801fb9..cd028ef90 100644
--- a/model/textparse/promparse.go
+++ b/model/textparse/promparse.go
@@ -26,6 +26,8 @@ import (
 	"unicode/utf8"
 	"unsafe"
 
+	"github.com/prometheus/common/model"
+
 	"github.com/prometheus/prometheus/model/exemplar"
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
@@ -305,15 +307,15 @@ func (p *PromParser) Next() (Entry, error) {
 		case tType:
 			switch s := yoloString(p.text); s {
 			case "counter":
-				p.mtype = MetricTypeCounter
+				p.mtype = model.MetricTypeCounter
 			case "gauge":
-				p.mtype = MetricTypeGauge
+				p.mtype = model.MetricTypeGauge
 			case "histogram":
-				p.mtype = MetricTypeHistogram
+				p.mtype = model.MetricTypeHistogram
 			case "summary":
-				p.mtype = MetricTypeSummary
+				p.mtype = model.MetricTypeSummary
 			case "untyped":
-				p.mtype = MetricTypeUnknown
+				p.mtype = model.MetricTypeUnknown
 			default:
 				return EntryInvalid, fmt.Errorf("invalid metric type %q", s)
 			}
diff --git a/model/textparse/promparse_test.go b/model/textparse/promparse_test.go
index ac79a1394..d34b26ba5 100644
--- a/model/textparse/promparse_test.go
+++ b/model/textparse/promparse_test.go
@@ -65,7 +65,7 @@ testmetric{label="\"bar\""} 1`
 		m       string
 		t       *int64
 		v       float64
-		typ     MetricType
+		typ     model.MetricType
 		help    string
 		comment string
 	}{
@@ -74,7 +74,7 @@ testmetric{label="\"bar\""} 1`
 			help: "A summary of the GC invocation durations.",
 		}, {
 			m:   "go_gc_duration_seconds",
-			typ: MetricTypeSummary,
+			typ: model.MetricTypeSummary,
 		}, {
 			m:    `go_gc_duration_seconds{quantile="0"}`,
 			v:    4.9351e-05,
@@ -142,7 +142,7 @@ testmetric{label="\"bar\""} 1`
 			help: "Number of goroutines that currently exist.",
 		}, {
 			m:   "go_goroutines",
-			typ: MetricTypeGauge,
+			typ: model.MetricTypeGauge,
 		}, {
 			m:    `go_goroutines`,
 			v:    33,
diff --git a/model/textparse/protobufparse.go b/model/textparse/protobufparse.go
index baede7e1d..534bbebb2 100644
--- a/model/textparse/protobufparse.go
+++ b/model/textparse/protobufparse.go
@@ -252,21 +252,21 @@ func (p *ProtobufParser) Help() ([]byte, []byte) {
 // Type returns the metric name and type in the current entry.
 // Must only be called after Next returned a type entry.
 // The returned byte slices become invalid after the next call to Next.
-func (p *ProtobufParser) Type() ([]byte, MetricType) {
+func (p *ProtobufParser) Type() ([]byte, model.MetricType) {
 	n := p.metricBytes.Bytes()
 	switch p.mf.GetType() {
 	case dto.MetricType_COUNTER:
-		return n, MetricTypeCounter
+		return n, model.MetricTypeCounter
 	case dto.MetricType_GAUGE:
-		return n, MetricTypeGauge
+		return n, model.MetricTypeGauge
 	case dto.MetricType_HISTOGRAM:
-		return n, MetricTypeHistogram
+		return n, model.MetricTypeHistogram
 	case dto.MetricType_GAUGE_HISTOGRAM:
-		return n, MetricTypeGaugeHistogram
+		return n, model.MetricTypeGaugeHistogram
 	case dto.MetricType_SUMMARY:
-		return n, MetricTypeSummary
+		return n, model.MetricTypeSummary
 	}
-	return n, MetricTypeUnknown
+	return n, model.MetricTypeUnknown
 }
 
 // Unit always returns (nil, nil) because units aren't supported by the protobuf
diff --git a/model/textparse/protobufparse_test.go b/model/textparse/protobufparse_test.go
index c5b672dbc..f994ff966 100644
--- a/model/textparse/protobufparse_test.go
+++ b/model/textparse/protobufparse_test.go
@@ -21,6 +21,7 @@ import (
 	"testing"
 
 	"github.com/gogo/protobuf/proto"
+	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 
 	"github.com/prometheus/prometheus/model/exemplar"
@@ -649,7 +650,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{
 					m:   "go_build_info",
-					typ: MetricTypeGauge,
+					typ: model.MetricTypeGauge,
 				},
 				{
 					m: "go_build_info\xFFchecksum\xFF\xFFpath\xFFgithub.com/prometheus/client_golang\xFFversion\xFF(devel)",
@@ -667,7 +668,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{
 					m:   "go_memstats_alloc_bytes_total",
-					typ: MetricTypeCounter,
+					typ: model.MetricTypeCounter,
 				},
 				{
 					m: "go_memstats_alloc_bytes_total",
@@ -685,7 +686,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{
 					m:   "something_untyped",
-					typ: MetricTypeUnknown,
+					typ: model.MetricTypeUnknown,
 				},
 				{
 					m: "something_untyped",
@@ -701,7 +702,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{
 					m:   "test_histogram",
-					typ: MetricTypeHistogram,
+					typ: model.MetricTypeHistogram,
 				},
 				{
 					m: "test_histogram",
@@ -736,7 +737,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{
 					m:   "test_gauge_histogram",
-					typ: MetricTypeGaugeHistogram,
+					typ: model.MetricTypeGaugeHistogram,
 				},
 				{
 					m: "test_gauge_histogram",
@@ -772,7 +773,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{
 					m:   "test_float_histogram",
-					typ: MetricTypeHistogram,
+					typ: model.MetricTypeHistogram,
 				},
 				{
 					m: "test_float_histogram",
@@ -807,7 +808,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{
 					m:   "test_gauge_float_histogram",
-					typ: MetricTypeGaugeHistogram,
+					typ: model.MetricTypeGaugeHistogram,
 				},
 				{
 					m: "test_gauge_float_histogram",
@@ -843,7 +844,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{
 					m:   "test_histogram2",
-					typ: MetricTypeHistogram,
+					typ: model.MetricTypeHistogram,
 				},
 				{
 					m: "test_histogram2_count",
@@ -903,7 +904,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{
 					m:   "test_histogram_family",
-					typ: MetricTypeHistogram,
+					typ: model.MetricTypeHistogram,
 				},
 				{
 					m: "test_histogram_family\xfffoo\xffbar",
@@ -947,7 +948,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{
 					m:   "test_float_histogram_with_zerothreshold_zero",
-					typ: MetricTypeHistogram,
+					typ: model.MetricTypeHistogram,
 				},
 				{
 					m: "test_float_histogram_with_zerothreshold_zero",
@@ -971,7 +972,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{
 					m:   "rpc_durations_seconds",
-					typ: MetricTypeSummary,
+					typ: model.MetricTypeSummary,
 				},
 				{
 					m: "rpc_durations_seconds_count\xffservice\xffexponential",
@@ -1022,7 +1023,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{
 					m:   "without_quantiles",
-					typ: MetricTypeSummary,
+					typ: model.MetricTypeSummary,
 				},
 				{
 					m: "without_quantiles_count",
@@ -1044,7 +1045,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{
 					m:   "empty_histogram",
-					typ: MetricTypeHistogram,
+					typ: model.MetricTypeHistogram,
 				},
 				{
 					m: "empty_histogram",
@@ -1063,7 +1064,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{
 					m:   "test_counter_with_createdtimestamp",
-					typ: MetricTypeCounter,
+					typ: model.MetricTypeCounter,
 				},
 				{
 					m:  "test_counter_with_createdtimestamp",
@@ -1079,7 +1080,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{
 					m:   "test_summary_with_createdtimestamp",
-					typ: MetricTypeSummary,
+					typ: model.MetricTypeSummary,
 				},
 				{
 					m:  "test_summary_with_createdtimestamp_count",
@@ -1103,7 +1104,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{
 					m:   "test_histogram_with_createdtimestamp",
-					typ: MetricTypeHistogram,
+					typ: model.MetricTypeHistogram,
 				},
 				{
 					m:  "test_histogram_with_createdtimestamp",
@@ -1123,7 +1124,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{
 					m:   "test_gaugehistogram_with_createdtimestamp",
-					typ: MetricTypeGaugeHistogram,
+					typ: model.MetricTypeGaugeHistogram,
 				},
 				{
 					m:  "test_gaugehistogram_with_createdtimestamp",
@@ -1149,7 +1150,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{ // 1
 					m:   "go_build_info",
-					typ: MetricTypeGauge,
+					typ: model.MetricTypeGauge,
 				},
 				{ // 2
 					m: "go_build_info\xFFchecksum\xFF\xFFpath\xFFgithub.com/prometheus/client_golang\xFFversion\xFF(devel)",
@@ -1167,7 +1168,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{ // 4
 					m:   "go_memstats_alloc_bytes_total",
-					typ: MetricTypeCounter,
+					typ: model.MetricTypeCounter,
 				},
 				{ // 5
 					m: "go_memstats_alloc_bytes_total",
@@ -1185,7 +1186,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{ // 7
 					m:   "something_untyped",
-					typ: MetricTypeUnknown,
+					typ: model.MetricTypeUnknown,
 				},
 				{ // 8
 					m: "something_untyped",
@@ -1201,7 +1202,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{ // 10
 					m:   "test_histogram",
-					typ: MetricTypeHistogram,
+					typ: model.MetricTypeHistogram,
 				},
 				{ // 11
 					m: "test_histogram",
@@ -1294,7 +1295,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{ // 19
 					m:   "test_gauge_histogram",
-					typ: MetricTypeGaugeHistogram,
+					typ: model.MetricTypeGaugeHistogram,
 				},
 				{ // 20
 					m: "test_gauge_histogram",
@@ -1388,7 +1389,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{ // 28
 					m:   "test_float_histogram",
-					typ: MetricTypeHistogram,
+					typ: model.MetricTypeHistogram,
 				},
 				{ // 29
 					m: "test_float_histogram",
@@ -1481,7 +1482,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{ // 37
 					m:   "test_gauge_float_histogram",
-					typ: MetricTypeGaugeHistogram,
+					typ: model.MetricTypeGaugeHistogram,
 				},
 				{ // 38
 					m: "test_gauge_float_histogram",
@@ -1575,7 +1576,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{ // 46
 					m:   "test_histogram2",
-					typ: MetricTypeHistogram,
+					typ: model.MetricTypeHistogram,
 				},
 				{ // 47
 					m: "test_histogram2_count",
@@ -1635,7 +1636,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{ // 54
 					m:   "test_histogram_family",
-					typ: MetricTypeHistogram,
+					typ: model.MetricTypeHistogram,
 				},
 				{ // 55
 					m: "test_histogram_family\xfffoo\xffbar",
@@ -1765,7 +1766,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{ // 68
 					m:   "test_float_histogram_with_zerothreshold_zero",
-					typ: MetricTypeHistogram,
+					typ: model.MetricTypeHistogram,
 				},
 				{ // 69
 					m: "test_float_histogram_with_zerothreshold_zero",
@@ -1789,7 +1790,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{ // 71
 					m:   "rpc_durations_seconds",
-					typ: MetricTypeSummary,
+					typ: model.MetricTypeSummary,
 				},
 				{ // 72
 					m: "rpc_durations_seconds_count\xffservice\xffexponential",
@@ -1840,7 +1841,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{ // 78
 					m:   "without_quantiles",
-					typ: MetricTypeSummary,
+					typ: model.MetricTypeSummary,
 				},
 				{ // 79
 					m: "without_quantiles_count",
@@ -1862,7 +1863,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{ // 79
 					m:   "empty_histogram",
-					typ: MetricTypeHistogram,
+					typ: model.MetricTypeHistogram,
 				},
 				{ // 80
 					m: "empty_histogram",
@@ -1881,7 +1882,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{ // 82
 					m:   "test_counter_with_createdtimestamp",
-					typ: MetricTypeCounter,
+					typ: model.MetricTypeCounter,
 				},
 				{ // 83
 					m:  "test_counter_with_createdtimestamp",
@@ -1897,7 +1898,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{ // 85
 					m:   "test_summary_with_createdtimestamp",
-					typ: MetricTypeSummary,
+					typ: model.MetricTypeSummary,
 				},
 				{ // 86
 					m:  "test_summary_with_createdtimestamp_count",
@@ -1921,7 +1922,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{ // 89
 					m:   "test_histogram_with_createdtimestamp",
-					typ: MetricTypeHistogram,
+					typ: model.MetricTypeHistogram,
 				},
 				{ // 90
 					m:  "test_histogram_with_createdtimestamp",
@@ -1941,7 +1942,7 @@ func TestProtobufParse(t *testing.T) {
 				},
 				{ // 92
 					m:   "test_gaugehistogram_with_createdtimestamp",
-					typ: MetricTypeGaugeHistogram,
+					typ: model.MetricTypeGaugeHistogram,
 				},
 				{ // 93
 					m:  "test_gaugehistogram_with_createdtimestamp",
diff --git a/scrape/scrape.go b/scrape/scrape.go
index be27a5d48..de74987cc 100644
--- a/scrape/scrape.go
+++ b/scrape/scrape.go
@@ -966,7 +966,7 @@ func (c *scrapeCache) setType(metric []byte, t textparse.MetricType) {
 
 	e, ok := c.metadata[string(metric)]
 	if !ok {
-		e = &metaEntry{Metadata: metadata.Metadata{Type: textparse.MetricTypeUnknown}}
+		e = &metaEntry{Metadata: metadata.Metadata{Type: model.MetricTypeUnknown}}
 		c.metadata[string(metric)] = e
 	}
 	if e.Type != t {
@@ -983,7 +983,7 @@ func (c *scrapeCache) setHelp(metric, help []byte) {
 
 	e, ok := c.metadata[string(metric)]
 	if !ok {
-		e = &metaEntry{Metadata: metadata.Metadata{Type: textparse.MetricTypeUnknown}}
+		e = &metaEntry{Metadata: metadata.Metadata{Type: model.MetricTypeUnknown}}
 		c.metadata[string(metric)] = e
 	}
 	if e.Help != string(help) {
@@ -1000,7 +1000,7 @@ func (c *scrapeCache) setUnit(metric, unit []byte) {
 
 	e, ok := c.metadata[string(metric)]
 	if !ok {
-		e = &metaEntry{Metadata: metadata.Metadata{Type: textparse.MetricTypeUnknown}}
+		e = &metaEntry{Metadata: metadata.Metadata{Type: model.MetricTypeUnknown}}
 		c.metadata[string(metric)] = e
 	}
 	if e.Unit != string(unit) {
diff --git a/scrape/scrape_test.go b/scrape/scrape_test.go
index 90578f2e9..1a416eeb6 100644
--- a/scrape/scrape_test.go
+++ b/scrape/scrape_test.go
@@ -971,19 +971,19 @@ test_metric 1
 
 	md, ok := cache.GetMetadata("test_metric")
 	require.True(t, ok, "expected metadata to be present")
-	require.Equal(t, textparse.MetricTypeCounter, md.Type, "unexpected metric type")
+	require.Equal(t, model.MetricTypeCounter, md.Type, "unexpected metric type")
 	require.Equal(t, "some help text", md.Help)
 	require.Equal(t, "metric", md.Unit)
 
 	md, ok = cache.GetMetadata("test_metric_no_help")
 	require.True(t, ok, "expected metadata to be present")
-	require.Equal(t, textparse.MetricTypeGauge, md.Type, "unexpected metric type")
+	require.Equal(t, model.MetricTypeGauge, md.Type, "unexpected metric type")
 	require.Equal(t, "", md.Help)
 	require.Equal(t, "", md.Unit)
 
 	md, ok = cache.GetMetadata("test_metric_no_type")
 	require.True(t, ok, "expected metadata to be present")
-	require.Equal(t, textparse.MetricTypeUnknown, md.Type, "unexpected metric type")
+	require.Equal(t, model.MetricTypeUnknown, md.Type, "unexpected metric type")
 	require.Equal(t, "other help text", md.Help)
 	require.Equal(t, "", md.Unit)
 }
diff --git a/storage/remote/codec.go b/storage/remote/codec.go
index 67035cd8e..ffab821a5 100644
--- a/storage/remote/codec.go
+++ b/storage/remote/codec.go
@@ -33,7 +33,6 @@ import (
 	"github.com/prometheus/prometheus/model/exemplar"
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
-	"github.com/prometheus/prometheus/model/textparse"
 	"github.com/prometheus/prometheus/prompb"
 	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
@@ -784,7 +783,7 @@ func labelsToLabelsProto(lbls labels.Labels, buf []prompb.Label) []prompb.Label
 }
 
 // metricTypeToMetricTypeProto transforms a Prometheus metricType into prompb metricType. Since the former is a string we need to transform it to an enum.
-func metricTypeToMetricTypeProto(t textparse.MetricType) prompb.MetricMetadata_MetricType {
+func metricTypeToMetricTypeProto(t model.MetricType) prompb.MetricMetadata_MetricType {
 	mt := strings.ToUpper(string(t))
 	v, ok := prompb.MetricMetadata_MetricType_value[mt]
 	if !ok {
diff --git a/storage/remote/codec_test.go b/storage/remote/codec_test.go
index d2a7d45be..ac8b0f0b5 100644
--- a/storage/remote/codec_test.go
+++ b/storage/remote/codec_test.go
@@ -20,11 +20,11 @@ import (
 	"testing"
 
 	"github.com/gogo/protobuf/proto"
+	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
-	"github.com/prometheus/prometheus/model/textparse"
 	"github.com/prometheus/prometheus/prompb"
 	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
@@ -488,17 +488,17 @@ func TestMergeLabels(t *testing.T) {
 func TestMetricTypeToMetricTypeProto(t *testing.T) {
 	tc := []struct {
 		desc     string
-		input    textparse.MetricType
+		input    model.MetricType
 		expected prompb.MetricMetadata_MetricType
 	}{
 		{
 			desc:     "with a single-word metric",
-			input:    textparse.MetricTypeCounter,
+			input:    model.MetricTypeCounter,
 			expected: prompb.MetricMetadata_COUNTER,
 		},
 		{
 			desc:     "with a two-word metric",
-			input:    textparse.MetricTypeStateset,
+			input:    model.MetricTypeStateset,
 			expected: prompb.MetricMetadata_STATESET,
 		},
 		{
diff --git a/storage/remote/metadata_watcher_test.go b/storage/remote/metadata_watcher_test.go
index cd664bc8b..0cd6027a8 100644
--- a/storage/remote/metadata_watcher_test.go
+++ b/storage/remote/metadata_watcher_test.go
@@ -22,7 +22,6 @@ import (
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 
-	"github.com/prometheus/prometheus/model/textparse"
 	"github.com/prometheus/prometheus/scrape"
 )
 
@@ -108,13 +107,13 @@ func TestWatchScrapeManager_ReadyForCollection(t *testing.T) {
 		Metadata: []scrape.MetricMetadata{
 			{
 				Metric: "prometheus_tsdb_head_chunks_created_total",
-				Type:   textparse.MetricTypeCounter,
+				Type:   model.MetricTypeCounter,
 				Help:   "Total number",
 				Unit:   "",
 			},
 			{
 				Metric: "prometheus_remote_storage_retried_samples_total",
-				Type:   textparse.MetricTypeCounter,
+				Type:   model.MetricTypeCounter,
 				Help:   "Total number",
 				Unit:   "",
 			},
@@ -124,7 +123,7 @@ func TestWatchScrapeManager_ReadyForCollection(t *testing.T) {
 		Metadata: []scrape.MetricMetadata{
 			{
 				Metric: "prometheus_tsdb_head_chunks_created_total",
-				Type:   textparse.MetricTypeCounter,
+				Type:   model.MetricTypeCounter,
 				Help:   "Total number",
 				Unit:   "",
 			},
diff --git a/storage/remote/queue_manager_test.go b/storage/remote/queue_manager_test.go
index c878c750b..17a904fcd 100644
--- a/storage/remote/queue_manager_test.go
+++ b/storage/remote/queue_manager_test.go
@@ -38,7 +38,6 @@ import (
 	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
-	"github.com/prometheus/prometheus/model/textparse"
 	"github.com/prometheus/prometheus/model/timestamp"
 	"github.com/prometheus/prometheus/prompb"
 	"github.com/prometheus/prometheus/scrape"
@@ -180,7 +179,7 @@ func TestMetadataDelivery(t *testing.T) {
 	for i := 0; i < numMetadata; i++ {
 		metadata = append(metadata, scrape.MetricMetadata{
 			Metric: "prometheus_remote_storage_sent_metadata_bytes_total_" + strconv.Itoa(i),
-			Type:   textparse.MetricTypeCounter,
+			Type:   model.MetricTypeCounter,
 			Help:   "a nice help text",
 			Unit:   "",
 		})
diff --git a/tsdb/head_append.go b/tsdb/head_append.go
index f509317c8..f112ffa3a 100644
--- a/tsdb/head_append.go
+++ b/tsdb/head_append.go
@@ -1038,7 +1038,7 @@ func (a *headAppender) Commit() (err error) {
 	for i, m := range a.metadata {
 		series = a.metadataSeries[i]
 		series.Lock()
-		series.meta = &metadata.Metadata{Type: record.ToTextparseMetricType(m.Type), Unit: m.Unit, Help: m.Help}
+		series.meta = &metadata.Metadata{Type: record.ToMetricType(m.Type), Unit: m.Unit, Help: m.Help}
 		series.Unlock()
 	}
 
diff --git a/tsdb/head_wal.go b/tsdb/head_wal.go
index a492a85a0..1be65f134 100644
--- a/tsdb/head_wal.go
+++ b/tsdb/head_wal.go
@@ -388,7 +388,7 @@ Outer:
 					continue
 				}
 				s.meta = &metadata.Metadata{
-					Type: record.ToTextparseMetricType(m.Type),
+					Type: record.ToMetricType(m.Type),
 					Unit: m.Unit,
 					Help: m.Help,
 				}
diff --git a/tsdb/record/record.go b/tsdb/record/record.go
index 42a656dfe..3931ad05d 100644
--- a/tsdb/record/record.go
+++ b/tsdb/record/record.go
@@ -20,9 +20,10 @@ import (
 	"fmt"
 	"math"
 
+	"github.com/prometheus/common/model"
+
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
-	"github.com/prometheus/prometheus/model/textparse"
 	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/tsdb/chunks"
 	"github.com/prometheus/prometheus/tsdb/encoding"
@@ -90,45 +91,45 @@ const (
 	Stateset        MetricType = 7
 )
 
-func GetMetricType(t textparse.MetricType) uint8 {
+func GetMetricType(t model.MetricType) uint8 {
 	switch t {
-	case textparse.MetricTypeCounter:
+	case model.MetricTypeCounter:
 		return uint8(Counter)
-	case textparse.MetricTypeGauge:
+	case model.MetricTypeGauge:
 		return uint8(Gauge)
-	case textparse.MetricTypeHistogram:
+	case model.MetricTypeHistogram:
 		return uint8(HistogramSample)
-	case textparse.MetricTypeGaugeHistogram:
+	case model.MetricTypeGaugeHistogram:
 		return uint8(GaugeHistogram)
-	case textparse.MetricTypeSummary:
+	case model.MetricTypeSummary:
 		return uint8(Summary)
-	case textparse.MetricTypeInfo:
+	case model.MetricTypeInfo:
 		return uint8(Info)
-	case textparse.MetricTypeStateset:
+	case model.MetricTypeStateset:
 		return uint8(Stateset)
 	default:
 		return uint8(UnknownMT)
 	}
 }
 
-func ToTextparseMetricType(m uint8) textparse.MetricType {
+func ToMetricType(m uint8) model.MetricType {
 	switch m {
 	case uint8(Counter):
-		return textparse.MetricTypeCounter
+		return model.MetricTypeCounter
 	case uint8(Gauge):
-		return textparse.MetricTypeGauge
+		return model.MetricTypeGauge
 	case uint8(HistogramSample):
-		return textparse.MetricTypeHistogram
+		return model.MetricTypeHistogram
 	case uint8(GaugeHistogram):
-		return textparse.MetricTypeGaugeHistogram
+		return model.MetricTypeGaugeHistogram
 	case uint8(Summary):
-		return textparse.MetricTypeSummary
+		return model.MetricTypeSummary
 	case uint8(Info):
-		return textparse.MetricTypeInfo
+		return model.MetricTypeInfo
 	case uint8(Stateset):
-		return textparse.MetricTypeStateset
+		return model.MetricTypeStateset
 	default:
-		return textparse.MetricTypeUnknown
+		return model.MetricTypeUnknown
 	}
 }
 
diff --git a/web/api/v1/api.go b/web/api/v1/api.go
index 8fa7ce14a..dd35d1fe9 100644
--- a/web/api/v1/api.go
+++ b/web/api/v1/api.go
@@ -41,7 +41,7 @@ import (
 
 	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/model/labels"
-	"github.com/prometheus/prometheus/model/textparse"
+	"github.com/prometheus/prometheus/model/metadata"
 	"github.com/prometheus/prometheus/model/timestamp"
 	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/promql/parser"
@@ -1141,11 +1141,11 @@ func (api *API) targetMetadata(r *http.Request) apiFuncResult {
 }
 
 type metricMetadata struct {
-	Target labels.Labels        `json:"target"`
-	Metric string               `json:"metric,omitempty"`
-	Type   textparse.MetricType `json:"type"`
-	Help   string               `json:"help"`
-	Unit   string               `json:"unit"`
+	Target labels.Labels    `json:"target"`
+	Metric string           `json:"metric,omitempty"`
+	Type   model.MetricType `json:"type"`
+	Help   string           `json:"help"`
+	Unit   string           `json:"unit"`
 }
 
 // AlertmanagerDiscovery has all the active Alertmanagers.
@@ -1221,14 +1221,8 @@ func rulesAlertsToAPIAlerts(rulesAlerts []*rules.Alert) []*Alert {
 	return apiAlerts
 }
 
-type metadata struct {
-	Type textparse.MetricType `json:"type"`
-	Help string               `json:"help"`
-	Unit string               `json:"unit"`
-}
-
 func (api *API) metricMetadata(r *http.Request) apiFuncResult {
-	metrics := map[string]map[metadata]struct{}{}
+	metrics := map[string]map[metadata.Metadata]struct{}{}
 
 	limit := -1
 	if s := r.FormValue("limit"); s != "" {
@@ -1250,7 +1244,7 @@ func (api *API) metricMetadata(r *http.Request) apiFuncResult {
 		for _, t := range tt {
 			if metric == "" {
 				for _, mm := range t.ListMetadata() {
-					m := metadata{Type: mm.Type, Help: mm.Help, Unit: mm.Unit}
+					m := metadata.Metadata{Type: mm.Type, Help: mm.Help, Unit: mm.Unit}
 					ms, ok := metrics[mm.Metric]
 
 					if limitPerMetric > 0 && len(ms) >= limitPerMetric {
@@ -1258,7 +1252,7 @@ func (api *API) metricMetadata(r *http.Request) apiFuncResult {
 					}
 
 					if !ok {
-						ms = map[metadata]struct{}{}
+						ms = map[metadata.Metadata]struct{}{}
 						metrics[mm.Metric] = ms
 					}
 					ms[m] = struct{}{}
@@ -1267,7 +1261,7 @@ func (api *API) metricMetadata(r *http.Request) apiFuncResult {
 			}
 
 			if md, ok := t.GetMetadata(metric); ok {
-				m := metadata{Type: md.Type, Help: md.Help, Unit: md.Unit}
+				m := metadata.Metadata{Type: md.Type, Help: md.Help, Unit: md.Unit}
 				ms, ok := metrics[md.Metric]
 
 				if limitPerMetric > 0 && len(ms) >= limitPerMetric {
@@ -1275,7 +1269,7 @@ func (api *API) metricMetadata(r *http.Request) apiFuncResult {
 				}
 
 				if !ok {
-					ms = map[metadata]struct{}{}
+					ms = map[metadata.Metadata]struct{}{}
 					metrics[md.Metric] = ms
 				}
 				ms[m] = struct{}{}
@@ -1284,13 +1278,13 @@ func (api *API) metricMetadata(r *http.Request) apiFuncResult {
 	}
 
 	// Put the elements from the pseudo-set into a slice for marshaling.
-	res := map[string][]metadata{}
+	res := map[string][]metadata.Metadata{}
 	for name, set := range metrics {
 		if limit >= 0 && len(res) >= limit {
 			break
 		}
 
-		s := []metadata{}
+		s := []metadata.Metadata{}
 		for metadata := range set {
 			s = append(s, metadata)
 		}
diff --git a/web/api/v1/api_test.go b/web/api/v1/api_test.go
index d4da05e46..c9ab84087 100644
--- a/web/api/v1/api_test.go
+++ b/web/api/v1/api_test.go
@@ -44,7 +44,7 @@ import (
 	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/model/exemplar"
 	"github.com/prometheus/prometheus/model/labels"
-	"github.com/prometheus/prometheus/model/textparse"
+	"github.com/prometheus/prometheus/model/metadata"
 	"github.com/prometheus/prometheus/model/timestamp"
 	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/promql/parser"
@@ -1584,7 +1584,7 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 					metadata: []scrape.MetricMetadata{
 						{
 							Metric: "go_threads",
-							Type:   textparse.MetricTypeGauge,
+							Type:   model.MetricTypeGauge,
 							Help:   "Number of OS threads created.",
 							Unit:   "",
 						},
@@ -1597,7 +1597,7 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 						"job": "test",
 					}),
 					Help: "Number of OS threads created.",
-					Type: textparse.MetricTypeGauge,
+					Type: model.MetricTypeGauge,
 					Unit: "",
 				},
 			},
@@ -1614,7 +1614,7 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 					metadata: []scrape.MetricMetadata{
 						{
 							Metric: "prometheus_tsdb_storage_blocks_bytes",
-							Type:   textparse.MetricTypeGauge,
+							Type:   model.MetricTypeGauge,
 							Help:   "The number of bytes that are currently used for local storage by all blocks.",
 							Unit:   "",
 						},
@@ -1628,7 +1628,7 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 					}),
 					Metric: "prometheus_tsdb_storage_blocks_bytes",
 					Help:   "The number of bytes that are currently used for local storage by all blocks.",
-					Type:   textparse.MetricTypeGauge,
+					Type:   model.MetricTypeGauge,
 					Unit:   "",
 				},
 			},
@@ -1642,7 +1642,7 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 					metadata: []scrape.MetricMetadata{
 						{
 							Metric: "go_threads",
-							Type:   textparse.MetricTypeGauge,
+							Type:   model.MetricTypeGauge,
 							Help:   "Number of OS threads created.",
 							Unit:   "",
 						},
@@ -1653,7 +1653,7 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 					metadata: []scrape.MetricMetadata{
 						{
 							Metric: "prometheus_tsdb_storage_blocks_bytes",
-							Type:   textparse.MetricTypeGauge,
+							Type:   model.MetricTypeGauge,
 							Help:   "The number of bytes that are currently used for local storage by all blocks.",
 							Unit:   "",
 						},
@@ -1667,7 +1667,7 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 					}),
 					Metric: "go_threads",
 					Help:   "Number of OS threads created.",
-					Type:   textparse.MetricTypeGauge,
+					Type:   model.MetricTypeGauge,
 					Unit:   "",
 				},
 				{
@@ -1676,7 +1676,7 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 					}),
 					Metric: "prometheus_tsdb_storage_blocks_bytes",
 					Help:   "The number of bytes that are currently used for local storage by all blocks.",
-					Type:   textparse.MetricTypeGauge,
+					Type:   model.MetricTypeGauge,
 					Unit:   "",
 				},
 			},
@@ -1719,22 +1719,22 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 					metadata: []scrape.MetricMetadata{
 						{
 							Metric: "prometheus_engine_query_duration_seconds",
-							Type:   textparse.MetricTypeSummary,
+							Type:   model.MetricTypeSummary,
 							Help:   "Query timings",
 							Unit:   "",
 						},
 						{
 							Metric: "go_info",
-							Type:   textparse.MetricTypeGauge,
+							Type:   model.MetricTypeGauge,
 							Help:   "Information about the Go environment.",
 							Unit:   "",
 						},
 					},
 				},
 			},
-			response: map[string][]metadata{
-				"prometheus_engine_query_duration_seconds": {{textparse.MetricTypeSummary, "Query timings", ""}},
-				"go_info": {{textparse.MetricTypeGauge, "Information about the Go environment.", ""}},
+			response: map[string][]metadata.Metadata{
+				"prometheus_engine_query_duration_seconds": {{Type: model.MetricTypeSummary, Help: "Query timings", Unit: ""}},
+				"go_info": {{Type: model.MetricTypeGauge, Help: "Information about the Go environment.", Unit: ""}},
 			},
 		},
 		// With duplicate metadata for a metric that comes from different targets.
@@ -1746,7 +1746,7 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 					metadata: []scrape.MetricMetadata{
 						{
 							Metric: "go_threads",
-							Type:   textparse.MetricTypeGauge,
+							Type:   model.MetricTypeGauge,
 							Help:   "Number of OS threads created",
 							Unit:   "",
 						},
@@ -1757,15 +1757,15 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 					metadata: []scrape.MetricMetadata{
 						{
 							Metric: "go_threads",
-							Type:   textparse.MetricTypeGauge,
+							Type:   model.MetricTypeGauge,
 							Help:   "Number of OS threads created",
 							Unit:   "",
 						},
 					},
 				},
 			},
-			response: map[string][]metadata{
-				"go_threads": {{textparse.MetricTypeGauge, "Number of OS threads created", ""}},
+			response: map[string][]metadata.Metadata{
+				"go_threads": {{Type: model.MetricTypeGauge, Help: "Number of OS threads created"}},
 			},
 		},
 		// With non-duplicate metadata for the same metric from different targets.
@@ -1777,7 +1777,7 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 					metadata: []scrape.MetricMetadata{
 						{
 							Metric: "go_threads",
-							Type:   textparse.MetricTypeGauge,
+							Type:   model.MetricTypeGauge,
 							Help:   "Number of OS threads created",
 							Unit:   "",
 						},
@@ -1788,21 +1788,21 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 					metadata: []scrape.MetricMetadata{
 						{
 							Metric: "go_threads",
-							Type:   textparse.MetricTypeGauge,
+							Type:   model.MetricTypeGauge,
 							Help:   "Number of OS threads that were created.",
 							Unit:   "",
 						},
 					},
 				},
 			},
-			response: map[string][]metadata{
+			response: map[string][]metadata.Metadata{
 				"go_threads": {
-					{textparse.MetricTypeGauge, "Number of OS threads created", ""},
-					{textparse.MetricTypeGauge, "Number of OS threads that were created.", ""},
+					{Type: model.MetricTypeGauge, Help: "Number of OS threads created"},
+					{Type: model.MetricTypeGauge, Help: "Number of OS threads that were created."},
 				},
 			},
 			sorter: func(m interface{}) {
-				v := m.(map[string][]metadata)["go_threads"]
+				v := m.(map[string][]metadata.Metadata)["go_threads"]
 
 				sort.Slice(v, func(i, j int) bool {
 					return v[i].Help < v[j].Help
@@ -1821,13 +1821,13 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 					metadata: []scrape.MetricMetadata{
 						{
 							Metric: "go_threads",
-							Type:   textparse.MetricTypeGauge,
+							Type:   model.MetricTypeGauge,
 							Help:   "Number of OS threads created",
 							Unit:   "",
 						},
 						{
 							Metric: "prometheus_engine_query_duration_seconds",
-							Type:   textparse.MetricTypeSummary,
+							Type:   model.MetricTypeSummary,
 							Help:   "Query Timmings.",
 							Unit:   "",
 						},
@@ -1838,7 +1838,7 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 					metadata: []scrape.MetricMetadata{
 						{
 							Metric: "go_gc_duration_seconds",
-							Type:   textparse.MetricTypeSummary,
+							Type:   model.MetricTypeSummary,
 							Help:   "A summary of the GC invocation durations.",
 							Unit:   "",
 						},
@@ -1857,31 +1857,31 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 					metadata: []scrape.MetricMetadata{
 						{
 							Metric: "go_threads",
-							Type:   textparse.MetricTypeGauge,
+							Type:   model.MetricTypeGauge,
 							Help:   "Number of OS threads created",
 							Unit:   "",
 						},
 						{
 							Metric: "go_threads",
-							Type:   textparse.MetricTypeGauge,
+							Type:   model.MetricTypeGauge,
 							Help:   "Repeated metadata",
 							Unit:   "",
 						},
 						{
 							Metric: "go_gc_duration_seconds",
-							Type:   textparse.MetricTypeSummary,
+							Type:   model.MetricTypeSummary,
 							Help:   "A summary of the GC invocation durations.",
 							Unit:   "",
 						},
 					},
 				},
 			},
-			response: map[string][]metadata{
+			response: map[string][]metadata.Metadata{
 				"go_threads": {
-					{textparse.MetricTypeGauge, "Number of OS threads created", ""},
+					{Type: model.MetricTypeGauge, Help: "Number of OS threads created"},
 				},
 				"go_gc_duration_seconds": {
-					{textparse.MetricTypeSummary, "A summary of the GC invocation durations.", ""},
+					{Type: model.MetricTypeSummary, Help: "A summary of the GC invocation durations."},
 				},
 			},
 		},
@@ -1895,19 +1895,19 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 					metadata: []scrape.MetricMetadata{
 						{
 							Metric: "go_threads",
-							Type:   textparse.MetricTypeGauge,
+							Type:   model.MetricTypeGauge,
 							Help:   "Number of OS threads created",
 							Unit:   "",
 						},
 						{
 							Metric: "go_threads",
-							Type:   textparse.MetricTypeGauge,
+							Type:   model.MetricTypeGauge,
 							Help:   "Repeated metadata",
 							Unit:   "",
 						},
 						{
 							Metric: "go_gc_duration_seconds",
-							Type:   textparse.MetricTypeSummary,
+							Type:   model.MetricTypeSummary,
 							Help:   "A summary of the GC invocation durations.",
 							Unit:   "",
 						},
@@ -1928,19 +1928,19 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 					metadata: []scrape.MetricMetadata{
 						{
 							Metric: "go_threads",
-							Type:   textparse.MetricTypeGauge,
+							Type:   model.MetricTypeGauge,
 							Help:   "Number of OS threads created",
 							Unit:   "",
 						},
 						{
 							Metric: "go_threads",
-							Type:   textparse.MetricTypeGauge,
+							Type:   model.MetricTypeGauge,
 							Help:   "Repeated metadata",
 							Unit:   "",
 						},
 						{
 							Metric: "go_gc_duration_seconds",
-							Type:   textparse.MetricTypeSummary,
+							Type:   model.MetricTypeSummary,
 							Help:   "A summary of the GC invocation durations.",
 							Unit:   "",
 						},
@@ -1951,13 +1951,13 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 					metadata: []scrape.MetricMetadata{
 						{
 							Metric: "go_threads",
-							Type:   textparse.MetricTypeGauge,
+							Type:   model.MetricTypeGauge,
 							Help:   "Number of OS threads created, but from a different target",
 							Unit:   "",
 						},
 						{
 							Metric: "go_gc_duration_seconds",
-							Type:   textparse.MetricTypeSummary,
+							Type:   model.MetricTypeSummary,
 							Help:   "A summary of the GC invocation durations, but from a different target.",
 							Unit:   "",
 						},
@@ -1977,7 +1977,7 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 					metadata: []scrape.MetricMetadata{
 						{
 							Metric: "go_threads",
-							Type:   textparse.MetricTypeGauge,
+							Type:   model.MetricTypeGauge,
 							Help:   "Number of OS threads created",
 							Unit:   "",
 						},
@@ -1988,27 +1988,27 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 					metadata: []scrape.MetricMetadata{
 						{
 							Metric: "go_gc_duration_seconds",
-							Type:   textparse.MetricTypeSummary,
+							Type:   model.MetricTypeSummary,
 							Help:   "A summary of the GC invocation durations.",
 							Unit:   "",
 						},
 						{
 							Metric: "go_threads",
-							Type:   textparse.MetricTypeGauge,
+							Type:   model.MetricTypeGauge,
 							Help:   "Number of OS threads that were created.",
 							Unit:   "",
 						},
 					},
 				},
 			},
-			response: map[string][]metadata{
+			response: map[string][]metadata.Metadata{
 				"go_threads": {
-					{textparse.MetricTypeGauge, "Number of OS threads created", ""},
-					{textparse.MetricTypeGauge, "Number of OS threads that were created.", ""},
+					{Type: model.MetricTypeGauge, Help: "Number of OS threads created"},
+					{Type: model.MetricTypeGauge, Help: "Number of OS threads that were created."},
 				},
 			},
 			sorter: func(m interface{}) {
-				v := m.(map[string][]metadata)["go_threads"]
+				v := m.(map[string][]metadata.Metadata)["go_threads"]
 
 				sort.Slice(v, func(i, j int) bool {
 					return v[i].Help < v[j].Help
@@ -2025,19 +2025,19 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 					metadata: []scrape.MetricMetadata{
 						{
 							Metric: "go_threads",
-							Type:   textparse.MetricTypeGauge,
+							Type:   model.MetricTypeGauge,
 							Help:   "Number of OS threads created",
 							Unit:   "",
 						},
 					},
 				},
 			},
-			response: map[string][]metadata{},
+			response: map[string][]metadata.Metadata{},
 		},
 		// With no available metadata.
 		{
 			endpoint: api.metricMetadata,
-			response: map[string][]metadata{},
+			response: map[string][]metadata.Metadata{},
 		},
 		{
 			endpoint: api.serveConfig,
@@ -2931,7 +2931,7 @@ func assertAPIResponseMetadataLen(t *testing.T, got interface{}, expLen int) {
 	t.Helper()
 
 	var gotLen int
-	response := got.(map[string][]metadata)
+	response := got.(map[string][]metadata.Metadata)
 	for _, m := range response {
 		gotLen += len(m)
 	}

From c83e1fc5748be3bd35bf0a31eb53690b412846a4 Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Tue, 12 Dec 2023 12:14:36 +0000
Subject: [PATCH 048/115] textparse: remove MetricType alias

No backwards-compatibility; make a clean break.

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
---
 model/textparse/interface.go          | 4 +---
 model/textparse/openmetricsparse.go   | 2 +-
 model/textparse/promparse.go          | 4 ++--
 model/textparse/protobufparse_test.go | 2 +-
 scrape/scrape.go                      | 2 +-
 scrape/target.go                      | 3 +--
 6 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/model/textparse/interface.go b/model/textparse/interface.go
index f6d93f063..3a363ebfb 100644
--- a/model/textparse/interface.go
+++ b/model/textparse/interface.go
@@ -44,7 +44,7 @@ type Parser interface {
 	// Type returns the metric name and type in the current entry.
 	// Must only be called after Next returned a type entry.
 	// The returned byte slices become invalid after the next call to Next.
-	Type() ([]byte, MetricType)
+	Type() ([]byte, model.MetricType)
 
 	// Unit returns the metric name and unit in the current entry.
 	// Must only be called after Next returned a unit entry.
@@ -111,5 +111,3 @@ const (
 	EntryUnit      Entry = 4
 	EntryHistogram Entry = 5 // A series with a native histogram as a value.
 )
-
-type MetricType = model.MetricType
diff --git a/model/textparse/openmetricsparse.go b/model/textparse/openmetricsparse.go
index d6b01eb8e..ddfbe4fc5 100644
--- a/model/textparse/openmetricsparse.go
+++ b/model/textparse/openmetricsparse.go
@@ -128,7 +128,7 @@ func (p *OpenMetricsParser) Help() ([]byte, []byte) {
 // Type returns the metric name and type in the current entry.
 // Must only be called after Next returned a type entry.
 // The returned byte slices become invalid after the next call to Next.
-func (p *OpenMetricsParser) Type() ([]byte, MetricType) {
+func (p *OpenMetricsParser) Type() ([]byte, model.MetricType) {
 	return p.l.b[p.offsets[0]:p.offsets[1]], p.mtype
 }
 
diff --git a/model/textparse/promparse.go b/model/textparse/promparse.go
index cd028ef90..7123e52c3 100644
--- a/model/textparse/promparse.go
+++ b/model/textparse/promparse.go
@@ -148,7 +148,7 @@ type PromParser struct {
 	builder labels.ScratchBuilder
 	series  []byte
 	text    []byte
-	mtype   MetricType
+	mtype   model.MetricType
 	val     float64
 	ts      int64
 	hasTS   bool
@@ -192,7 +192,7 @@ func (p *PromParser) Help() ([]byte, []byte) {
 // Type returns the metric name and type in the current entry.
 // Must only be called after Next returned a type entry.
 // The returned byte slices become invalid after the next call to Next.
-func (p *PromParser) Type() ([]byte, MetricType) {
+func (p *PromParser) Type() ([]byte, model.MetricType) {
 	return p.l.b[p.offsets[0]:p.offsets[1]], p.mtype
 }
 
diff --git a/model/textparse/protobufparse_test.go b/model/textparse/protobufparse_test.go
index f994ff966..7dcc85f54 100644
--- a/model/textparse/protobufparse_test.go
+++ b/model/textparse/protobufparse_test.go
@@ -623,7 +623,7 @@ func TestProtobufParse(t *testing.T) {
 		m       string
 		t       int64
 		v       float64
-		typ     MetricType
+		typ     model.MetricType
 		help    string
 		unit    string
 		comment string
diff --git a/scrape/scrape.go b/scrape/scrape.go
index de74987cc..dd425db90 100644
--- a/scrape/scrape.go
+++ b/scrape/scrape.go
@@ -961,7 +961,7 @@ func (c *scrapeCache) forEachStale(f func(labels.Labels) bool) {
 	}
 }
 
-func (c *scrapeCache) setType(metric []byte, t textparse.MetricType) {
+func (c *scrapeCache) setType(metric []byte, t model.MetricType) {
 	c.metaMtx.Lock()
 
 	e, ok := c.metadata[string(metric)]
diff --git a/scrape/target.go b/scrape/target.go
index fd984f5a6..0605f5349 100644
--- a/scrape/target.go
+++ b/scrape/target.go
@@ -30,7 +30,6 @@ import (
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/relabel"
-	"github.com/prometheus/prometheus/model/textparse"
 	"github.com/prometheus/prometheus/model/value"
 	"github.com/prometheus/prometheus/storage"
 )
@@ -87,7 +86,7 @@ type MetricMetadataStore interface {
 // MetricMetadata is a piece of metadata for a metric.
 type MetricMetadata struct {
 	Metric string
-	Type   textparse.MetricType
+	Type   model.MetricType
 	Help   string
 	Unit   string
 }

From 096ec129120b124ea14c78c8721c80dd0961b8ce Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Tue, 19 Dec 2023 18:53:09 +0000
Subject: [PATCH 049/115] Update comment about metadata in types.proto

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
---
 prompb/types.pb.go | 2 +-
 prompb/types.proto | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/prompb/types.pb.go b/prompb/types.pb.go
index 125f868e9..93883daa1 100644
--- a/prompb/types.pb.go
+++ b/prompb/types.pb.go
@@ -164,7 +164,7 @@ func (Chunk_Encoding) EnumDescriptor() ([]byte, []int) {
 
 type MetricMetadata struct {
 	// Represents the metric type, these match the set from Prometheus.
-	// Refer to model/textparse/interface.go for details.
+	// Refer to github.com/prometheus/common/model/metadata.go for details.
 	Type                 MetricMetadata_MetricType `protobuf:"varint,1,opt,name=type,proto3,enum=prometheus.MetricMetadata_MetricType" json:"type,omitempty"`
 	MetricFamilyName     string                    `protobuf:"bytes,2,opt,name=metric_family_name,json=metricFamilyName,proto3" json:"metric_family_name,omitempty"`
 	Help                 string                    `protobuf:"bytes,4,opt,name=help,proto3" json:"help,omitempty"`
diff --git a/prompb/types.proto b/prompb/types.proto
index aa322515c..61fc1e014 100644
--- a/prompb/types.proto
+++ b/prompb/types.proto
@@ -31,7 +31,7 @@ message MetricMetadata {
   }
 
   // Represents the metric type, these match the set from Prometheus.
-  // Refer to model/textparse/interface.go for details.
+  // Refer to github.com/prometheus/common/model/metadata.go for details.
   MetricType type = 1;
   string metric_family_name = 2;
   string help = 4;

From 51e78d9a32aefc80eccdd2360a6045cbd734bdd9 Mon Sep 17 00:00:00 2001
From: Arianna Vespri <arianna.vespri@yahoo.it>
Date: Wed, 20 Dec 2023 09:31:58 +0100
Subject: [PATCH 050/115] Delete debugging lines, amend error message for unit

Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
---
 model/textparse/protobufparse.go | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/model/textparse/protobufparse.go b/model/textparse/protobufparse.go
index cb622cf84..c40176bcc 100644
--- a/model/textparse/protobufparse.go
+++ b/model/textparse/protobufparse.go
@@ -424,15 +424,13 @@ func (p *ProtobufParser) Next() (Entry, error) {
 			return EntryInvalid, fmt.Errorf("unknown metric type for metric %q: %s", name, p.mf.GetType())
 		}
 		unit := p.mf.GetUnit()
-		fmt.Println("This is the unit:", unit)
-		fmt.Println("This is the name:", name)
 		if len(unit) > 0 {
 			sfx := fmt.Sprintf("_%s", unit)
 			if p.mf.GetType() == dto.MetricType_COUNTER {
 				sfx = fmt.Sprintf("_%s_total", unit)
 			}
 			if !strings.HasSuffix(name, sfx) {
-				return EntryInvalid, fmt.Errorf("invalid unit for metric %q: %s", name, unit)
+				return EntryInvalid, fmt.Errorf("unit %q not a suffix of metric %q", name, unit)
 			}
 		}
 		p.metricBytes.Reset()

From 9a664b515a2f86a313bd70af58de9210c7fcb04b Mon Sep 17 00:00:00 2001
From: Arianna Vespri <arianna.vespri@yahoo.it>
Date: Wed, 20 Dec 2023 09:41:37 +0100
Subject: [PATCH 051/115] Correct order in error message

Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
---
 model/textparse/protobufparse.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/model/textparse/protobufparse.go b/model/textparse/protobufparse.go
index c40176bcc..5ecdbf6ef 100644
--- a/model/textparse/protobufparse.go
+++ b/model/textparse/protobufparse.go
@@ -430,7 +430,7 @@ func (p *ProtobufParser) Next() (Entry, error) {
 				sfx = fmt.Sprintf("_%s_total", unit)
 			}
 			if !strings.HasSuffix(name, sfx) {
-				return EntryInvalid, fmt.Errorf("unit %q not a suffix of metric %q", name, unit)
+				return EntryInvalid, fmt.Errorf("unit %q not a suffix of metric %q", unit, name)
 			}
 		}
 		p.metricBytes.Reset()

From b012366c33cb673110421b9e20e35fde99c992db Mon Sep 17 00:00:00 2001
From: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
Date: Wed, 20 Dec 2023 00:28:59 +0530
Subject: [PATCH 052/115] Issue #13268: fix quality value in accept header

Signed-off-by: Kumar Kalpadiptya Roy <kalpadiptya.roy@outlook.com>
---
 scrape/scrape.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrape/scrape.go b/scrape/scrape.go
index be27a5d48..cb65053ba 100644
--- a/scrape/scrape.go
+++ b/scrape/scrape.go
@@ -675,7 +675,7 @@ func acceptHeader(sps []config.ScrapeProtocol) string {
 		weight--
 	}
 	// Default match anything.
-	vals = append(vals, fmt.Sprintf("*/*;q=%d", weight))
+	vals = append(vals, fmt.Sprintf("*/*;q=0.%d", weight))
 	return strings.Join(vals, ",")
 }
 

From 5df3820c7a5e099f0cdf8c503a01ac4b0498cf96 Mon Sep 17 00:00:00 2001
From: Filip Petkovski <filip.petkovsky@gmail.com>
Date: Mon, 25 Dec 2023 11:19:16 +0100
Subject: [PATCH 053/115] Copy last histogram point

Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
---
 promql/engine.go | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/promql/engine.go b/promql/engine.go
index 12755663d..5d6f7b9a0 100644
--- a/promql/engine.go
+++ b/promql/engine.go
@@ -2126,17 +2126,20 @@ loop:
 	// The sought sample might also be in the range.
 	switch soughtValueType {
 	case chunkenc.ValFloatHistogram, chunkenc.ValHistogram:
-		t, h := it.AtFloatHistogram()
-		if t == maxt && !value.IsStaleNaN(h.Sum) {
-			if ev.currentSamples >= ev.maxSamples {
-				ev.error(ErrTooManySamples(env))
+		t := it.AtT()
+		if t == maxt {
+			_, h := it.AtFloatHistogram()
+			if !value.IsStaleNaN(h.Sum) {
+				if ev.currentSamples >= ev.maxSamples {
+					ev.error(ErrTooManySamples(env))
+				}
+				if histograms == nil {
+					histograms = getHPointSlice(16)
+				}
+				point := HPoint{T: t, H: h.Copy()}
+				histograms = append(histograms, point)
+				ev.currentSamples += point.size()
 			}
-			if histograms == nil {
-				histograms = getHPointSlice(16)
-			}
-			point := HPoint{T: t, H: h}
-			histograms = append(histograms, point)
-			ev.currentSamples += point.size()
 		}
 	case chunkenc.ValFloat:
 		t, f := it.At()

From 35f9620cd1f212b37cc3074bbd59bdfae713f87d Mon Sep 17 00:00:00 2001
From: Filip Petkovski <filip.petkovsky@gmail.com>
Date: Mon, 25 Dec 2023 11:30:29 +0100
Subject: [PATCH 054/115] Expand benchmark

Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
---
 promql/bench_test.go | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/promql/bench_test.go b/promql/bench_test.go
index 13eba3714..b7a4978de 100644
--- a/promql/bench_test.go
+++ b/promql/bench_test.go
@@ -296,8 +296,12 @@ func BenchmarkNativeHistograms(b *testing.B) {
 			query: "sum(native_histogram_series)",
 		},
 		{
-			name:  "sum rate",
-			query: "sum(rate(native_histogram_series[1m]))",
+			name:  "sum rate with short rate interval",
+			query: "sum(rate(native_histogram_series[2m]))",
+		},
+		{
+			name:  "sum rate with long rate interval",
+			query: "sum(rate(native_histogram_series[20m]))",
 		},
 	}
 

From 0e1ae1d1caa65c398d0024a4ecb7977405e15a07 Mon Sep 17 00:00:00 2001
From: Filip Petkovski <filip.petkovsky@gmail.com>
Date: Mon, 25 Dec 2023 11:41:07 +0100
Subject: [PATCH 055/115] Add comment

Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
---
 promql/engine.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/promql/engine.go b/promql/engine.go
index 5d6f7b9a0..2ea37dae6 100644
--- a/promql/engine.go
+++ b/promql/engine.go
@@ -2136,6 +2136,8 @@ loop:
 				if histograms == nil {
 					histograms = getHPointSlice(16)
 				}
+				// The last sample comes directly from the iterator, so we need to copy it to
+				// avoid having the same reference twice in the buffer.
 				point := HPoint{T: t, H: h.Copy()}
 				histograms = append(histograms, point)
 				ev.currentSamples += point.size()

From b2185d96af9a45c7efa384b1f57138700c95bce5 Mon Sep 17 00:00:00 2001
From: Daniel Kerbel <nmdanny@gmail.com>
Date: Tue, 26 Dec 2023 18:44:39 +0200
Subject: [PATCH 056/115] Consider storage.ErrTooOldSample as non-retryable

Signed-off-by: Daniel Kerbel <nmdanny@gmail.com>
---
 storage/remote/write_handler.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/storage/remote/write_handler.go b/storage/remote/write_handler.go
index 9891c6aae..971f7b457 100644
--- a/storage/remote/write_handler.go
+++ b/storage/remote/write_handler.go
@@ -68,7 +68,7 @@ func (h *writeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	err = h.write(r.Context(), req)
 	switch {
 	case err == nil:
-	case errors.Is(err, storage.ErrOutOfOrderSample), errors.Is(err, storage.ErrOutOfBounds), errors.Is(err, storage.ErrDuplicateSampleForTimestamp):
+	case errors.Is(err, storage.ErrOutOfOrderSample), errors.Is(err, storage.ErrOutOfBounds), errors.Is(err, storage.ErrDuplicateSampleForTimestamp), errors.Is(err, storage.ErrTooOldSample):
 		// Indicated an out of order sample is a bad request to prevent retries.
 		http.Error(w, err.Error(), http.StatusBadRequest)
 		return

From 2f601772034511fec65af20cd7c676a19f18d7f9 Mon Sep 17 00:00:00 2001
From: machine424 <ayoubmrini424@gmail.com>
Date: Tue, 26 Dec 2023 22:06:03 +0100
Subject: [PATCH 057/115] scrape_test.go: Increase scrape interval in
 TestScrapeLoopCache to reduce potential flakiness

Signed-off-by: machine424 <ayoubmrini424@gmail.com>
---
 scrape/scrape_test.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scrape/scrape_test.go b/scrape/scrape_test.go
index 1a416eeb6..21a461a72 100644
--- a/scrape/scrape_test.go
+++ b/scrape/scrape_test.go
@@ -1194,7 +1194,9 @@ func TestScrapeLoopCache(t *testing.T) {
 	)
 
 	ctx, cancel := context.WithCancel(context.Background())
-	sl := newBasicScrapeLoop(t, ctx, scraper, app, 10*time.Millisecond)
+	// Decreasing the scrape interval could make the test fail, as multiple scrapes might be initiated at identical millisecond timestamps.
+	// See https://github.com/prometheus/prometheus/issues/12727.
+	sl := newBasicScrapeLoop(t, ctx, scraper, app, 100*time.Millisecond)
 
 	numScrapes := 0
 

From 8f07f9dd90bd2694acb1ca5476e9d596a1449fd9 Mon Sep 17 00:00:00 2001
From: Arianna Vespri <arianna.vespri@yahoo.it>
Date: Thu, 28 Dec 2023 15:41:38 +0100
Subject: [PATCH 058/115] Avoid creating string for suffix, consider counters
 without _total suffix

Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
---
 model/textparse/protobufparse.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/model/textparse/protobufparse.go b/model/textparse/protobufparse.go
index 5ecdbf6ef..9ccd00f6e 100644
--- a/model/textparse/protobufparse.go
+++ b/model/textparse/protobufparse.go
@@ -425,11 +425,11 @@ func (p *ProtobufParser) Next() (Entry, error) {
 		}
 		unit := p.mf.GetUnit()
 		if len(unit) > 0 {
-			sfx := fmt.Sprintf("_%s", unit)
-			if p.mf.GetType() == dto.MetricType_COUNTER {
-				sfx = fmt.Sprintf("_%s_total", unit)
-			}
-			if !strings.HasSuffix(name, sfx) {
+			if p.mf.GetType() == dto.MetricType_COUNTER && strings.HasSuffix(name, "_total") {
+				if !strings.HasSuffix(name[:len(name)-6], unit) || len(name)-6 < len(unit)+1 || name[len(name)-6-len(unit)-1] != '_' {
+					return EntryInvalid, fmt.Errorf("unit %q not a suffix of counter %q", unit, name)
+				}
+			} else if !strings.HasSuffix(name, unit) || len(name) < len(unit)+1 || name[len(name)-len(unit)-1] != '_' {
 				return EntryInvalid, fmt.Errorf("unit %q not a suffix of metric %q", unit, name)
 			}
 		}

From 2ddb3596ef4d2b1de482337ce1943476699b82fb Mon Sep 17 00:00:00 2001
From: Mile Druzijanic <153705375+zedGGs@users.noreply.github.com>
Date: Thu, 28 Dec 2023 21:49:57 +0100
Subject: [PATCH 059/115] Adding small test update for temp dir using t.TempDir
 (#13293)

* Adding small test update for temp dir using t.TempDir

Signed-off-by: Mile Druzijanic <miledruz@gmail.com>
Signed-off-by: Mile Druzijanic <zedsprogramms@gmail.com>

* removing not required cleanup

Signed-off-by: Mile Druzijanic <zedsprogramms@gmail.com>

---------

Signed-off-by: Mile Druzijanic <miledruz@gmail.com>
Signed-off-by: Mile Druzijanic <zedsprogramms@gmail.com>
---
 promql/engine_test.go                |  5 +----
 tsdb/tsdbutil/dir_locker_testutil.go | 10 +++-------
 2 files changed, 4 insertions(+), 11 deletions(-)

diff --git a/promql/engine_test.go b/promql/engine_test.go
index 105cdc10d..b3e7d4094 100644
--- a/promql/engine_test.go
+++ b/promql/engine_test.go
@@ -18,7 +18,6 @@ import (
 	"errors"
 	"fmt"
 	"math"
-	"os"
 	"sort"
 	"testing"
 	"time"
@@ -47,9 +46,7 @@ func TestMain(m *testing.M) {
 func TestQueryConcurrency(t *testing.T) {
 	maxConcurrency := 10
 
-	dir, err := os.MkdirTemp("", "test_concurrency")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
+	dir := t.TempDir()
 	queryTracker := NewActiveQueryTracker(dir, maxConcurrency, nil)
 	t.Cleanup(queryTracker.Close)
 
diff --git a/tsdb/tsdbutil/dir_locker_testutil.go b/tsdb/tsdbutil/dir_locker_testutil.go
index a4cf5abd6..a7f43d8ee 100644
--- a/tsdb/tsdbutil/dir_locker_testutil.go
+++ b/tsdb/tsdbutil/dir_locker_testutil.go
@@ -60,12 +60,8 @@ func TestDirLockerUsage(t *testing.T, open func(t *testing.T, data string, creat
 
 	for _, c := range cases {
 		t.Run(fmt.Sprintf("%+v", c), func(t *testing.T) {
-			tmpdir, err := os.MkdirTemp("", "test")
-			require.NoError(t, err)
-			t.Cleanup(func() {
-				require.NoError(t, os.RemoveAll(tmpdir))
-			})
-
+			tmpdir := t.TempDir()
+			
 			// Test preconditions (file already exists + lockfile option)
 			if c.fileAlreadyExists {
 				tmpLocker, err := NewDirLocker(tmpdir, "tsdb", log.NewNopLogger(), nil)
@@ -82,7 +78,7 @@ func TestDirLockerUsage(t *testing.T, open func(t *testing.T, data string, creat
 
 			// Check that the lockfile is always deleted
 			if !c.lockFileDisabled {
-				_, err = os.Stat(locker.path)
+				_, err := os.Stat(locker.path)
 				require.True(t, os.IsNotExist(err), "lockfile was not deleted")
 			}
 		})

From 252031c86f5c6fe7acad504bc6f3162175f347b2 Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Sat, 30 Dec 2023 19:13:59 +0000
Subject: [PATCH 060/115] Revert "Adding small test update for temp dir using
 t.TempDir (#13293)"

This reverts commit 2ddb3596ef4d2b1de482337ce1943476699b82fb.

Various tests are failing in CI after this change; reverting to free up
other work.

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
---
 promql/engine_test.go                |  5 ++++-
 tsdb/tsdbutil/dir_locker_testutil.go | 10 +++++++---
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/promql/engine_test.go b/promql/engine_test.go
index b3e7d4094..105cdc10d 100644
--- a/promql/engine_test.go
+++ b/promql/engine_test.go
@@ -18,6 +18,7 @@ import (
 	"errors"
 	"fmt"
 	"math"
+	"os"
 	"sort"
 	"testing"
 	"time"
@@ -46,7 +47,9 @@ func TestMain(m *testing.M) {
 func TestQueryConcurrency(t *testing.T) {
 	maxConcurrency := 10
 
-	dir := t.TempDir()
+	dir, err := os.MkdirTemp("", "test_concurrency")
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
 	queryTracker := NewActiveQueryTracker(dir, maxConcurrency, nil)
 	t.Cleanup(queryTracker.Close)
 
diff --git a/tsdb/tsdbutil/dir_locker_testutil.go b/tsdb/tsdbutil/dir_locker_testutil.go
index a7f43d8ee..a4cf5abd6 100644
--- a/tsdb/tsdbutil/dir_locker_testutil.go
+++ b/tsdb/tsdbutil/dir_locker_testutil.go
@@ -60,8 +60,12 @@ func TestDirLockerUsage(t *testing.T, open func(t *testing.T, data string, creat
 
 	for _, c := range cases {
 		t.Run(fmt.Sprintf("%+v", c), func(t *testing.T) {
-			tmpdir := t.TempDir()
-			
+			tmpdir, err := os.MkdirTemp("", "test")
+			require.NoError(t, err)
+			t.Cleanup(func() {
+				require.NoError(t, os.RemoveAll(tmpdir))
+			})
+
 			// Test preconditions (file already exists + lockfile option)
 			if c.fileAlreadyExists {
 				tmpLocker, err := NewDirLocker(tmpdir, "tsdb", log.NewNopLogger(), nil)
@@ -78,7 +82,7 @@ func TestDirLockerUsage(t *testing.T, open func(t *testing.T, data string, creat
 
 			// Check that the lockfile is always deleted
 			if !c.lockFileDisabled {
-				_, err := os.Stat(locker.path)
+				_, err = os.Stat(locker.path)
 				require.True(t, os.IsNotExist(err), "lockfile was not deleted")
 			}
 		})

From e64d7d8928323d3ec95c5bfccb66b30a6bd5393e Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Thu, 7 Dec 2023 15:27:59 +0000
Subject: [PATCH 061/115] agent: make the global hash lookup table smaller

This is the same change made in #13040, plus subsequent improvements,
applied to agent-mode code.

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
---
 tsdb/agent/series.go      | 130 +++++++++++++++++++++++++-------------
 tsdb/agent/series_test.go |  60 ++++++++++++++++++
 2 files changed, 146 insertions(+), 44 deletions(-)

diff --git a/tsdb/agent/series.go b/tsdb/agent/series.go
index d38518f71..76e734217 100644
--- a/tsdb/agent/series.go
+++ b/tsdb/agent/series.go
@@ -45,14 +45,23 @@ func (m *memSeries) updateTimestamp(newTs int64) bool {
 	return false
 }
 
-// seriesHashmap is a simple hashmap for memSeries by their label set.
-// It is built on top of a regular hashmap and holds a slice of series to
-// resolve hash collisions. Its methods require the hash to be submitted
+// seriesHashmap lets agent find a memSeries by its label set, via a 64-bit hash.
+// There is one map for the common case where the hash value is unique, and a
+// second map for the case that two series have the same hash value.
+// Each series is in only one of the maps. Its methods require the hash to be submitted
 // with the label set to avoid re-computing hash throughout the code.
-type seriesHashmap map[uint64][]*memSeries
+type seriesHashmap struct {
+	unique    map[uint64]*memSeries
+	conflicts map[uint64][]*memSeries
+}
 
-func (m seriesHashmap) Get(hash uint64, lset labels.Labels) *memSeries {
-	for _, s := range m[hash] {
+func (m *seriesHashmap) Get(hash uint64, lset labels.Labels) *memSeries {
+	if s, found := m.unique[hash]; found {
+		if labels.Equal(s.lset, lset) {
+			return s
+		}
+	}
+	for _, s := range m.conflicts[hash] {
 		if labels.Equal(s.lset, lset) {
 			return s
 		}
@@ -60,28 +69,49 @@ func (m seriesHashmap) Get(hash uint64, lset labels.Labels) *memSeries {
 	return nil
 }
 
-func (m seriesHashmap) Set(hash uint64, s *memSeries) {
-	seriesSet := m[hash]
+func (m *seriesHashmap) Set(hash uint64, s *memSeries) {
+	if existing, found := m.unique[hash]; !found || labels.Equal(existing.lset, s.lset) {
+		m.unique[hash] = s
+		return
+	}
+	if m.conflicts == nil {
+		m.conflicts = make(map[uint64][]*memSeries)
+	}
+	seriesSet := m.conflicts[hash]
 	for i, prev := range seriesSet {
 		if labels.Equal(prev.lset, s.lset) {
 			seriesSet[i] = s
 			return
 		}
 	}
-	m[hash] = append(seriesSet, s)
+	m.conflicts[hash] = append(seriesSet, s)
 }
 
-func (m seriesHashmap) Delete(hash uint64, ref chunks.HeadSeriesRef) {
+func (m *seriesHashmap) Delete(hash uint64, ref chunks.HeadSeriesRef) {
 	var rem []*memSeries
-	for _, s := range m[hash] {
-		if s.ref != ref {
-			rem = append(rem, s)
+	unique, found := m.unique[hash]
+	switch {
+	case !found: // Supplied hash is not stored.
+		return
+	case unique.ref == ref:
+		conflicts := m.conflicts[hash]
+		if len(conflicts) == 0 { // Exactly one series with this hash was stored
+			delete(m.unique, hash)
+			return
+		}
+		m.unique[hash] = conflicts[0] // First remaining series goes in 'unique'.
+		rem = conflicts[1:]           // Keep the rest.
+	default: // The series to delete is somewhere in 'conflicts'. Keep all the ones that don't match.
+		for _, s := range m.conflicts[hash] {
+			if s.ref != ref {
+				rem = append(rem, s)
+			}
 		}
 	}
 	if len(rem) == 0 {
-		delete(m, hash)
+		delete(m.conflicts, hash)
 	} else {
-		m[hash] = rem
+		m.conflicts[hash] = rem
 	}
 }
 
@@ -117,7 +147,10 @@ func newStripeSeries(stripeSize int) *stripeSeries {
 		s.series[i] = map[chunks.HeadSeriesRef]*memSeries{}
 	}
 	for i := range s.hashes {
-		s.hashes[i] = seriesHashmap{}
+		s.hashes[i] = seriesHashmap{
+			unique:    map[uint64]*memSeries{},
+			conflicts: nil, // Initialized on demand in set().
+		}
 	}
 	for i := range s.exemplars {
 		s.exemplars[i] = map[chunks.HeadSeriesRef]*exemplar.Exemplar{}
@@ -136,40 +169,49 @@ func (s *stripeSeries) GC(mint int64) map[chunks.HeadSeriesRef]struct{} {
 	defer s.gcMut.Unlock()
 
 	deleted := map[chunks.HeadSeriesRef]struct{}{}
+
+	// For one series, truncate old chunks and check if any chunks left. If not, mark as deleted and collect the ID.
+	check := func(hashLock int, hash uint64, series *memSeries) {
+		series.Lock()
+
+		// Any series that has received a write since mint is still alive.
+		if series.lastTs >= mint {
+			series.Unlock()
+			return
+		}
+
+		// The series is stale. We need to obtain a second lock for the
+		// ref if it's different than the hash lock.
+		refLock := int(series.ref) & (s.size - 1)
+		if hashLock != refLock {
+			s.locks[refLock].Lock()
+		}
+
+		deleted[series.ref] = struct{}{}
+		delete(s.series[refLock], series.ref)
+		s.hashes[hashLock].Delete(hash, series.ref)
+
+		// Since the series is gone, we'll also delete
+		// the latest stored exemplar.
+		delete(s.exemplars[refLock], series.ref)
+
+		if hashLock != refLock {
+			s.locks[refLock].Unlock()
+		}
+		series.Unlock()
+	}
+
 	for hashLock := 0; hashLock < s.size; hashLock++ {
 		s.locks[hashLock].Lock()
 
-		for hash, all := range s.hashes[hashLock] {
+		for hash, all := range s.hashes[hashLock].conflicts {
 			for _, series := range all {
-				series.Lock()
-
-				// Any series that has received a write since mint is still alive.
-				if series.lastTs >= mint {
-					series.Unlock()
-					continue
-				}
-
-				// The series is stale. We need to obtain a second lock for the
-				// ref if it's different than the hash lock.
-				refLock := int(series.ref) & (s.size - 1)
-				if hashLock != refLock {
-					s.locks[refLock].Lock()
-				}
-
-				deleted[series.ref] = struct{}{}
-				delete(s.series[refLock], series.ref)
-				s.hashes[hashLock].Delete(hash, series.ref)
-
-				// Since the series is gone, we'll also delete
-				// the latest stored exemplar.
-				delete(s.exemplars[refLock], series.ref)
-
-				if hashLock != refLock {
-					s.locks[refLock].Unlock()
-				}
-				series.Unlock()
+				check(hashLock, hash, series)
 			}
 		}
+		for hash, series := range s.hashes[hashLock].unique {
+			check(hashLock, hash, series)
+		}
 
 		s.locks[hashLock].Unlock()
 	}
diff --git a/tsdb/agent/series_test.go b/tsdb/agent/series_test.go
index 480f6c026..ae327d858 100644
--- a/tsdb/agent/series_test.go
+++ b/tsdb/agent/series_test.go
@@ -74,3 +74,63 @@ func TestNoDeadlock(t *testing.T) {
 		require.FailNow(t, "deadlock detected")
 	}
 }
+
+func labelsWithHashCollision() (labels.Labels, labels.Labels) {
+	// These two series have the same XXHash; thanks to https://github.com/pstibrany/labels_hash_collisions
+	ls1 := labels.FromStrings("__name__", "metric", "lbl1", "value", "lbl2", "l6CQ5y")
+	ls2 := labels.FromStrings("__name__", "metric", "lbl1", "value", "lbl2", "v7uDlF")
+
+	if ls1.Hash() != ls2.Hash() {
+		// These ones are the same when using -tags stringlabels
+		ls1 = labels.FromStrings("__name__", "metric", "lbl", "HFnEaGl")
+		ls2 = labels.FromStrings("__name__", "metric", "lbl", "RqcXatm")
+	}
+
+	if ls1.Hash() != ls2.Hash() {
+		panic("This code needs to be updated: find new labels with colliding hash values.")
+	}
+
+	return ls1, ls2
+}
+
+// stripeSeriesWithCollidingSeries returns a stripeSeries with two memSeries having the same, colliding, hash.
+func stripeSeriesWithCollidingSeries(_ *testing.T) (*stripeSeries, *memSeries, *memSeries) {
+	lbls1, lbls2 := labelsWithHashCollision()
+	ms1 := memSeries{
+		lset: lbls1,
+	}
+	ms2 := memSeries{
+		lset: lbls2,
+	}
+	hash := lbls1.Hash()
+	s := newStripeSeries(1)
+
+	s.Set(hash, &ms1)
+	s.Set(hash, &ms2)
+
+	return s, &ms1, &ms2
+}
+
+func TestStripeSeries_Get(t *testing.T) {
+	s, ms1, ms2 := stripeSeriesWithCollidingSeries(t)
+	hash := ms1.lset.Hash()
+
+	// Verify that we can get both of the series despite the hash collision
+	got := s.GetByHash(hash, ms1.lset)
+	require.Same(t, ms1, got)
+	got = s.GetByHash(hash, ms2.lset)
+	require.Same(t, ms2, got)
+}
+
+func TestStripeSeries_gc(t *testing.T) {
+	s, ms1, ms2 := stripeSeriesWithCollidingSeries(t)
+	hash := ms1.lset.Hash()
+
+	s.GC(1)
+
+	// Verify that we can get neither ms1 nor ms2 after gc-ing corresponding series
+	got := s.GetByHash(hash, ms1.lset)
+	require.Nil(t, got)
+	got = s.GetByHash(hash, ms2.lset)
+	require.Nil(t, got)
+}

From bad3f23f23817881d8d91f3705b8c7d31ccecfe9 Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Sat, 30 Dec 2023 14:07:09 +0000
Subject: [PATCH 062/115] agent: add BenchmarkCreateSeries

Based on the one in tsdb/head_test.go.

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
---
 tsdb/agent/db_test.go | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/tsdb/agent/db_test.go b/tsdb/agent/db_test.go
index 1e0976c3f..32b7e4394 100644
--- a/tsdb/agent/db_test.go
+++ b/tsdb/agent/db_test.go
@@ -84,7 +84,7 @@ func TestDB_InvalidSeries(t *testing.T) {
 	})
 }
 
-func createTestAgentDB(t *testing.T, reg prometheus.Registerer, opts *Options) *DB {
+func createTestAgentDB(t testing.TB, reg prometheus.Registerer, opts *Options) *DB {
 	t.Helper()
 
 	dbDir := t.TempDir()
@@ -878,3 +878,21 @@ func TestDBAllowOOOSamples(t *testing.T) {
 	require.Equal(t, float64(80), m.Metric[1].Counter.GetValue(), "agent wal mismatch of total appended histograms")
 	require.NoError(t, db.Close())
 }
+
+func BenchmarkCreateSeries(b *testing.B) {
+	s := createTestAgentDB(b, nil, DefaultOptions())
+	defer s.Close()
+
+	app := s.Appender(context.Background()).(*appender)
+	lbls := make([]labels.Labels, b.N)
+
+	for i, l := range labelsForTest("benchmark", b.N) {
+		lbls[i] = labels.New(l...)
+	}
+
+	b.ResetTimer()
+
+	for _, l := range lbls {
+		app.getOrCreate(l)
+	}
+}

From 153632063f4140851c4303a375a7ccda46f4b31c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 1 Jan 2024 23:13:46 +0000
Subject: [PATCH 063/115] build(deps): bump github.com/prometheus/client_golang

Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.17.0 to 1.18.0.
- [Release notes](https://github.com/prometheus/client_golang/releases)
- [Changelog](https://github.com/prometheus/client_golang/blob/main/CHANGELOG.md)
- [Commits](https://github.com/prometheus/client_golang/compare/v1.17.0...v1.18.0)

---
updated-dependencies:
- dependency-name: github.com/prometheus/client_golang
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 documentation/examples/remote_storage/go.mod |  6 +++---
 documentation/examples/remote_storage/go.sum | 12 ++++++------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/documentation/examples/remote_storage/go.mod b/documentation/examples/remote_storage/go.mod
index d0fec0f13..3a84bfa98 100644
--- a/documentation/examples/remote_storage/go.mod
+++ b/documentation/examples/remote_storage/go.mod
@@ -8,7 +8,7 @@ require (
 	github.com/gogo/protobuf v1.3.2
 	github.com/golang/snappy v0.0.4
 	github.com/influxdata/influxdb v1.11.2
-	github.com/prometheus/client_golang v1.17.0
+	github.com/prometheus/client_golang v1.18.0
 	github.com/prometheus/common v0.45.0
 	github.com/prometheus/prometheus v0.48.0
 	github.com/stretchr/testify v1.8.4
@@ -48,7 +48,7 @@ require (
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/common/sigv4 v0.1.0 // indirect
-	github.com/prometheus/procfs v0.11.1 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
 	go.opentelemetry.io/collector/pdata v1.0.0-rcv0016 // indirect
 	go.opentelemetry.io/collector/semconv v0.87.0 // indirect
@@ -62,7 +62,7 @@ require (
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/oauth2 v0.13.0 // indirect
-	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
diff --git a/documentation/examples/remote_storage/go.sum b/documentation/examples/remote_storage/go.sum
index a30c315b8..031c54ff0 100644
--- a/documentation/examples/remote_storage/go.sum
+++ b/documentation/examples/remote_storage/go.sum
@@ -194,8 +194,8 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
-github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
+github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=
+github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -213,8 +213,8 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
-github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
 github.com/prometheus/prometheus v0.48.0 h1:yrBloImGQ7je4h8M10ujGh4R6oxYQJQKlMuETwNskGk=
 github.com/prometheus/prometheus v0.48.0/go.mod h1:SRw624aMAxTfryAcP8rOjg4S/sHHaetx2lyJJ2nM83g=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
@@ -317,8 +317,8 @@ golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
-golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

From b90554f7a58cc2ace40dec3362f5c44145ff3231 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 1 Jan 2024 23:35:55 +0000
Subject: [PATCH 064/115] build(deps): bump actions/setup-node from 3.8.1 to
 4.0.1

Bumps [actions/setup-node](https://github.com/actions/setup-node) from 3.8.1 to 4.0.1.
- [Release notes](https://github.com/actions/setup-node/releases)
- [Commits](https://github.com/actions/setup-node/compare/5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d...b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8)

---
updated-dependencies:
- dependency-name: actions/setup-node
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8ba154e25..f1e2b66bf 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -197,7 +197,7 @@ jobs:
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - uses: prometheus/promci@3cb0c3871f223bd5ce1226995bd52ffb314798b6 # v0.1.0
       - name: Install nodejs
-        uses: actions/setup-node@5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d # v3.8.1
+        uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4.0.1
         with:
           node-version-file: "web/ui/.nvmrc"
           registry-url: "https://registry.npmjs.org"

From f29e0adb964cb9de93c56c2e2ebcd6c1fc00e12d Mon Sep 17 00:00:00 2001
From: Jan Fajerski <jfajersk@redhat.com>
Date: Tue, 2 Jan 2024 12:00:42 +0100
Subject: [PATCH 065/115] scripts: sort file list in embed directive

Otherwise the resulting string depends on find, which afaict depends on
the underlying filesystem. A stable file list make it easier to detect
UI changes in downstreams that need to track UI assets.

Signed-off-by: Jan Fajerski <jfajersk@redhat.com>
---
 scripts/compress_assets.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/compress_assets.sh b/scripts/compress_assets.sh
index 615e30c43..6608677bb 100755
--- a/scripts/compress_assets.sh
+++ b/scripts/compress_assets.sh
@@ -12,5 +12,5 @@ GZIP_OPTS="-fk"
 if ! gzip -k -h &>/dev/null; then GZIP_OPTS="-f"; fi
 
 find static -type f -name '*.gz' -delete
-find static -type f -exec gzip $GZIP_OPTS '{}' \; -print0 | xargs -0 -I % echo %.gz | xargs echo //go:embed >> embed.go
+find static -type f -exec gzip $GZIP_OPTS '{}' \; -print0 | xargs -0 -I % echo %.gz | sort | xargs echo //go:embed >> embed.go
 echo var EmbedFS embed.FS >> embed.go

From c126c21baa217c9cdce55819acb6d3483f9cd572 Mon Sep 17 00:00:00 2001
From: Kevin Mingtarja <kevin.mingtarja@gmail.com>
Date: Thu, 4 Jan 2024 01:51:56 +0700
Subject: [PATCH 066/115] Fix DataTableProps['data'] for resultType string

Signed-off-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
---
 web/ui/react-app/src/pages/graph/DataTable.test.tsx | 4 ++--
 web/ui/react-app/src/pages/graph/DataTable.tsx      | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/web/ui/react-app/src/pages/graph/DataTable.test.tsx b/web/ui/react-app/src/pages/graph/DataTable.test.tsx
index d6974cf90..1252b5bfc 100755
--- a/web/ui/react-app/src/pages/graph/DataTable.test.tsx
+++ b/web/ui/react-app/src/pages/graph/DataTable.test.tsx
@@ -338,7 +338,7 @@ describe('DataTable', () => {
     const dataTableProps: DataTableProps = {
       data: {
         resultType: 'string',
-        result: 'string',
+        result: [1572098246.599, 'test'],
       },
       useLocalTime: false,
     };
@@ -346,7 +346,7 @@ describe('DataTable', () => {
     it('renders a string row', () => {
       const table = dataTable.find(Table);
       const rows = table.find('tr');
-      expect(rows.text()).toEqual('stringt');
+      expect(rows.text()).toEqual('stringtest');
     });
   });
 });
diff --git a/web/ui/react-app/src/pages/graph/DataTable.tsx b/web/ui/react-app/src/pages/graph/DataTable.tsx
index 901d7bb48..1885c7a80 100644
--- a/web/ui/react-app/src/pages/graph/DataTable.tsx
+++ b/web/ui/react-app/src/pages/graph/DataTable.tsx
@@ -24,7 +24,7 @@ export interface DataTableProps {
       }
     | {
         resultType: 'string';
-        result: string;
+        result: SampleValue;
       };
   useLocalTime: boolean;
 }

From 40941d450bd85a0ec31011951b40afc0101b744f Mon Sep 17 00:00:00 2001
From: Kevin Mingtarja <kevin.mingtarja@gmail.com>
Date: Thu, 4 Jan 2024 02:48:54 +0700
Subject: [PATCH 067/115] Fix handling of scalar and string in isHeatmapData

Signed-off-by: Kevin Mingtarja <kevin.mingtarja@gmail.com>
---
 .../pages/graph/GraphHeatmapHelpers.test.ts   | 55 ++++++++++++++++---
 .../src/pages/graph/GraphHeatmapHelpers.ts    |  8 ++-
 2 files changed, 52 insertions(+), 11 deletions(-)

diff --git a/web/ui/react-app/src/pages/graph/GraphHeatmapHelpers.test.ts b/web/ui/react-app/src/pages/graph/GraphHeatmapHelpers.test.ts
index dfc718587..26c44be68 100644
--- a/web/ui/react-app/src/pages/graph/GraphHeatmapHelpers.test.ts
+++ b/web/ui/react-app/src/pages/graph/GraphHeatmapHelpers.test.ts
@@ -1,8 +1,23 @@
+import { DataTableProps } from './DataTable';
 import { isHeatmapData } from './GraphHeatmapHelpers';
 
 describe('GraphHeatmapHelpers', () => {
-  it('isHeatmapData should return false for a vector', () => {
-    const data = {
+  it('isHeatmapData should return false for scalar and string resultType', () => {
+    let data = {
+      resultType: 'scalar',
+      result: [1703091180.125, '1703091180.125'],
+    } as DataTableProps['data'];
+    expect(isHeatmapData(data)).toBe(false);
+
+    data = {
+      resultType: 'string',
+      result: [1704305680.332, '2504'],
+    } as DataTableProps['data'];
+    expect(isHeatmapData(data)).toBe(false);
+  });
+
+  it('isHeatmapData should return false for a vector and matrix if length < 2', () => {
+    let data = {
       resultType: 'vector',
       result: [
         {
@@ -13,15 +28,39 @@ describe('GraphHeatmapHelpers', () => {
           value: [1703091180.683, '6'],
         },
       ],
-    };
+    } as DataTableProps['data'];
+    expect(isHeatmapData(data)).toBe(false);
+
+    data = {
+      resultType: 'matrix',
+      result: [
+        {
+          metric: {},
+          values: [[1703091180.683, '6']],
+        },
+      ],
+    } as DataTableProps['data'];
     expect(isHeatmapData(data)).toBe(false);
   });
 
-  it('isHeatmapData should return false for scalar resultType', () => {
+  it('isHeatmapData should return true for valid heatmap data', () => {
     const data = {
-      resultType: 'scalar',
-      result: [1703091180.125, '1703091180.125'],
-    };
-    expect(isHeatmapData(data)).toBe(false);
+      resultType: 'matrix',
+      result: [
+        {
+          metric: {
+            le: '100',
+          },
+          values: [[1703091180.683, '6']],
+        },
+        {
+          metric: {
+            le: '1000',
+          },
+          values: [[1703091190.683, '6.1']],
+        },
+      ],
+    } as DataTableProps['data'];
+    expect(isHeatmapData(data)).toBe(true);
   });
 });
diff --git a/web/ui/react-app/src/pages/graph/GraphHeatmapHelpers.ts b/web/ui/react-app/src/pages/graph/GraphHeatmapHelpers.ts
index d564959cb..d8b249c30 100644
--- a/web/ui/react-app/src/pages/graph/GraphHeatmapHelpers.ts
+++ b/web/ui/react-app/src/pages/graph/GraphHeatmapHelpers.ts
@@ -1,10 +1,12 @@
+import { DataTableProps } from './DataTable';
 import { GraphProps, GraphSeries } from './Graph';
 
-export function isHeatmapData(data: GraphProps['data']) {
-  if (!data?.result?.length || data?.result?.length < 2) {
+export function isHeatmapData(data: DataTableProps['data']) {
+  if (data?.resultType === 'scalar' || data?.resultType === 'string' || !data?.result?.length || data?.result?.length < 2) {
     return false;
   }
-  const result = data.result;
+  // Type assertion to prevent TS2349 error.
+  const result = data.result as GraphProps['data']['result'];
   const firstLabels = Object.keys(result[0].metric).filter((label) => label !== 'le');
   return result.every(({ metric }) => {
     const labels = Object.keys(metric).filter((label) => label !== 'le');

From 377c75036c4f382ad5edc20008bccdc0a976ca4c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 4 Jan 2024 08:42:49 +0000
Subject: [PATCH 068/115] build(deps): bump github.com/influxdata/influxdb

Bumps [github.com/influxdata/influxdb](https://github.com/influxdata/influxdb) from 1.11.2 to 1.11.4.
- [Release notes](https://github.com/influxdata/influxdb/releases)
- [Commits](https://github.com/influxdata/influxdb/compare/v1.11.2...v1.11.4)

---
updated-dependencies:
- dependency-name: github.com/influxdata/influxdb
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 documentation/examples/remote_storage/go.mod | 2 +-
 documentation/examples/remote_storage/go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/documentation/examples/remote_storage/go.mod b/documentation/examples/remote_storage/go.mod
index 3a84bfa98..4c33a5087 100644
--- a/documentation/examples/remote_storage/go.mod
+++ b/documentation/examples/remote_storage/go.mod
@@ -7,7 +7,7 @@ require (
 	github.com/go-kit/log v0.2.1
 	github.com/gogo/protobuf v1.3.2
 	github.com/golang/snappy v0.0.4
-	github.com/influxdata/influxdb v1.11.2
+	github.com/influxdata/influxdb v1.11.4
 	github.com/prometheus/client_golang v1.18.0
 	github.com/prometheus/common v0.45.0
 	github.com/prometheus/prometheus v0.48.0
diff --git a/documentation/examples/remote_storage/go.sum b/documentation/examples/remote_storage/go.sum
index 031c54ff0..4b0526163 100644
--- a/documentation/examples/remote_storage/go.sum
+++ b/documentation/examples/remote_storage/go.sum
@@ -123,8 +123,8 @@ github.com/hashicorp/nomad/api v0.0.0-20230721134942-515895c7690c h1:Nc3Mt2BAnq0
 github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY=
 github.com/hetznercloud/hcloud-go/v2 v2.4.0 h1:MqlAE+w125PLvJRCpAJmEwrIxoVdUdOyuFUhE/Ukbok=
 github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
-github.com/influxdata/influxdb v1.11.2 h1:qOF3uQN1mDfJNEKwbAgJsqehf8IXgKok2vlGm736oGo=
-github.com/influxdata/influxdb v1.11.2/go.mod h1:eUMkLTE2vQwvSk6KGMrTBLKPaqSuczuelGbggigMPFw=
+github.com/influxdata/influxdb v1.11.4 h1:H3pVW+/tWQ4lkHhZxVQ13Ov1hmhHYaAzz8L5aq3ZNtw=
+github.com/influxdata/influxdb v1.11.4/go.mod h1:VO6X2zlamfmEf+Esc9dR+7UQhdE/krspWNEZPwxCrp0=
 github.com/ionos-cloud/sdk-go/v6 v6.1.9 h1:Iq3VIXzeEbc8EbButuACgfLMiY5TPVWUPNrF+Vsddo4=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=

From cef6eba91431f123f5a2eef997f0cb49f91fa89c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 4 Jan 2024 08:43:28 +0000
Subject: [PATCH 069/115] build(deps): bump github.com/prometheus/prometheus

Bumps [github.com/prometheus/prometheus](https://github.com/prometheus/prometheus) from 0.48.0 to 0.48.1.
- [Release notes](https://github.com/prometheus/prometheus/releases)
- [Changelog](https://github.com/prometheus/prometheus/blob/main/CHANGELOG.md)
- [Commits](https://github.com/prometheus/prometheus/compare/v0.48.0...v0.48.1)

---
updated-dependencies:
- dependency-name: github.com/prometheus/prometheus
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 documentation/examples/remote_storage/go.mod | 2 +-
 documentation/examples/remote_storage/go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/documentation/examples/remote_storage/go.mod b/documentation/examples/remote_storage/go.mod
index 3a84bfa98..dcc643749 100644
--- a/documentation/examples/remote_storage/go.mod
+++ b/documentation/examples/remote_storage/go.mod
@@ -10,7 +10,7 @@ require (
 	github.com/influxdata/influxdb v1.11.2
 	github.com/prometheus/client_golang v1.18.0
 	github.com/prometheus/common v0.45.0
-	github.com/prometheus/prometheus v0.48.0
+	github.com/prometheus/prometheus v0.48.1
 	github.com/stretchr/testify v1.8.4
 )
 
diff --git a/documentation/examples/remote_storage/go.sum b/documentation/examples/remote_storage/go.sum
index 031c54ff0..c89eff124 100644
--- a/documentation/examples/remote_storage/go.sum
+++ b/documentation/examples/remote_storage/go.sum
@@ -215,8 +215,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
 github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
-github.com/prometheus/prometheus v0.48.0 h1:yrBloImGQ7je4h8M10ujGh4R6oxYQJQKlMuETwNskGk=
-github.com/prometheus/prometheus v0.48.0/go.mod h1:SRw624aMAxTfryAcP8rOjg4S/sHHaetx2lyJJ2nM83g=
+github.com/prometheus/prometheus v0.48.1 h1:CTszphSNTXkuCG6O0IfpKdHcJkvvnAAE1GbELKS+NFk=
+github.com/prometheus/prometheus v0.48.1/go.mod h1:SRw624aMAxTfryAcP8rOjg4S/sHHaetx2lyJJ2nM83g=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21 h1:yWfiTPwYxB0l5fGMhl/G+liULugVIHD9AU77iNLrURQ=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=

From bb9503c12c9df66403867d454aeecf0917142a0a Mon Sep 17 00:00:00 2001
From: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Date: Thu, 4 Jan 2024 15:19:19 +0200
Subject: [PATCH 070/115] Bump client_golang to v1.18.0 (#13373)

Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
---
 go.mod | 4 ++--
 go.sum | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/go.mod b/go.mod
index 0ecfd4374..428b6edec 100644
--- a/go.mod
+++ b/go.mod
@@ -46,7 +46,7 @@ require (
 	github.com/oklog/ulid v1.3.1
 	github.com/ovh/go-ovh v1.4.3
 	github.com/prometheus/alertmanager v0.26.0
-	github.com/prometheus/client_golang v1.17.0
+	github.com/prometheus/client_golang v1.18.0
 	github.com/prometheus/client_model v0.5.0
 	github.com/prometheus/common v0.45.1-0.20231122191551-832cd6e99f99
 	github.com/prometheus/common/assets v0.2.0
@@ -168,7 +168,7 @@ require (
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/procfs v0.11.1 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/objx v0.5.0 // indirect
 	github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
diff --git a/go.sum b/go.sum
index fb31c7a84..8ce4efb5c 100644
--- a/go.sum
+++ b/go.sum
@@ -636,8 +636,8 @@ github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeD
 github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
-github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
+github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=
+github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -667,8 +667,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
-github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=

From 78c5ce3196e729ded7dbe13cc5c92eb4196c2d19 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc=20Tudur=C3=AD?= <marctc@protonmail.com>
Date: Fri, 5 Jan 2024 19:40:30 +0100
Subject: [PATCH 071/115] Drop old inmemory samples (#13002)

* Drop old inmemory samples

Co-authored-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* Avoid copying timeseries when the feature is disabled

Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* Run gofmt

Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* Clarify docs

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* Add more logging info

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* Remove loggers

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* optimize function and add tests

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* Simplify filter

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* rename var

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* Update help info from metrics

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* use metrics to keep track of drop elements during buildWriteRequest

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* rename var in tests

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* pass time.Now as parameter

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* Change buildwriterequest during retries

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* Revert "Remove loggers"

This reverts commit 54f91dfcae20488944162335ab4ad8be459df1ab.

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* use log level debug for loggers

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* Fix linter

Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>

* Remove noisy debug-level logs; add 'reason' label to drop metrics

Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>

* Remove accidentally committed files

Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>

* Propagate logger to buildWriteRequest to log dropped data

Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>

* Fix docs comment

Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>

* Make drop reason more specific

Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>

* Remove unnecessary pass of logger

Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>

* Use snake_case for reason label

Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>

* Fix dropped samples metric

Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>

---------

Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Signed-off-by: Marc Tuduri <marctc@protonmail.com>
Signed-off-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
Co-authored-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Co-authored-by: Paschalis Tsilias <tpaschalis@users.noreply.github.com>
---
 config/config.go                     |   3 +
 docs/configuration/configuration.md  |   4 +
 storage/remote/codec_test.go         |   2 +-
 storage/remote/queue_manager.go      | 190 ++++++++++++++++---
 storage/remote/queue_manager_test.go | 272 ++++++++++++++++++++++++++-
 storage/remote/write_handler_test.go |  24 +--
 6 files changed, 449 insertions(+), 46 deletions(-)

diff --git a/config/config.go b/config/config.go
index ddcca84dc..c544d7e74 100644
--- a/config/config.go
+++ b/config/config.go
@@ -1124,6 +1124,9 @@ type QueueConfig struct {
 	MinBackoff       model.Duration `yaml:"min_backoff,omitempty"`
 	MaxBackoff       model.Duration `yaml:"max_backoff,omitempty"`
 	RetryOnRateLimit bool           `yaml:"retry_on_http_429,omitempty"`
+
+	// Samples older than the limit will be dropped.
+	SampleAgeLimit model.Duration `yaml:"sample_age_limit,omitempty"`
 }
 
 // MetadataConfig is the configuration for sending metadata to remote
diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md
index 5e2f31c1c..3a7b71a2f 100644
--- a/docs/configuration/configuration.md
+++ b/docs/configuration/configuration.md
@@ -3619,6 +3619,10 @@ queue_config:
   # Retry upon receiving a 429 status code from the remote-write storage.
   # This is experimental and might change in the future.
   [ retry_on_http_429: <boolean> | default = false ]
+  # If set, any sample that is older than sample_age_limit
+  # will not be sent to the remote storage. The default value is 0s,
+  # which means that all samples are sent.
+  [ sample_age_limit: <duration> | default = 0s ]
 
 # Configures the sending of series metadata to remote storage.
 # Metadata configuration is subject to change at any point
diff --git a/storage/remote/codec_test.go b/storage/remote/codec_test.go
index ac8b0f0b5..dbc22a377 100644
--- a/storage/remote/codec_test.go
+++ b/storage/remote/codec_test.go
@@ -517,7 +517,7 @@ func TestMetricTypeToMetricTypeProto(t *testing.T) {
 }
 
 func TestDecodeWriteRequest(t *testing.T) {
-	buf, _, err := buildWriteRequest(writeRequestFixture.Timeseries, nil, nil, nil)
+	buf, _, _, err := buildWriteRequest(nil, writeRequestFixture.Timeseries, nil, nil, nil, nil)
 	require.NoError(t, err)
 
 	actual, err := DecodeWriteRequest(bytes.NewReader(buf))
diff --git a/storage/remote/queue_manager.go b/storage/remote/queue_manager.go
index a25c7d90c..e37ec8c70 100644
--- a/storage/remote/queue_manager.go
+++ b/storage/remote/queue_manager.go
@@ -36,6 +36,7 @@ import (
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/relabel"
+	"github.com/prometheus/prometheus/model/timestamp"
 	"github.com/prometheus/prometheus/prompb"
 	"github.com/prometheus/prometheus/scrape"
 	"github.com/prometheus/prometheus/tsdb/chunks"
@@ -51,6 +52,10 @@ const (
 
 	// Allow 30% too many shards before scaling down.
 	shardToleranceFraction = 0.3
+
+	reasonTooOld                     = "too_old"
+	reasonDroppedSeries              = "dropped_series"
+	reasonUnintentionalDroppedSeries = "unintentionally_dropped_series"
 )
 
 type queueManagerMetrics struct {
@@ -68,9 +73,9 @@ type queueManagerMetrics struct {
 	retriedExemplarsTotal  prometheus.Counter
 	retriedHistogramsTotal prometheus.Counter
 	retriedMetadataTotal   prometheus.Counter
-	droppedSamplesTotal    prometheus.Counter
-	droppedExemplarsTotal  prometheus.Counter
-	droppedHistogramsTotal prometheus.Counter
+	droppedSamplesTotal    *prometheus.CounterVec
+	droppedExemplarsTotal  *prometheus.CounterVec
+	droppedHistogramsTotal *prometheus.CounterVec
 	enqueueRetriesTotal    prometheus.Counter
 	sentBatchDuration      prometheus.Histogram
 	highestSentTimestamp   *maxTimestamp
@@ -180,27 +185,27 @@ func newQueueManagerMetrics(r prometheus.Registerer, rn, e string) *queueManager
 		Help:        "Total number of metadata entries which failed on send to remote storage but were retried because the send error was recoverable.",
 		ConstLabels: constLabels,
 	})
-	m.droppedSamplesTotal = prometheus.NewCounter(prometheus.CounterOpts{
+	m.droppedSamplesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace:   namespace,
 		Subsystem:   subsystem,
 		Name:        "samples_dropped_total",
-		Help:        "Total number of samples which were dropped after being read from the WAL before being sent via remote write, either via relabelling or unintentionally because of an unknown reference ID.",
+		Help:        "Total number of samples which were dropped after being read from the WAL before being sent via remote write, either via relabelling, due to being too old or unintentionally because of an unknown reference ID.",
 		ConstLabels: constLabels,
-	})
-	m.droppedExemplarsTotal = prometheus.NewCounter(prometheus.CounterOpts{
+	}, []string{"reason"})
+	m.droppedExemplarsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace:   namespace,
 		Subsystem:   subsystem,
 		Name:        "exemplars_dropped_total",
-		Help:        "Total number of exemplars which were dropped after being read from the WAL before being sent via remote write, either via relabelling or unintentionally because of an unknown reference ID.",
+		Help:        "Total number of exemplars which were dropped after being read from the WAL before being sent via remote write, either via relabelling, due to being too old or unintentionally because of an unknown reference ID.",
 		ConstLabels: constLabels,
-	})
-	m.droppedHistogramsTotal = prometheus.NewCounter(prometheus.CounterOpts{
+	}, []string{"reason"})
+	m.droppedHistogramsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace:   namespace,
 		Subsystem:   subsystem,
 		Name:        "histograms_dropped_total",
-		Help:        "Total number of histograms which were dropped after being read from the WAL before being sent via remote write, either via relabelling or unintentionally because of an unknown reference ID.",
+		Help:        "Total number of histograms which were dropped after being read from the WAL before being sent via remote write, either via relabelling, due to being too old or unintentionally because of an unknown reference ID.",
 		ConstLabels: constLabels,
-	})
+	}, []string{"reason"})
 	m.enqueueRetriesTotal = prometheus.NewCounter(prometheus.CounterOpts{
 		Namespace:   namespace,
 		Subsystem:   subsystem,
@@ -391,7 +396,8 @@ type WriteClient interface {
 // indicated by the provided WriteClient. Implements writeTo interface
 // used by WAL Watcher.
 type QueueManager struct {
-	lastSendTimestamp atomic.Int64
+	lastSendTimestamp          atomic.Int64
+	buildRequestLimitTimestamp atomic.Int64
 
 	logger               log.Logger
 	flushDeadline        time.Duration
@@ -529,7 +535,7 @@ func (t *QueueManager) AppendMetadata(ctx context.Context, metadata []scrape.Met
 
 func (t *QueueManager) sendMetadataWithBackoff(ctx context.Context, metadata []prompb.MetricMetadata, pBuf *proto.Buffer) error {
 	// Build the WriteRequest with no samples.
-	req, _, err := buildWriteRequest(nil, metadata, pBuf, nil)
+	req, _, _, err := buildWriteRequest(t.logger, nil, metadata, pBuf, nil, nil)
 	if err != nil {
 		return err
 	}
@@ -575,18 +581,65 @@ func (t *QueueManager) sendMetadataWithBackoff(ctx context.Context, metadata []p
 	return nil
 }
 
+func isSampleOld(baseTime time.Time, sampleAgeLimit time.Duration, ts int64) bool {
+	if sampleAgeLimit == 0 {
+		// If sampleAgeLimit is unset, then we never skip samples due to their age.
+		return false
+	}
+	limitTs := baseTime.Add(-sampleAgeLimit)
+	sampleTs := timestamp.Time(ts)
+	return sampleTs.Before(limitTs)
+}
+
+func isTimeSeriesOldFilter(metrics *queueManagerMetrics, baseTime time.Time, sampleAgeLimit time.Duration) func(ts prompb.TimeSeries) bool {
+	return func(ts prompb.TimeSeries) bool {
+		if sampleAgeLimit == 0 {
+			// If sampleAgeLimit is unset, then we never skip samples due to their age.
+			return false
+		}
+		switch {
+		// Only the first element should be set in the series, therefore we only check the first element.
+		case len(ts.Samples) > 0:
+			if isSampleOld(baseTime, sampleAgeLimit, ts.Samples[0].Timestamp) {
+				metrics.droppedSamplesTotal.WithLabelValues(reasonTooOld).Inc()
+				return true
+			}
+		case len(ts.Histograms) > 0:
+			if isSampleOld(baseTime, sampleAgeLimit, ts.Histograms[0].Timestamp) {
+				metrics.droppedHistogramsTotal.WithLabelValues(reasonTooOld).Inc()
+				return true
+			}
+		case len(ts.Exemplars) > 0:
+			if isSampleOld(baseTime, sampleAgeLimit, ts.Exemplars[0].Timestamp) {
+				metrics.droppedExemplarsTotal.WithLabelValues(reasonTooOld).Inc()
+				return true
+			}
+		default:
+			return false
+		}
+		return false
+	}
+}
+
 // Append queues a sample to be sent to the remote storage. Blocks until all samples are
 // enqueued on their shards or a shutdown signal is received.
 func (t *QueueManager) Append(samples []record.RefSample) bool {
+	currentTime := time.Now()
 outer:
 	for _, s := range samples {
+		if isSampleOld(currentTime, time.Duration(t.cfg.SampleAgeLimit), s.T) {
+			t.metrics.droppedSamplesTotal.WithLabelValues(reasonTooOld).Inc()
+			continue
+		}
 		t.seriesMtx.Lock()
 		lbls, ok := t.seriesLabels[s.Ref]
 		if !ok {
-			t.metrics.droppedSamplesTotal.Inc()
 			t.dataDropped.incr(1)
 			if _, ok := t.droppedSeries[s.Ref]; !ok {
 				level.Info(t.logger).Log("msg", "Dropped sample for series that was not explicitly dropped via relabelling", "ref", s.Ref)
+				t.metrics.droppedSamplesTotal.WithLabelValues(reasonUnintentionalDroppedSeries).Inc()
+			} else {
+				t.metrics.droppedSamplesTotal.WithLabelValues(reasonDroppedSeries).Inc()
 			}
 			t.seriesMtx.Unlock()
 			continue
@@ -629,17 +682,23 @@ func (t *QueueManager) AppendExemplars(exemplars []record.RefExemplar) bool {
 	if !t.sendExemplars {
 		return true
 	}
-
+	currentTime := time.Now()
 outer:
 	for _, e := range exemplars {
+		if isSampleOld(currentTime, time.Duration(t.cfg.SampleAgeLimit), e.T) {
+			t.metrics.droppedExemplarsTotal.WithLabelValues(reasonTooOld).Inc()
+			continue
+		}
 		t.seriesMtx.Lock()
 		lbls, ok := t.seriesLabels[e.Ref]
 		if !ok {
-			t.metrics.droppedExemplarsTotal.Inc()
 			// Track dropped exemplars in the same EWMA for sharding calc.
 			t.dataDropped.incr(1)
 			if _, ok := t.droppedSeries[e.Ref]; !ok {
 				level.Info(t.logger).Log("msg", "Dropped exemplar for series that was not explicitly dropped via relabelling", "ref", e.Ref)
+				t.metrics.droppedExemplarsTotal.WithLabelValues(reasonUnintentionalDroppedSeries).Inc()
+			} else {
+				t.metrics.droppedExemplarsTotal.WithLabelValues(reasonDroppedSeries).Inc()
 			}
 			t.seriesMtx.Unlock()
 			continue
@@ -678,16 +737,22 @@ func (t *QueueManager) AppendHistograms(histograms []record.RefHistogramSample)
 	if !t.sendNativeHistograms {
 		return true
 	}
-
+	currentTime := time.Now()
 outer:
 	for _, h := range histograms {
+		if isSampleOld(currentTime, time.Duration(t.cfg.SampleAgeLimit), h.T) {
+			t.metrics.droppedHistogramsTotal.WithLabelValues(reasonTooOld).Inc()
+			continue
+		}
 		t.seriesMtx.Lock()
 		lbls, ok := t.seriesLabels[h.Ref]
 		if !ok {
-			t.metrics.droppedHistogramsTotal.Inc()
 			t.dataDropped.incr(1)
 			if _, ok := t.droppedSeries[h.Ref]; !ok {
 				level.Info(t.logger).Log("msg", "Dropped histogram for series that was not explicitly dropped via relabelling", "ref", h.Ref)
+				t.metrics.droppedHistogramsTotal.WithLabelValues(reasonUnintentionalDroppedSeries).Inc()
+			} else {
+				t.metrics.droppedHistogramsTotal.WithLabelValues(reasonDroppedSeries).Inc()
 			}
 			t.seriesMtx.Unlock()
 			continue
@@ -725,16 +790,22 @@ func (t *QueueManager) AppendFloatHistograms(floatHistograms []record.RefFloatHi
 	if !t.sendNativeHistograms {
 		return true
 	}
-
+	currentTime := time.Now()
 outer:
 	for _, h := range floatHistograms {
+		if isSampleOld(currentTime, time.Duration(t.cfg.SampleAgeLimit), h.T) {
+			t.metrics.droppedHistogramsTotal.WithLabelValues(reasonTooOld).Inc()
+			continue
+		}
 		t.seriesMtx.Lock()
 		lbls, ok := t.seriesLabels[h.Ref]
 		if !ok {
-			t.metrics.droppedHistogramsTotal.Inc()
 			t.dataDropped.incr(1)
 			if _, ok := t.droppedSeries[h.Ref]; !ok {
 				level.Info(t.logger).Log("msg", "Dropped histogram for series that was not explicitly dropped via relabelling", "ref", h.Ref)
+				t.metrics.droppedHistogramsTotal.WithLabelValues(reasonUnintentionalDroppedSeries).Inc()
+			} else {
+				t.metrics.droppedHistogramsTotal.WithLabelValues(reasonDroppedSeries).Inc()
 			}
 			t.seriesMtx.Unlock()
 			continue
@@ -1490,7 +1561,8 @@ func (s *shards) sendSamples(ctx context.Context, samples []prompb.TimeSeries, s
 // sendSamples to the remote storage with backoff for recoverable errors.
 func (s *shards) sendSamplesWithBackoff(ctx context.Context, samples []prompb.TimeSeries, sampleCount, exemplarCount, histogramCount int, pBuf *proto.Buffer, buf *[]byte) error {
 	// Build the WriteRequest with no metadata.
-	req, highest, err := buildWriteRequest(samples, nil, pBuf, *buf)
+	req, highest, lowest, err := buildWriteRequest(s.qm.logger, samples, nil, pBuf, *buf, nil)
+	s.qm.buildRequestLimitTimestamp.Store(lowest)
 	if err != nil {
 		// Failing to build the write request is non-recoverable, since it will
 		// only error if marshaling the proto to bytes fails.
@@ -1504,6 +1576,25 @@ func (s *shards) sendSamplesWithBackoff(ctx context.Context, samples []prompb.Ti
 	// without causing a memory leak, and it has the nice effect of not propagating any
 	// parameters for sendSamplesWithBackoff/3.
 	attemptStore := func(try int) error {
+		currentTime := time.Now()
+		lowest := s.qm.buildRequestLimitTimestamp.Load()
+		if isSampleOld(currentTime, time.Duration(s.qm.cfg.SampleAgeLimit), lowest) {
+			// This will filter out old samples during retries.
+			req, _, lowest, err := buildWriteRequest(
+				s.qm.logger,
+				samples,
+				nil,
+				pBuf,
+				*buf,
+				isTimeSeriesOldFilter(s.qm.metrics, currentTime, time.Duration(s.qm.cfg.SampleAgeLimit)),
+			)
+			s.qm.buildRequestLimitTimestamp.Store(lowest)
+			if err != nil {
+				return err
+			}
+			*buf = req
+		}
+
 		ctx, span := otel.Tracer("").Start(ctx, "Remote Send Batch")
 		defer span.End()
 
@@ -1608,9 +1699,27 @@ func sendWriteRequestWithBackoff(ctx context.Context, cfg config.QueueConfig, l
 	}
 }
 
-func buildWriteRequest(samples []prompb.TimeSeries, metadata []prompb.MetricMetadata, pBuf *proto.Buffer, buf []byte) ([]byte, int64, error) {
+func buildTimeSeries(timeSeries []prompb.TimeSeries, filter func(prompb.TimeSeries) bool) (int64, int64, []prompb.TimeSeries, int, int, int) {
 	var highest int64
-	for _, ts := range samples {
+	var lowest int64
+	var droppedSamples, droppedExemplars, droppedHistograms int
+
+	keepIdx := 0
+	lowest = math.MaxInt64
+	for i, ts := range timeSeries {
+		if filter != nil && filter(ts) {
+			if len(ts.Samples) > 0 {
+				droppedSamples++
+			}
+			if len(ts.Exemplars) > 0 {
+				droppedExemplars++
+			}
+			if len(ts.Histograms) > 0 {
+				droppedHistograms++
+			}
+			continue
+		}
+
 		// At the moment we only ever append a TimeSeries with a single sample or exemplar in it.
 		if len(ts.Samples) > 0 && ts.Samples[0].Timestamp > highest {
 			highest = ts.Samples[0].Timestamp
@@ -1621,10 +1730,37 @@ func buildWriteRequest(samples []prompb.TimeSeries, metadata []prompb.MetricMeta
 		if len(ts.Histograms) > 0 && ts.Histograms[0].Timestamp > highest {
 			highest = ts.Histograms[0].Timestamp
 		}
+
+		// Get lowest timestamp
+		if len(ts.Samples) > 0 && ts.Samples[0].Timestamp < lowest {
+			lowest = ts.Samples[0].Timestamp
+		}
+		if len(ts.Exemplars) > 0 && ts.Exemplars[0].Timestamp < lowest {
+			lowest = ts.Exemplars[0].Timestamp
+		}
+		if len(ts.Histograms) > 0 && ts.Histograms[0].Timestamp < lowest {
+			lowest = ts.Histograms[0].Timestamp
+		}
+
+		// Move the current element to the write position and increment the write pointer
+		timeSeries[keepIdx] = timeSeries[i]
+		keepIdx++
+	}
+
+	timeSeries = timeSeries[:keepIdx]
+	return highest, lowest, timeSeries, droppedSamples, droppedExemplars, droppedHistograms
+}
+
+func buildWriteRequest(logger log.Logger, timeSeries []prompb.TimeSeries, metadata []prompb.MetricMetadata, pBuf *proto.Buffer, buf []byte, filter func(prompb.TimeSeries) bool) ([]byte, int64, int64, error) {
+	highest, lowest, timeSeries,
+		droppedSamples, droppedExemplars, droppedHistograms := buildTimeSeries(timeSeries, filter)
+
+	if droppedSamples > 0 || droppedExemplars > 0 || droppedHistograms > 0 {
+		level.Debug(logger).Log("msg", "dropped data due to their age", "droppedSamples", droppedSamples, "droppedExemplars", droppedExemplars, "droppedHistograms", droppedHistograms)
 	}
 
 	req := &prompb.WriteRequest{
-		Timeseries: samples,
+		Timeseries: timeSeries,
 		Metadata:   metadata,
 	}
 
@@ -1635,7 +1771,7 @@ func buildWriteRequest(samples []prompb.TimeSeries, metadata []prompb.MetricMeta
 	}
 	err := pBuf.Marshal(req)
 	if err != nil {
-		return nil, highest, err
+		return nil, highest, lowest, err
 	}
 
 	// snappy uses len() to see if it needs to allocate a new slice. Make the
@@ -1644,5 +1780,5 @@ func buildWriteRequest(samples []prompb.TimeSeries, metadata []prompb.MetricMeta
 		buf = buf[0:cap(buf)]
 	}
 	compressed := snappy.Encode(buf, pBuf.Bytes())
-	return compressed, highest, nil
+	return compressed, highest, lowest, nil
 }
diff --git a/storage/remote/queue_manager_test.go b/storage/remote/queue_manager_test.go
index 17a904fcd..778861e2b 100644
--- a/storage/remote/queue_manager_test.go
+++ b/storage/remote/queue_manager_test.go
@@ -548,7 +548,7 @@ func TestShouldReshard(t *testing.T) {
 func createTimeseries(numSamples, numSeries int, extraLabels ...labels.Label) ([]record.RefSample, []record.RefSeries) {
 	samples := make([]record.RefSample, 0, numSamples)
 	series := make([]record.RefSeries, 0, numSeries)
-	b := labels.ScratchBuilder{}
+	lb := labels.ScratchBuilder{}
 	for i := 0; i < numSeries; i++ {
 		name := fmt.Sprintf("test_metric_%d", i)
 		for j := 0; j < numSamples; j++ {
@@ -559,15 +559,15 @@ func createTimeseries(numSamples, numSeries int, extraLabels ...labels.Label) ([
 			})
 		}
 		// Create Labels that is name of series plus any extra labels supplied.
-		b.Reset()
-		b.Add(labels.MetricName, name)
+		lb.Reset()
+		lb.Add(labels.MetricName, name)
 		for _, l := range extraLabels {
-			b.Add(l.Name, l.Value)
+			lb.Add(l.Name, l.Value)
 		}
-		b.Sort()
+		lb.Sort()
 		series = append(series, record.RefSeries{
 			Ref:    chunks.HeadSeriesRef(i),
-			Labels: b.Labels(),
+			Labels: lb.Labels(),
 		})
 	}
 	return samples, series
@@ -1321,3 +1321,263 @@ func TestQueue_FlushAndShutdownDoesNotDeadlock(t *testing.T) {
 		t.FailNow()
 	}
 }
+
+func TestDropOldTimeSeries(t *testing.T) {
+	size := 10
+	nSeries := 6
+	nSamples := config.DefaultQueueConfig.Capacity * size
+	samples, newSamples, series := createTimeseriesWithOldSamples(nSamples, nSeries)
+
+	c := NewTestWriteClient()
+	c.expectSamples(newSamples, series)
+
+	cfg := config.DefaultQueueConfig
+	mcfg := config.DefaultMetadataConfig
+	cfg.MaxShards = 1
+	cfg.SampleAgeLimit = model.Duration(60 * time.Second)
+	dir := t.TempDir()
+
+	metrics := newQueueManagerMetrics(nil, "", "")
+	m := NewQueueManager(metrics, nil, nil, nil, dir, newEWMARate(ewmaWeight, shardUpdateDuration), cfg, mcfg, labels.EmptyLabels(), nil, c, defaultFlushDeadline, newPool(), newHighestTimestampMetric(), nil, false, false)
+	m.StoreSeries(series, 0)
+
+	m.Start()
+	defer m.Stop()
+
+	m.Append(samples)
+	c.waitForExpectedData(t)
+}
+
+func TestIsSampleOld(t *testing.T) {
+	currentTime := time.Now()
+	require.True(t, isSampleOld(currentTime, 60*time.Second, timestamp.FromTime(currentTime.Add(-61*time.Second))))
+	require.False(t, isSampleOld(currentTime, 60*time.Second, timestamp.FromTime(currentTime.Add(-59*time.Second))))
+}
+
+func createTimeseriesWithOldSamples(numSamples, numSeries int, extraLabels ...labels.Label) ([]record.RefSample, []record.RefSample, []record.RefSeries) {
+	newSamples := make([]record.RefSample, 0, numSamples)
+	samples := make([]record.RefSample, 0, numSamples)
+	series := make([]record.RefSeries, 0, numSeries)
+	lb := labels.ScratchBuilder{}
+	for i := 0; i < numSeries; i++ {
+		name := fmt.Sprintf("test_metric_%d", i)
+		// We create half of the samples in the past.
+		past := timestamp.FromTime(time.Now().Add(-5 * time.Minute))
+		for j := 0; j < numSamples/2; j++ {
+			samples = append(samples, record.RefSample{
+				Ref: chunks.HeadSeriesRef(i),
+				T:   past + int64(j),
+				V:   float64(i),
+			})
+		}
+		for j := 0; j < numSamples/2; j++ {
+			sample := record.RefSample{
+				Ref: chunks.HeadSeriesRef(i),
+				T:   int64(int(time.Now().UnixMilli()) + j),
+				V:   float64(i),
+			}
+			samples = append(samples, sample)
+			newSamples = append(newSamples, sample)
+		}
+		// Create Labels that is name of series plus any extra labels supplied.
+		lb.Reset()
+		lb.Add(labels.MetricName, name)
+		for _, l := range extraLabels {
+			lb.Add(l.Name, l.Value)
+		}
+		lb.Sort()
+		series = append(series, record.RefSeries{
+			Ref:    chunks.HeadSeriesRef(i),
+			Labels: lb.Labels(),
+		})
+	}
+	return samples, newSamples, series
+}
+
+func filterTsLimit(limit int64, ts prompb.TimeSeries) bool {
+	return limit > ts.Samples[0].Timestamp
+}
+
+func TestBuildTimeSeries(t *testing.T) {
+	testCases := []struct {
+		name           string
+		ts             []prompb.TimeSeries
+		filter         func(ts prompb.TimeSeries) bool
+		lowestTs       int64
+		highestTs      int64
+		droppedSamples int
+		responseLen    int
+	}{
+		{
+			name: "No filter applied",
+			ts: []prompb.TimeSeries{
+				{
+					Samples: []prompb.Sample{
+						{
+							Timestamp: 1234567890,
+							Value:     1.23,
+						},
+					},
+				},
+				{
+					Samples: []prompb.Sample{
+						{
+							Timestamp: 1234567891,
+							Value:     2.34,
+						},
+					},
+				},
+				{
+					Samples: []prompb.Sample{
+						{
+							Timestamp: 1234567892,
+							Value:     3.34,
+						},
+					},
+				},
+			},
+			filter:      nil,
+			responseLen: 3,
+			lowestTs:    1234567890,
+			highestTs:   1234567892,
+		},
+		{
+			name: "Filter applied, samples in order",
+			ts: []prompb.TimeSeries{
+				{
+					Samples: []prompb.Sample{
+						{
+							Timestamp: 1234567890,
+							Value:     1.23,
+						},
+					},
+				},
+				{
+					Samples: []prompb.Sample{
+						{
+							Timestamp: 1234567891,
+							Value:     2.34,
+						},
+					},
+				},
+				{
+					Samples: []prompb.Sample{
+						{
+							Timestamp: 1234567892,
+							Value:     3.45,
+						},
+					},
+				},
+				{
+					Samples: []prompb.Sample{
+						{
+							Timestamp: 1234567893,
+							Value:     3.45,
+						},
+					},
+				},
+			},
+			filter:         func(ts prompb.TimeSeries) bool { return filterTsLimit(1234567892, ts) },
+			responseLen:    2,
+			lowestTs:       1234567892,
+			highestTs:      1234567893,
+			droppedSamples: 2,
+		},
+		{
+			name: "Filter applied, samples out of order",
+			ts: []prompb.TimeSeries{
+				{
+					Samples: []prompb.Sample{
+						{
+							Timestamp: 1234567892,
+							Value:     3.45,
+						},
+					},
+				},
+				{
+					Samples: []prompb.Sample{
+						{
+							Timestamp: 1234567890,
+							Value:     1.23,
+						},
+					},
+				},
+				{
+					Samples: []prompb.Sample{
+						{
+							Timestamp: 1234567893,
+							Value:     3.45,
+						},
+					},
+				},
+				{
+					Samples: []prompb.Sample{
+						{
+							Timestamp: 1234567891,
+							Value:     2.34,
+						},
+					},
+				},
+			},
+			filter:         func(ts prompb.TimeSeries) bool { return filterTsLimit(1234567892, ts) },
+			responseLen:    2,
+			lowestTs:       1234567892,
+			highestTs:      1234567893,
+			droppedSamples: 2,
+		},
+		{
+			name: "Filter applied, samples not consecutive",
+			ts: []prompb.TimeSeries{
+				{
+					Samples: []prompb.Sample{
+						{
+							Timestamp: 1234567890,
+							Value:     1.23,
+						},
+					},
+				},
+				{
+					Samples: []prompb.Sample{
+						{
+							Timestamp: 1234567892,
+							Value:     3.45,
+						},
+					},
+				},
+				{
+					Samples: []prompb.Sample{
+						{
+							Timestamp: 1234567895,
+							Value:     6.78,
+						},
+					},
+				},
+				{
+					Samples: []prompb.Sample{
+						{
+							Timestamp: 1234567897,
+							Value:     6.78,
+						},
+					},
+				},
+			},
+			filter:         func(ts prompb.TimeSeries) bool { return filterTsLimit(1234567895, ts) },
+			responseLen:    2,
+			lowestTs:       1234567895,
+			highestTs:      1234567897,
+			droppedSamples: 2,
+		},
+	}
+
+	// Run the test cases
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			highest, lowest, result, droppedSamples, _, _ := buildTimeSeries(tc.ts, tc.filter)
+			require.NotNil(t, result)
+			require.Len(t, result, tc.responseLen)
+			require.Equal(t, tc.highestTs, highest)
+			require.Equal(t, tc.lowestTs, lowest)
+			require.Equal(t, tc.droppedSamples, droppedSamples)
+		})
+	}
+}
diff --git a/storage/remote/write_handler_test.go b/storage/remote/write_handler_test.go
index fd5b34ecd..9f7dcd175 100644
--- a/storage/remote/write_handler_test.go
+++ b/storage/remote/write_handler_test.go
@@ -38,7 +38,7 @@ import (
 )
 
 func TestRemoteWriteHandler(t *testing.T) {
-	buf, _, err := buildWriteRequest(writeRequestFixture.Timeseries, nil, nil, nil)
+	buf, _, _, err := buildWriteRequest(nil, writeRequestFixture.Timeseries, nil, nil, nil, nil)
 	require.NoError(t, err)
 
 	req, err := http.NewRequest("", "", bytes.NewReader(buf))
@@ -84,10 +84,10 @@ func TestRemoteWriteHandler(t *testing.T) {
 }
 
 func TestOutOfOrderSample(t *testing.T) {
-	buf, _, err := buildWriteRequest([]prompb.TimeSeries{{
+	buf, _, _, err := buildWriteRequest(nil, []prompb.TimeSeries{{
 		Labels:  []prompb.Label{{Name: "__name__", Value: "test_metric"}},
 		Samples: []prompb.Sample{{Value: 1, Timestamp: 0}},
-	}}, nil, nil, nil)
+	}}, nil, nil, nil, nil)
 	require.NoError(t, err)
 
 	req, err := http.NewRequest("", "", bytes.NewReader(buf))
@@ -109,10 +109,10 @@ func TestOutOfOrderSample(t *testing.T) {
 // don't fail on ingestion errors since the exemplar storage is
 // still experimental.
 func TestOutOfOrderExemplar(t *testing.T) {
-	buf, _, err := buildWriteRequest([]prompb.TimeSeries{{
+	buf, _, _, err := buildWriteRequest(nil, []prompb.TimeSeries{{
 		Labels:    []prompb.Label{{Name: "__name__", Value: "test_metric"}},
 		Exemplars: []prompb.Exemplar{{Labels: []prompb.Label{{Name: "foo", Value: "bar"}}, Value: 1, Timestamp: 0}},
-	}}, nil, nil, nil)
+	}}, nil, nil, nil, nil)
 	require.NoError(t, err)
 
 	req, err := http.NewRequest("", "", bytes.NewReader(buf))
@@ -132,10 +132,10 @@ func TestOutOfOrderExemplar(t *testing.T) {
 }
 
 func TestOutOfOrderHistogram(t *testing.T) {
-	buf, _, err := buildWriteRequest([]prompb.TimeSeries{{
+	buf, _, _, err := buildWriteRequest(nil, []prompb.TimeSeries{{
 		Labels:     []prompb.Label{{Name: "__name__", Value: "test_metric"}},
 		Histograms: []prompb.Histogram{HistogramToHistogramProto(0, &testHistogram), FloatHistogramToHistogramProto(1, testHistogram.ToFloat(nil))},
-	}}, nil, nil, nil)
+	}}, nil, nil, nil, nil)
 	require.NoError(t, err)
 
 	req, err := http.NewRequest("", "", bytes.NewReader(buf))
@@ -158,13 +158,13 @@ func BenchmarkRemoteWritehandler(b *testing.B) {
 	reqs := []*http.Request{}
 	for i := 0; i < b.N; i++ {
 		num := strings.Repeat(strconv.Itoa(i), 16)
-		buf, _, err := buildWriteRequest([]prompb.TimeSeries{{
+		buf, _, _, err := buildWriteRequest(nil, []prompb.TimeSeries{{
 			Labels: []prompb.Label{
 				{Name: "__name__", Value: "test_metric"},
 				{Name: "test_label_name_" + num, Value: labelValue + num},
 			},
 			Histograms: []prompb.Histogram{HistogramToHistogramProto(0, &testHistogram)},
-		}}, nil, nil, nil)
+		}}, nil, nil, nil, nil)
 		require.NoError(b, err)
 		req, err := http.NewRequest("", "", bytes.NewReader(buf))
 		require.NoError(b, err)
@@ -182,7 +182,7 @@ func BenchmarkRemoteWritehandler(b *testing.B) {
 }
 
 func TestCommitErr(t *testing.T) {
-	buf, _, err := buildWriteRequest(writeRequestFixture.Timeseries, nil, nil, nil)
+	buf, _, _, err := buildWriteRequest(nil, writeRequestFixture.Timeseries, nil, nil, nil, nil)
 	require.NoError(t, err)
 
 	req, err := http.NewRequest("", "", bytes.NewReader(buf))
@@ -219,7 +219,7 @@ func BenchmarkRemoteWriteOOOSamples(b *testing.B) {
 
 	handler := NewWriteHandler(log.NewNopLogger(), nil, db.Head())
 
-	buf, _, err := buildWriteRequest(genSeriesWithSample(1000, 200*time.Minute.Milliseconds()), nil, nil, nil)
+	buf, _, _, err := buildWriteRequest(nil, genSeriesWithSample(1000, 200*time.Minute.Milliseconds()), nil, nil, nil, nil)
 	require.NoError(b, err)
 
 	req, err := http.NewRequest("", "", bytes.NewReader(buf))
@@ -232,7 +232,7 @@ func BenchmarkRemoteWriteOOOSamples(b *testing.B) {
 
 	var bufRequests [][]byte
 	for i := 0; i < 100; i++ {
-		buf, _, err = buildWriteRequest(genSeriesWithSample(1000, int64(80+i)*time.Minute.Milliseconds()), nil, nil, nil)
+		buf, _, _, err = buildWriteRequest(nil, genSeriesWithSample(1000, int64(80+i)*time.Minute.Milliseconds()), nil, nil, nil, nil)
 		require.NoError(b, err)
 		bufRequests = append(bufRequests, buf)
 	}

From afc05129f1b0dc678c2744bf3823d6a81fbf65d6 Mon Sep 17 00:00:00 2001
From: machine424 <ayoubmrini424@gmail.com>
Date: Sun, 7 Jan 2024 20:57:53 +0100
Subject: [PATCH 072/115] fix(discovery): allow requireUpdate util to timeout
 in discovery/file/file_test.go.

The loop ran indefinitely if the condition isn't met.

Before, each iteration created a new timer channel which was always outpaced by
the other timer channel with smaller duration.

minor detail: There was a memory leak: resources of the ~10 previous timers were
constantly kept. With the fix, we may keep the resources of one timer around for defaultWait
but this isn't worth the changes to make it right.

Signed-off-by: machine424 <ayoubmrini424@gmail.com>
---
 discovery/file/file_test.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/discovery/file/file_test.go b/discovery/file/file_test.go
index c138fc8a9..731611832 100644
--- a/discovery/file/file_test.go
+++ b/discovery/file/file_test.go
@@ -193,9 +193,10 @@ func (t *testRunner) targets() []*targetgroup.Group {
 func (t *testRunner) requireUpdate(ref time.Time, expected []*targetgroup.Group) {
 	t.Helper()
 
+	timeout := time.After(defaultWait)
 	for {
 		select {
-		case <-time.After(defaultWait):
+		case <-timeout:
 			t.Fatalf("Expected update but got none")
 			return
 		case <-time.After(defaultWait / 10):

From 61b4080a14c12cd5334ee4e6a31c1b8df20503e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?=
 <giedrius.statkevicius@vinted.com>
Date: Mon, 8 Jan 2024 11:48:27 +0200
Subject: [PATCH 073/115] tsdb/{index,compact}: allow using custom postings
 encoding format (#13242)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* tsdb/{index,compact}: allow using custom postings encoding format

We would like to experiment with a different postings encoding format in
Thanos so in this change I am proposing adding another argument to
`NewWriter` which would allow users to change the format if needed.
Also, wire the leveled compactor so that it would be possible to change
the format there too.

Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>

* tsdb/compact: use a struct for leveled compactor options

As discussed on Slack, let's use a struct for the options in leveled
compactor.

Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>

* tsdb: make changes after Bryan's review

- Make changes less intrusive
- Turn the postings encoder type into a function
- Add NewWriterWithEncoder()

Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>

---------

Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
---
 tsdb/compact.go     | 39 ++++++++++++++++++++++++++++++++++-----
 tsdb/db.go          |  3 +--
 tsdb/index/index.go | 43 ++++++++++++++++++++++++++++++++-----------
 3 files changed, 67 insertions(+), 18 deletions(-)

diff --git a/tsdb/compact.go b/tsdb/compact.go
index 7b4ff9c1b..c5bd2ed2a 100644
--- a/tsdb/compact.go
+++ b/tsdb/compact.go
@@ -82,6 +82,7 @@ type LeveledCompactor struct {
 	ctx                      context.Context
 	maxBlockChunkSegmentSize int64
 	mergeFunc                storage.VerticalChunkSeriesMergeFunc
+	postingsEncoder          index.PostingsEncoder
 }
 
 type CompactorMetrics struct {
@@ -144,12 +145,30 @@ func newCompactorMetrics(r prometheus.Registerer) *CompactorMetrics {
 	return m
 }
 
-// NewLeveledCompactor returns a LeveledCompactor.
-func NewLeveledCompactor(ctx context.Context, r prometheus.Registerer, l log.Logger, ranges []int64, pool chunkenc.Pool, mergeFunc storage.VerticalChunkSeriesMergeFunc) (*LeveledCompactor, error) {
-	return NewLeveledCompactorWithChunkSize(ctx, r, l, ranges, pool, chunks.DefaultChunkSegmentSize, mergeFunc)
+type LeveledCompactorOptions struct {
+	// PE specifies the postings encoder. It is called when compactor is writing out the postings for a label name/value pair during compaction.
+	// If it is nil then the default encoder is used. At the moment that is the "raw" encoder. See index.EncodePostingsRaw for more.
+	PE index.PostingsEncoder
+	// MaxBlockChunkSegmentSize is the max block chunk segment size. If it is 0 then the default chunks.DefaultChunkSegmentSize is used.
+	MaxBlockChunkSegmentSize int64
+	// MergeFunc is used for merging series together in vertical compaction. By default storage.NewCompactingChunkSeriesMerger(storage.ChainedSeriesMerge) is used.
+	MergeFunc storage.VerticalChunkSeriesMergeFunc
 }
 
 func NewLeveledCompactorWithChunkSize(ctx context.Context, r prometheus.Registerer, l log.Logger, ranges []int64, pool chunkenc.Pool, maxBlockChunkSegmentSize int64, mergeFunc storage.VerticalChunkSeriesMergeFunc) (*LeveledCompactor, error) {
+	return NewLeveledCompactorWithOptions(ctx, r, l, ranges, pool, LeveledCompactorOptions{
+		MaxBlockChunkSegmentSize: maxBlockChunkSegmentSize,
+		MergeFunc:                mergeFunc,
+	})
+}
+
+func NewLeveledCompactor(ctx context.Context, r prometheus.Registerer, l log.Logger, ranges []int64, pool chunkenc.Pool, mergeFunc storage.VerticalChunkSeriesMergeFunc) (*LeveledCompactor, error) {
+	return NewLeveledCompactorWithOptions(ctx, r, l, ranges, pool, LeveledCompactorOptions{
+		MergeFunc: mergeFunc,
+	})
+}
+
+func NewLeveledCompactorWithOptions(ctx context.Context, r prometheus.Registerer, l log.Logger, ranges []int64, pool chunkenc.Pool, opts LeveledCompactorOptions) (*LeveledCompactor, error) {
 	if len(ranges) == 0 {
 		return nil, fmt.Errorf("at least one range must be provided")
 	}
@@ -159,9 +178,18 @@ func NewLeveledCompactorWithChunkSize(ctx context.Context, r prometheus.Register
 	if l == nil {
 		l = log.NewNopLogger()
 	}
-	if mergeFunc == nil {
+	var mergeFunc storage.VerticalChunkSeriesMergeFunc
+	if opts.MergeFunc == nil {
 		mergeFunc = storage.NewCompactingChunkSeriesMerger(storage.ChainedSeriesMerge)
 	}
+	var maxBlockChunkSegmentSize int64
+	if opts.MaxBlockChunkSegmentSize == 0 {
+		maxBlockChunkSegmentSize = chunks.DefaultChunkSegmentSize
+	}
+	var pe index.PostingsEncoder
+	if opts.PE == nil {
+		pe = index.EncodePostingsRaw
+	}
 	return &LeveledCompactor{
 		ranges:                   ranges,
 		chunkPool:                pool,
@@ -170,6 +198,7 @@ func NewLeveledCompactorWithChunkSize(ctx context.Context, r prometheus.Register
 		ctx:                      ctx,
 		maxBlockChunkSegmentSize: maxBlockChunkSegmentSize,
 		mergeFunc:                mergeFunc,
+		postingsEncoder:          pe,
 	}, nil
 }
 
@@ -599,7 +628,7 @@ func (c *LeveledCompactor) write(dest string, meta *BlockMeta, blockPopulator Bl
 		}
 	}
 
-	indexw, err := index.NewWriter(c.ctx, filepath.Join(tmp, indexFilename))
+	indexw, err := index.NewWriterWithEncoder(c.ctx, filepath.Join(tmp, indexFilename), c.postingsEncoder)
 	if err != nil {
 		return fmt.Errorf("open index writer: %w", err)
 	}
diff --git a/tsdb/db.go b/tsdb/db.go
index b2cc37a19..bdeb3faee 100644
--- a/tsdb/db.go
+++ b/tsdb/db.go
@@ -442,8 +442,7 @@ func (db *DBReadOnly) FlushWAL(dir string) (returnErr error) {
 		nil,
 		db.logger,
 		ExponentialBlockRanges(DefaultOptions().MinBlockDuration, 3, 5),
-		chunkenc.NewPool(),
-		nil,
+		chunkenc.NewPool(), nil,
 	)
 	if err != nil {
 		return fmt.Errorf("create leveled compactor: %w", err)
diff --git a/tsdb/index/index.go b/tsdb/index/index.go
index 2b025a352..c2ca581f7 100644
--- a/tsdb/index/index.go
+++ b/tsdb/index/index.go
@@ -110,6 +110,8 @@ type symbolCacheEntry struct {
 	lastValue      string
 }
 
+type PostingsEncoder func(*encoding.Encbuf, []uint32) error
+
 // Writer implements the IndexWriter interface for the standard
 // serialization format.
 type Writer struct {
@@ -148,6 +150,8 @@ type Writer struct {
 	crc32 hash.Hash
 
 	Version int
+
+	postingsEncoder PostingsEncoder
 }
 
 // TOC represents index Table Of Content that states where each section of index starts.
@@ -186,7 +190,8 @@ func NewTOCFromByteSlice(bs ByteSlice) (*TOC, error) {
 }
 
 // NewWriter returns a new Writer to the given filename. It serializes data in format version 2.
-func NewWriter(ctx context.Context, fn string) (*Writer, error) {
+// It uses the given encoder to encode each postings list.
+func NewWriterWithEncoder(ctx context.Context, fn string, encoder PostingsEncoder) (*Writer, error) {
 	dir := filepath.Dir(fn)
 
 	df, err := fileutil.OpenDir(dir)
@@ -229,9 +234,10 @@ func NewWriter(ctx context.Context, fn string) (*Writer, error) {
 		buf1: encoding.Encbuf{B: make([]byte, 0, 1<<22)},
 		buf2: encoding.Encbuf{B: make([]byte, 0, 1<<22)},
 
-		symbolCache: make(map[string]symbolCacheEntry, 1<<8),
-		labelNames:  make(map[string]uint64, 1<<8),
-		crc32:       newCRC32(),
+		symbolCache:     make(map[string]symbolCacheEntry, 1<<8),
+		labelNames:      make(map[string]uint64, 1<<8),
+		crc32:           newCRC32(),
+		postingsEncoder: encoder,
 	}
 	if err := iw.writeMeta(); err != nil {
 		return nil, err
@@ -239,6 +245,12 @@ func NewWriter(ctx context.Context, fn string) (*Writer, error) {
 	return iw, nil
 }
 
+// NewWriter creates a new index writer using the default encoder. See
+// NewWriterWithEncoder.
+func NewWriter(ctx context.Context, fn string) (*Writer, error) {
+	return NewWriterWithEncoder(ctx, fn, EncodePostingsRaw)
+}
+
 func (w *Writer) write(bufs ...[]byte) error {
 	return w.f.Write(bufs...)
 }
@@ -941,6 +953,20 @@ func (w *Writer) writePostingsToTmpFiles() error {
 	return nil
 }
 
+// EncodePostingsRaw uses the "basic" postings list encoding format with no compression:
+// <BE uint32 len X><BE uint32 0><BE uint32 1>...<BE uint32 X-1>.
+func EncodePostingsRaw(e *encoding.Encbuf, offs []uint32) error {
+	e.PutBE32int(len(offs))
+
+	for _, off := range offs {
+		if off > (1<<32)-1 {
+			return fmt.Errorf("series offset %d exceeds 4 bytes", off)
+		}
+		e.PutBE32(off)
+	}
+	return nil
+}
+
 func (w *Writer) writePosting(name, value string, offs []uint32) error {
 	// Align beginning to 4 bytes for more efficient postings list scans.
 	if err := w.fP.AddPadding(4); err != nil {
@@ -959,13 +985,8 @@ func (w *Writer) writePosting(name, value string, offs []uint32) error {
 	w.cntPO++
 
 	w.buf1.Reset()
-	w.buf1.PutBE32int(len(offs))
-
-	for _, off := range offs {
-		if off > (1<<32)-1 {
-			return fmt.Errorf("series offset %d exceeds 4 bytes", off)
-		}
-		w.buf1.PutBE32(off)
+	if err := w.postingsEncoder(&w.buf1, offs); err != nil {
+		return err
 	}
 
 	w.buf2.Reset()

From 3c44fbcced24d61212b621d96adee3cc22e9b16d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 8 Jan 2024 16:36:24 +0100
Subject: [PATCH 074/115] build(deps): bump actions/setup-go from 3.5.0 to
 5.0.0 in /scripts (#13362)

Bumps [actions/setup-go](https://github.com/actions/setup-go) from 3.5.0 to 5.0.0.
- [Release notes](https://github.com/actions/setup-go/releases)
- [Commits](https://github.com/actions/setup-go/compare/6edd4406fa81c3da01a34fa6f6343087c207a568...0c52d547c9bc32b1aa3301fd7a9cb496313a4491)

---
updated-dependencies:
- dependency-name: actions/setup-go
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 scripts/golangci-lint.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/golangci-lint.yml b/scripts/golangci-lint.yml
index 4b292229d..8f2527911 100644
--- a/scripts/golangci-lint.yml
+++ b/scripts/golangci-lint.yml
@@ -26,7 +26,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - name: install Go
-        uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
+        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
         with:
           go-version: 1.21.x
       - name: Install snmp_exporter/generator dependencies

From 88a6b3a80a7c44717bfc32b0e68410147f628cf7 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 8 Jan 2024 16:36:53 +0100
Subject: [PATCH 075/115] build(deps): bump github/codeql-action from 2.22.8 to
 3.22.12 (#13358)

Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2.22.8 to 3.22.12.
- [Release notes](https://github.com/github/codeql-action/releases)
- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/github/codeql-action/compare/407ffafae6a767df3e0230c3df91b6443ae8df75...012739e5082ff0c22ca6d6ab32e07c36df03c4a4)

---
updated-dependencies:
- dependency-name: github/codeql-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/codeql-analysis.yml | 6 +++---
 .github/workflows/scorecards.yml      | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index 5e14936a9..fd1ef19ef 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -30,12 +30,12 @@ jobs:
           go-version: 1.21.x
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@407ffafae6a767df3e0230c3df91b6443ae8df75 # v2.22.8
+        uses: github/codeql-action/init@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # v3.22.12
         with:
           languages: ${{ matrix.language }}
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@407ffafae6a767df3e0230c3df91b6443ae8df75 # v2.22.8
+        uses: github/codeql-action/autobuild@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # v3.22.12
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@407ffafae6a767df3e0230c3df91b6443ae8df75 # v2.22.8
+        uses: github/codeql-action/analyze@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # v3.22.12
diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml
index f71e1331b..636dd1bd1 100644
--- a/.github/workflows/scorecards.yml
+++ b/.github/workflows/scorecards.yml
@@ -45,6 +45,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@407ffafae6a767df3e0230c3df91b6443ae8df75 # tag=v2.22.8
+        uses: github/codeql-action/upload-sarif@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # tag=v3.22.12
         with:
           sarif_file: results.sarif

From 6cd24d87b0d4e3acd4503b061d32c5aa6d8883a9 Mon Sep 17 00:00:00 2001
From: Augustin Husson <husson.augustin@gmail.com>
Date: Mon, 8 Jan 2024 17:01:24 +0100
Subject: [PATCH 076/115] put @nexucis has a release shepherd (#13383)

Signed-off-by: Augustin Husson <augustin.husson@amadeus.com>
---
 RELEASE.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/RELEASE.md b/RELEASE.md
index 6ab2f6389..6815308f4 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -54,7 +54,8 @@ Release cadence of first pre-releases being cut is 6 weeks.
 | v2.47          | 2023-08-23                                 | Bryan Boreham (GitHub: @bboreham)           |
 | v2.48          | 2023-10-04                                 | Levi Harrison (GitHub: @LeviHarrison)       |
 | v2.49          | 2023-12-05                                 | Bartek Plotka (GitHub: @bwplotka)           |
-| v2.50          | 2024-01-16                                 | **searching for volunteer**                 |
+| v2.50          | 2024-01-16                                 | Augustin Husson (GitHub: @nexucis)          |
+| v2.51          | 2024-02-13                                 | **searching for volunteer**                 |
 
 If you are interested in volunteering please create a pull request against the [prometheus/prometheus](https://github.com/prometheus/prometheus) repository and propose yourself for the release series of your choice.
 

From 6150e1ca0ede508e56414363cc9062ef522db518 Mon Sep 17 00:00:00 2001
From: zenador <zenador@users.noreply.github.com>
Date: Thu, 11 Jan 2024 00:32:36 +0800
Subject: [PATCH 077/115] Add analyze histograms command to promtool (#12331)

Add `query analyze` command to promtool

This command analyzes the buckets of classic and native histograms,
based on data queried from the Prometheus query API, i.e. it
doesn't require direct access to the TSDB files.

Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>

---------

Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
---
 cmd/promtool/analyze.go       | 370 ++++++++++++++++++++++++++++++++++
 cmd/promtool/analyze_test.go  | 170 ++++++++++++++++
 cmd/promtool/main.go          | 260 ++----------------------
 cmd/promtool/query.go         | 251 +++++++++++++++++++++++
 docs/command-line/promtool.md |  19 ++
 5 files changed, 823 insertions(+), 247 deletions(-)
 create mode 100644 cmd/promtool/analyze.go
 create mode 100644 cmd/promtool/analyze_test.go
 create mode 100644 cmd/promtool/query.go

diff --git a/cmd/promtool/analyze.go b/cmd/promtool/analyze.go
new file mode 100644
index 000000000..c1f523de5
--- /dev/null
+++ b/cmd/promtool/analyze.go
@@ -0,0 +1,370 @@
+// Copyright 2023 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"net/url"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
+	"github.com/prometheus/common/model"
+
+	"github.com/prometheus/prometheus/model/labels"
+)
+
+var (
+	errNotNativeHistogram = fmt.Errorf("not a native histogram")
+	errNotEnoughData      = fmt.Errorf("not enough data")
+
+	outputHeader = `Bucket stats for each histogram series over time
+------------------------------------------------
+First the min, avg, and max number of populated buckets, followed by the total
+number of buckets (only if different from the max number of populated buckets
+which is typical for classic but not native histograms).`
+	outputFooter = `Aggregated bucket stats
+-----------------------
+Each line shows min/avg/max over the series above.`
+)
+
+type QueryAnalyzeConfig struct {
+	metricType string
+	duration   time.Duration
+	time       string
+	matchers   []string
+}
+
+// run retrieves metrics that look like conventional histograms (i.e. have _bucket
+// suffixes) or native histograms, depending on metricType flag.
+func (c *QueryAnalyzeConfig) run(url *url.URL, roundtripper http.RoundTripper) error {
+	if c.metricType != "histogram" {
+		return fmt.Errorf("analyze type is %s, must be 'histogram'", c.metricType)
+	}
+
+	ctx := context.Background()
+
+	api, err := newAPI(url, roundtripper, nil)
+	if err != nil {
+		return err
+	}
+
+	var endTime time.Time
+	if c.time != "" {
+		endTime, err = parseTime(c.time)
+		if err != nil {
+			return fmt.Errorf("error parsing time '%s': %w", c.time, err)
+		}
+	} else {
+		endTime = time.Now()
+	}
+
+	return c.getStatsFromMetrics(ctx, api, endTime, os.Stdout, c.matchers)
+}
+
+func (c *QueryAnalyzeConfig) getStatsFromMetrics(ctx context.Context, api v1.API, endTime time.Time, out io.Writer, matchers []string) error {
+	fmt.Fprintf(out, "%s\n\n", outputHeader)
+	metastatsNative := newMetaStatistics()
+	metastatsClassic := newMetaStatistics()
+	for _, matcher := range matchers {
+		seriesSel := seriesSelector(matcher, c.duration)
+		matrix, err := querySamples(ctx, api, seriesSel, endTime)
+		if err != nil {
+			return err
+		}
+
+		matrices := make(map[string]model.Matrix)
+		for _, series := range matrix {
+			// We do not handle mixed types. If there are float values, we assume it is a
+			// classic histogram, otherwise we assume it is a native histogram, and we
+			// ignore series with errors if they do not match the expected type.
+			if len(series.Values) == 0 {
+				stats, err := calcNativeBucketStatistics(series)
+				if err != nil {
+					if errors.Is(err, errNotNativeHistogram) || errors.Is(err, errNotEnoughData) {
+						continue
+					}
+					return err
+				}
+				fmt.Fprintf(out, "- %s (native): %v\n", series.Metric, *stats)
+				metastatsNative.update(stats)
+			} else {
+				lbs := model.LabelSet(series.Metric).Clone()
+				if _, ok := lbs["le"]; !ok {
+					continue
+				}
+				metricName := string(lbs[labels.MetricName])
+				if !strings.HasSuffix(metricName, "_bucket") {
+					continue
+				}
+				delete(lbs, labels.MetricName)
+				delete(lbs, "le")
+				key := formatSeriesName(metricName, lbs)
+				matrices[key] = append(matrices[key], series)
+			}
+		}
+
+		for key, matrix := range matrices {
+			stats, err := calcClassicBucketStatistics(matrix)
+			if err != nil {
+				if errors.Is(err, errNotEnoughData) {
+					continue
+				}
+				return err
+			}
+			fmt.Fprintf(out, "- %s (classic): %v\n", key, *stats)
+			metastatsClassic.update(stats)
+		}
+	}
+	fmt.Fprintf(out, "\n%s\n", outputFooter)
+	if metastatsNative.Count() > 0 {
+		fmt.Fprintf(out, "\nNative %s\n", metastatsNative)
+	}
+	if metastatsClassic.Count() > 0 {
+		fmt.Fprintf(out, "\nClassic %s\n", metastatsClassic)
+	}
+	return nil
+}
+
+func seriesSelector(metricName string, duration time.Duration) string {
+	builder := strings.Builder{}
+	builder.WriteString(metricName)
+	builder.WriteRune('[')
+	builder.WriteString(duration.String())
+	builder.WriteRune(']')
+	return builder.String()
+}
+
+func formatSeriesName(metricName string, lbs model.LabelSet) string {
+	builder := strings.Builder{}
+	builder.WriteString(metricName)
+	builder.WriteString(lbs.String())
+	return builder.String()
+}
+
+func querySamples(ctx context.Context, api v1.API, query string, end time.Time) (model.Matrix, error) {
+	values, _, err := api.Query(ctx, query, end)
+	if err != nil {
+		return nil, err
+	}
+
+	matrix, ok := values.(model.Matrix)
+	if !ok {
+		return nil, fmt.Errorf("query of buckets resulted in non-Matrix")
+	}
+
+	return matrix, nil
+}
+
+// minPop/avgPop/maxPop is for the number of populated (non-zero) buckets.
+// total is the total number of buckets across all samples in the series,
+// populated or not.
+type statistics struct {
+	minPop, maxPop, total int
+	avgPop                float64
+}
+
+func (s statistics) String() string {
+	if s.maxPop == s.total {
+		return fmt.Sprintf("%d/%.3f/%d", s.minPop, s.avgPop, s.maxPop)
+	}
+	return fmt.Sprintf("%d/%.3f/%d/%d", s.minPop, s.avgPop, s.maxPop, s.total)
+}
+
+func calcClassicBucketStatistics(matrix model.Matrix) (*statistics, error) {
+	numBuckets := len(matrix)
+
+	stats := &statistics{
+		minPop: math.MaxInt,
+		total:  numBuckets,
+	}
+
+	if numBuckets == 0 || len(matrix[0].Values) < 2 {
+		return stats, errNotEnoughData
+	}
+
+	numSamples := len(matrix[0].Values)
+
+	sortMatrix(matrix)
+
+	totalPop := 0
+	for timeIdx := 0; timeIdx < numSamples; timeIdx++ {
+		curr, err := getBucketCountsAtTime(matrix, numBuckets, timeIdx)
+		if err != nil {
+			return stats, err
+		}
+		countPop := 0
+		for _, b := range curr {
+			if b != 0 {
+				countPop++
+			}
+		}
+
+		totalPop += countPop
+		if stats.minPop > countPop {
+			stats.minPop = countPop
+		}
+		if stats.maxPop < countPop {
+			stats.maxPop = countPop
+		}
+	}
+	stats.avgPop = float64(totalPop) / float64(numSamples)
+	return stats, nil
+}
+
+func sortMatrix(matrix model.Matrix) {
+	sort.SliceStable(matrix, func(i, j int) bool {
+		return getLe(matrix[i]) < getLe(matrix[j])
+	})
+}
+
+func getLe(series *model.SampleStream) float64 {
+	lbs := model.LabelSet(series.Metric)
+	le, _ := strconv.ParseFloat(string(lbs["le"]), 64)
+	return le
+}
+
+func getBucketCountsAtTime(matrix model.Matrix, numBuckets, timeIdx int) ([]int, error) {
+	counts := make([]int, numBuckets)
+	if timeIdx >= len(matrix[0].Values) {
+		// Just return zeroes instead of erroring out so we can get partial results.
+		return counts, nil
+	}
+	counts[0] = int(matrix[0].Values[timeIdx].Value)
+	for i, bucket := range matrix[1:] {
+		if timeIdx >= len(bucket.Values) {
+			// Just return zeroes instead of erroring out so we can get partial results.
+			return counts, nil
+		}
+		curr := bucket.Values[timeIdx]
+		prev := matrix[i].Values[timeIdx]
+		// Assume the results are nicely aligned.
+		if curr.Timestamp != prev.Timestamp {
+			return counts, fmt.Errorf("matrix result is not time aligned")
+		}
+		counts[i+1] = int(curr.Value - prev.Value)
+	}
+	return counts, nil
+}
+
+type bucketBounds struct {
+	boundaries   int32
+	upper, lower float64
+}
+
+func makeBucketBounds(b *model.HistogramBucket) bucketBounds {
+	return bucketBounds{
+		boundaries: b.Boundaries,
+		upper:      float64(b.Upper),
+		lower:      float64(b.Lower),
+	}
+}
+
+func calcNativeBucketStatistics(series *model.SampleStream) (*statistics, error) {
+	stats := &statistics{
+		minPop: math.MaxInt,
+	}
+
+	overall := make(map[bucketBounds]struct{})
+	totalPop := 0
+	if len(series.Histograms) == 0 {
+		return nil, errNotNativeHistogram
+	}
+	if len(series.Histograms) == 1 {
+		return nil, errNotEnoughData
+	}
+	for _, histogram := range series.Histograms {
+		for _, bucket := range histogram.Histogram.Buckets {
+			bb := makeBucketBounds(bucket)
+			overall[bb] = struct{}{}
+		}
+		countPop := len(histogram.Histogram.Buckets)
+
+		totalPop += countPop
+		if stats.minPop > countPop {
+			stats.minPop = countPop
+		}
+		if stats.maxPop < countPop {
+			stats.maxPop = countPop
+		}
+	}
+	stats.avgPop = float64(totalPop) / float64(len(series.Histograms))
+	stats.total = len(overall)
+	return stats, nil
+}
+
+type distribution struct {
+	min, max, count int
+	avg             float64
+}
+
+func newDistribution() distribution {
+	return distribution{
+		min: math.MaxInt,
+	}
+}
+
+func (d *distribution) update(num int) {
+	if d.min > num {
+		d.min = num
+	}
+	if d.max < num {
+		d.max = num
+	}
+	d.count++
+	d.avg += float64(num)/float64(d.count) - d.avg/float64(d.count)
+}
+
+func (d distribution) String() string {
+	return fmt.Sprintf("%d/%.3f/%d", d.min, d.avg, d.max)
+}
+
+type metaStatistics struct {
+	minPop, avgPop, maxPop, total distribution
+}
+
+func newMetaStatistics() *metaStatistics {
+	return &metaStatistics{
+		minPop: newDistribution(),
+		avgPop: newDistribution(),
+		maxPop: newDistribution(),
+		total:  newDistribution(),
+	}
+}
+
+func (ms metaStatistics) Count() int {
+	return ms.minPop.count
+}
+
+func (ms metaStatistics) String() string {
+	if ms.maxPop == ms.total {
+		return fmt.Sprintf("histogram series (%d in total):\n- min populated: %v\n- avg populated: %v\n- max populated: %v", ms.Count(), ms.minPop, ms.avgPop, ms.maxPop)
+	}
+	return fmt.Sprintf("histogram series (%d in total):\n- min populated: %v\n- avg populated: %v\n- max populated: %v\n- total: %v", ms.Count(), ms.minPop, ms.avgPop, ms.maxPop, ms.total)
+}
+
+func (ms *metaStatistics) update(s *statistics) {
+	ms.minPop.update(s.minPop)
+	ms.avgPop.update(int(s.avgPop))
+	ms.maxPop.update(s.maxPop)
+	ms.total.update(s.total)
+}
diff --git a/cmd/promtool/analyze_test.go b/cmd/promtool/analyze_test.go
new file mode 100644
index 000000000..83d2ac4a3
--- /dev/null
+++ b/cmd/promtool/analyze_test.go
@@ -0,0 +1,170 @@
+// Copyright 2023 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/common/model"
+)
+
+var (
+	exampleMatrix = model.Matrix{
+		&model.SampleStream{
+			Metric: model.Metric{
+				"le": "+Inf",
+			},
+			Values: []model.SamplePair{
+				{
+					Value:     31,
+					Timestamp: 100,
+				},
+				{
+					Value:     32,
+					Timestamp: 200,
+				},
+				{
+					Value:     40,
+					Timestamp: 300,
+				},
+			},
+		},
+		&model.SampleStream{
+			Metric: model.Metric{
+				"le": "0.5",
+			},
+			Values: []model.SamplePair{
+				{
+					Value:     10,
+					Timestamp: 100,
+				},
+				{
+					Value:     11,
+					Timestamp: 200,
+				},
+				{
+					Value:     11,
+					Timestamp: 300,
+				},
+			},
+		},
+		&model.SampleStream{
+			Metric: model.Metric{
+				"le": "10",
+			},
+			Values: []model.SamplePair{
+				{
+					Value:     30,
+					Timestamp: 100,
+				},
+				{
+					Value:     31,
+					Timestamp: 200,
+				},
+				{
+					Value:     37,
+					Timestamp: 300,
+				},
+			},
+		},
+		&model.SampleStream{
+			Metric: model.Metric{
+				"le": "2",
+			},
+			Values: []model.SamplePair{
+				{
+					Value:     25,
+					Timestamp: 100,
+				},
+				{
+					Value:     26,
+					Timestamp: 200,
+				},
+				{
+					Value:     27,
+					Timestamp: 300,
+				},
+			},
+		},
+	}
+	exampleMatrixLength = len(exampleMatrix)
+)
+
+func init() {
+	sortMatrix(exampleMatrix)
+}
+
+func TestGetBucketCountsAtTime(t *testing.T) {
+	cases := []struct {
+		matrix   model.Matrix
+		length   int
+		timeIdx  int
+		expected []int
+	}{
+		{
+			exampleMatrix,
+			exampleMatrixLength,
+			0,
+			[]int{10, 15, 5, 1},
+		},
+		{
+			exampleMatrix,
+			exampleMatrixLength,
+			1,
+			[]int{11, 15, 5, 1},
+		},
+		{
+			exampleMatrix,
+			exampleMatrixLength,
+			2,
+			[]int{11, 16, 10, 3},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(fmt.Sprintf("exampleMatrix@%d", c.timeIdx), func(t *testing.T) {
+			res, err := getBucketCountsAtTime(c.matrix, c.length, c.timeIdx)
+			require.NoError(t, err)
+			require.Equal(t, c.expected, res)
+		})
+	}
+}
+
+func TestCalcClassicBucketStatistics(t *testing.T) {
+	cases := []struct {
+		matrix   model.Matrix
+		expected *statistics
+	}{
+		{
+			exampleMatrix,
+			&statistics{
+				minPop: 4,
+				avgPop: 4,
+				maxPop: 4,
+				total:  4,
+			},
+		},
+	}
+
+	for i, c := range cases {
+		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
+			res, err := calcClassicBucketStatistics(c.matrix)
+			require.NoError(t, err)
+			require.Equal(t, c.expected, res)
+		})
+	}
+}
diff --git a/cmd/promtool/main.go b/cmd/promtool/main.go
index 508b681b8..61dcabdc6 100644
--- a/cmd/promtool/main.go
+++ b/cmd/promtool/main.go
@@ -35,9 +35,7 @@ import (
 	"github.com/go-kit/log"
 	"github.com/google/pprof/profile"
 	"github.com/prometheus/client_golang/api"
-	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/prometheus/client_golang/prometheus/testutil/promlint"
 	config_util "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
@@ -185,6 +183,14 @@ func main() {
 	queryLabelsEnd := queryLabelsCmd.Flag("end", "End time (RFC3339 or Unix timestamp).").String()
 	queryLabelsMatch := queryLabelsCmd.Flag("match", "Series selector. Can be specified multiple times.").Strings()
 
+	queryAnalyzeCfg := &QueryAnalyzeConfig{}
+	queryAnalyzeCmd := queryCmd.Command("analyze", "Run queries against your Prometheus to analyze the usage pattern of certain metrics.")
+	queryAnalyzeCmd.Flag("server", "Prometheus server to query.").Required().URLVar(&serverURL)
+	queryAnalyzeCmd.Flag("type", "Type of metric: histogram.").Required().StringVar(&queryAnalyzeCfg.metricType)
+	queryAnalyzeCmd.Flag("duration", "Time frame to analyze.").Default("1h").DurationVar(&queryAnalyzeCfg.duration)
+	queryAnalyzeCmd.Flag("time", "Query time (RFC3339 or Unix timestamp), defaults to now.").StringVar(&queryAnalyzeCfg.time)
+	queryAnalyzeCmd.Flag("match", "Series selector. Can be specified multiple times.").Required().StringsVar(&queryAnalyzeCfg.matchers)
+
 	pushCmd := app.Command("push", "Push to a Prometheus server.")
 	pushCmd.Flag("http.config.file", "HTTP client configuration file for promtool to connect to Prometheus.").PlaceHolder("<filename>").ExistingFileVar(&httpConfigFilePath)
 	pushMetricsCmd := pushCmd.Command("metrics", "Push metrics to a prometheus remote write (for testing purpose only).")
@@ -390,6 +396,9 @@ func main() {
 	case importRulesCmd.FullCommand():
 		os.Exit(checkErr(importRules(serverURL, httpRoundTripper, *importRulesStart, *importRulesEnd, *importRulesOutputDir, *importRulesEvalInterval, *maxBlockDuration, *importRulesFiles...)))
 
+	case queryAnalyzeCmd.FullCommand():
+		os.Exit(checkErr(queryAnalyzeCfg.run(serverURL, httpRoundTripper)))
+
 	case documentationCmd.FullCommand():
 		os.Exit(checkErr(documentcli.GenerateMarkdown(app.Model(), os.Stdout)))
 
@@ -997,246 +1006,6 @@ func checkMetricsExtended(r io.Reader) ([]metricStat, int, error) {
 	return stats, total, nil
 }
 
-// QueryInstant performs an instant query against a Prometheus server.
-func QueryInstant(url *url.URL, roundTripper http.RoundTripper, query, evalTime string, p printer) int {
-	if url.Scheme == "" {
-		url.Scheme = "http"
-	}
-	config := api.Config{
-		Address:      url.String(),
-		RoundTripper: roundTripper,
-	}
-
-	// Create new client.
-	c, err := api.NewClient(config)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, "error creating API client:", err)
-		return failureExitCode
-	}
-
-	eTime := time.Now()
-	if evalTime != "" {
-		eTime, err = parseTime(evalTime)
-		if err != nil {
-			fmt.Fprintln(os.Stderr, "error parsing evaluation time:", err)
-			return failureExitCode
-		}
-	}
-
-	// Run query against client.
-	api := v1.NewAPI(c)
-
-	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
-	val, _, err := api.Query(ctx, query, eTime) // Ignoring warnings for now.
-	cancel()
-	if err != nil {
-		return handleAPIError(err)
-	}
-
-	p.printValue(val)
-
-	return successExitCode
-}
-
-// QueryRange performs a range query against a Prometheus server.
-func QueryRange(url *url.URL, roundTripper http.RoundTripper, headers map[string]string, query, start, end string, step time.Duration, p printer) int {
-	if url.Scheme == "" {
-		url.Scheme = "http"
-	}
-	config := api.Config{
-		Address:      url.String(),
-		RoundTripper: roundTripper,
-	}
-
-	if len(headers) > 0 {
-		config.RoundTripper = promhttp.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
-			for key, value := range headers {
-				req.Header.Add(key, value)
-			}
-			return roundTripper.RoundTrip(req)
-		})
-	}
-
-	// Create new client.
-	c, err := api.NewClient(config)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, "error creating API client:", err)
-		return failureExitCode
-	}
-
-	var stime, etime time.Time
-
-	if end == "" {
-		etime = time.Now()
-	} else {
-		etime, err = parseTime(end)
-		if err != nil {
-			fmt.Fprintln(os.Stderr, "error parsing end time:", err)
-			return failureExitCode
-		}
-	}
-
-	if start == "" {
-		stime = etime.Add(-5 * time.Minute)
-	} else {
-		stime, err = parseTime(start)
-		if err != nil {
-			fmt.Fprintln(os.Stderr, "error parsing start time:", err)
-			return failureExitCode
-		}
-	}
-
-	if !stime.Before(etime) {
-		fmt.Fprintln(os.Stderr, "start time is not before end time")
-		return failureExitCode
-	}
-
-	if step == 0 {
-		resolution := math.Max(math.Floor(etime.Sub(stime).Seconds()/250), 1)
-		// Convert seconds to nanoseconds such that time.Duration parses correctly.
-		step = time.Duration(resolution) * time.Second
-	}
-
-	// Run query against client.
-	api := v1.NewAPI(c)
-	r := v1.Range{Start: stime, End: etime, Step: step}
-	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
-	val, _, err := api.QueryRange(ctx, query, r) // Ignoring warnings for now.
-	cancel()
-
-	if err != nil {
-		return handleAPIError(err)
-	}
-
-	p.printValue(val)
-	return successExitCode
-}
-
-// QuerySeries queries for a series against a Prometheus server.
-func QuerySeries(url *url.URL, roundTripper http.RoundTripper, matchers []string, start, end string, p printer) int {
-	if url.Scheme == "" {
-		url.Scheme = "http"
-	}
-	config := api.Config{
-		Address:      url.String(),
-		RoundTripper: roundTripper,
-	}
-
-	// Create new client.
-	c, err := api.NewClient(config)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, "error creating API client:", err)
-		return failureExitCode
-	}
-
-	stime, etime, err := parseStartTimeAndEndTime(start, end)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		return failureExitCode
-	}
-
-	// Run query against client.
-	api := v1.NewAPI(c)
-	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
-	val, _, err := api.Series(ctx, matchers, stime, etime) // Ignoring warnings for now.
-	cancel()
-
-	if err != nil {
-		return handleAPIError(err)
-	}
-
-	p.printSeries(val)
-	return successExitCode
-}
-
-// QueryLabels queries for label values against a Prometheus server.
-func QueryLabels(url *url.URL, roundTripper http.RoundTripper, matchers []string, name, start, end string, p printer) int {
-	if url.Scheme == "" {
-		url.Scheme = "http"
-	}
-	config := api.Config{
-		Address:      url.String(),
-		RoundTripper: roundTripper,
-	}
-
-	// Create new client.
-	c, err := api.NewClient(config)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, "error creating API client:", err)
-		return failureExitCode
-	}
-
-	stime, etime, err := parseStartTimeAndEndTime(start, end)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		return failureExitCode
-	}
-
-	// Run query against client.
-	api := v1.NewAPI(c)
-	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
-	val, warn, err := api.LabelValues(ctx, name, matchers, stime, etime)
-	cancel()
-
-	for _, v := range warn {
-		fmt.Fprintln(os.Stderr, "query warning:", v)
-	}
-	if err != nil {
-		return handleAPIError(err)
-	}
-
-	p.printLabelValues(val)
-	return successExitCode
-}
-
-func handleAPIError(err error) int {
-	var apiErr *v1.Error
-	if errors.As(err, &apiErr) && apiErr.Detail != "" {
-		fmt.Fprintf(os.Stderr, "query error: %v (detail: %s)\n", apiErr, strings.TrimSpace(apiErr.Detail))
-	} else {
-		fmt.Fprintln(os.Stderr, "query error:", err)
-	}
-
-	return failureExitCode
-}
-
-func parseStartTimeAndEndTime(start, end string) (time.Time, time.Time, error) {
-	var (
-		minTime = time.Now().Add(-9999 * time.Hour)
-		maxTime = time.Now().Add(9999 * time.Hour)
-		err     error
-	)
-
-	stime := minTime
-	etime := maxTime
-
-	if start != "" {
-		stime, err = parseTime(start)
-		if err != nil {
-			return stime, etime, fmt.Errorf("error parsing start time: %w", err)
-		}
-	}
-
-	if end != "" {
-		etime, err = parseTime(end)
-		if err != nil {
-			return stime, etime, fmt.Errorf("error parsing end time: %w", err)
-		}
-	}
-	return stime, etime, nil
-}
-
-func parseTime(s string) (time.Time, error) {
-	if t, err := strconv.ParseFloat(s, 64); err == nil {
-		s, ns := math.Modf(t)
-		return time.Unix(int64(s), int64(ns*float64(time.Second))).UTC(), nil
-	}
-	if t, err := time.Parse(time.RFC3339Nano, s); err == nil {
-		return t, nil
-	}
-	return time.Time{}, fmt.Errorf("cannot parse %q to a valid timestamp", s)
-}
-
 type endpointsGroup struct {
 	urlToFilename map[string]string
 	postProcess   func(b []byte) ([]byte, error)
@@ -1390,15 +1159,12 @@ func importRules(url *url.URL, roundTripper http.RoundTripper, start, end, outpu
 		evalInterval:     evalInterval,
 		maxBlockDuration: maxBlockDuration,
 	}
-	client, err := api.NewClient(api.Config{
-		Address:      url.String(),
-		RoundTripper: roundTripper,
-	})
+	api, err := newAPI(url, roundTripper, nil)
 	if err != nil {
 		return fmt.Errorf("new api client error: %w", err)
 	}
 
-	ruleImporter := newRuleImporter(log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr)), cfg, v1.NewAPI(client))
+	ruleImporter := newRuleImporter(log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr)), cfg, api)
 	errs := ruleImporter.loadGroups(ctx, files)
 	for _, err := range errs {
 		if err != nil {
diff --git a/cmd/promtool/query.go b/cmd/promtool/query.go
new file mode 100644
index 000000000..0d7cb12cf
--- /dev/null
+++ b/cmd/promtool/query.go
@@ -0,0 +1,251 @@
+// Copyright 2023 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math"
+	"net/http"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/prometheus/client_golang/api"
+	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
+	_ "github.com/prometheus/prometheus/plugins" // Register plugins.
+)
+
+func newAPI(url *url.URL, roundTripper http.RoundTripper, headers map[string]string) (v1.API, error) {
+	if url.Scheme == "" {
+		url.Scheme = "http"
+	}
+	config := api.Config{
+		Address:      url.String(),
+		RoundTripper: roundTripper,
+	}
+
+	if len(headers) > 0 {
+		config.RoundTripper = promhttp.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
+			for key, value := range headers {
+				req.Header.Add(key, value)
+			}
+			return roundTripper.RoundTrip(req)
+		})
+	}
+
+	// Create new client.
+	client, err := api.NewClient(config)
+	if err != nil {
+		return nil, err
+	}
+
+	api := v1.NewAPI(client)
+	return api, nil
+}
+
+// QueryInstant performs an instant query against a Prometheus server.
+func QueryInstant(url *url.URL, roundTripper http.RoundTripper, query, evalTime string, p printer) int {
+	api, err := newAPI(url, roundTripper, nil)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "error creating API client:", err)
+		return failureExitCode
+	}
+
+	eTime := time.Now()
+	if evalTime != "" {
+		eTime, err = parseTime(evalTime)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, "error parsing evaluation time:", err)
+			return failureExitCode
+		}
+	}
+
+	// Run query against client.
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	val, _, err := api.Query(ctx, query, eTime) // Ignoring warnings for now.
+	cancel()
+	if err != nil {
+		return handleAPIError(err)
+	}
+
+	p.printValue(val)
+
+	return successExitCode
+}
+
+// QueryRange performs a range query against a Prometheus server.
+func QueryRange(url *url.URL, roundTripper http.RoundTripper, headers map[string]string, query, start, end string, step time.Duration, p printer) int {
+	api, err := newAPI(url, roundTripper, headers)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "error creating API client:", err)
+		return failureExitCode
+	}
+
+	var stime, etime time.Time
+
+	if end == "" {
+		etime = time.Now()
+	} else {
+		etime, err = parseTime(end)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, "error parsing end time:", err)
+			return failureExitCode
+		}
+	}
+
+	if start == "" {
+		stime = etime.Add(-5 * time.Minute)
+	} else {
+		stime, err = parseTime(start)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, "error parsing start time:", err)
+			return failureExitCode
+		}
+	}
+
+	if !stime.Before(etime) {
+		fmt.Fprintln(os.Stderr, "start time is not before end time")
+		return failureExitCode
+	}
+
+	if step == 0 {
+		resolution := math.Max(math.Floor(etime.Sub(stime).Seconds()/250), 1)
+		// Convert seconds to nanoseconds such that time.Duration parses correctly.
+		step = time.Duration(resolution) * time.Second
+	}
+
+	// Run query against client.
+	r := v1.Range{Start: stime, End: etime, Step: step}
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	val, _, err := api.QueryRange(ctx, query, r) // Ignoring warnings for now.
+	cancel()
+
+	if err != nil {
+		return handleAPIError(err)
+	}
+
+	p.printValue(val)
+	return successExitCode
+}
+
+// QuerySeries queries for a series against a Prometheus server.
+func QuerySeries(url *url.URL, roundTripper http.RoundTripper, matchers []string, start, end string, p printer) int {
+	api, err := newAPI(url, roundTripper, nil)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "error creating API client:", err)
+		return failureExitCode
+	}
+
+	stime, etime, err := parseStartTimeAndEndTime(start, end)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		return failureExitCode
+	}
+
+	// Run query against client.
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	val, _, err := api.Series(ctx, matchers, stime, etime) // Ignoring warnings for now.
+	cancel()
+
+	if err != nil {
+		return handleAPIError(err)
+	}
+
+	p.printSeries(val)
+	return successExitCode
+}
+
+// QueryLabels queries for label values against a Prometheus server.
+func QueryLabels(url *url.URL, roundTripper http.RoundTripper, matchers []string, name, start, end string, p printer) int {
+	api, err := newAPI(url, roundTripper, nil)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "error creating API client:", err)
+		return failureExitCode
+	}
+
+	stime, etime, err := parseStartTimeAndEndTime(start, end)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		return failureExitCode
+	}
+
+	// Run query against client.
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	val, warn, err := api.LabelValues(ctx, name, matchers, stime, etime)
+	cancel()
+
+	for _, v := range warn {
+		fmt.Fprintln(os.Stderr, "query warning:", v)
+	}
+	if err != nil {
+		return handleAPIError(err)
+	}
+
+	p.printLabelValues(val)
+	return successExitCode
+}
+
+func handleAPIError(err error) int {
+	var apiErr *v1.Error
+	if errors.As(err, &apiErr) && apiErr.Detail != "" {
+		fmt.Fprintf(os.Stderr, "query error: %v (detail: %s)\n", apiErr, strings.TrimSpace(apiErr.Detail))
+	} else {
+		fmt.Fprintln(os.Stderr, "query error:", err)
+	}
+
+	return failureExitCode
+}
+
+func parseStartTimeAndEndTime(start, end string) (time.Time, time.Time, error) {
+	var (
+		minTime = time.Now().Add(-9999 * time.Hour)
+		maxTime = time.Now().Add(9999 * time.Hour)
+		err     error
+	)
+
+	stime := minTime
+	etime := maxTime
+
+	if start != "" {
+		stime, err = parseTime(start)
+		if err != nil {
+			return stime, etime, fmt.Errorf("error parsing start time: %w", err)
+		}
+	}
+
+	if end != "" {
+		etime, err = parseTime(end)
+		if err != nil {
+			return stime, etime, fmt.Errorf("error parsing end time: %w", err)
+		}
+	}
+	return stime, etime, nil
+}
+
+func parseTime(s string) (time.Time, error) {
+	if t, err := strconv.ParseFloat(s, 64); err == nil {
+		s, ns := math.Modf(t)
+		return time.Unix(int64(s), int64(ns*float64(time.Second))).UTC(), nil
+	}
+	if t, err := time.Parse(time.RFC3339Nano, s); err == nil {
+		return t, nil
+	}
+	return time.Time{}, fmt.Errorf("cannot parse %q to a valid timestamp", s)
+}
diff --git a/docs/command-line/promtool.md b/docs/command-line/promtool.md
index 9b0fc7cc6..e9ee7597e 100644
--- a/docs/command-line/promtool.md
+++ b/docs/command-line/promtool.md
@@ -324,6 +324,25 @@ Run labels query.
 
 
 
+##### `promtool query analyze`
+
+Run queries against your Prometheus to analyze the usage pattern of certain metrics.
+
+
+
+###### Flags
+
+| Flag | Description | Default |
+| --- | --- | --- |
+| <code class="text-nowrap">--server</code> | Prometheus server to query. |  |
+| <code class="text-nowrap">--type</code> | Type of metric: histogram. |  |
+| <code class="text-nowrap">--duration</code> | Time frame to analyze. | `1h` |
+| <code class="text-nowrap">--time</code> | Query time (RFC3339 or Unix timestamp), defaults to now. |  |
+| <code class="text-nowrap">--match</code> | Series selector. Can be specified multiple times. |  |
+
+
+
+
 ### `promtool debug`
 
 Fetch debug information.

From d09d77b62a70f8bb2d7ad231452a47ca34d2df19 Mon Sep 17 00:00:00 2001
From: Erik Sommer <ersotech@posteo.de>
Date: Wed, 10 Jan 2024 19:37:10 +0100
Subject: [PATCH 078/115] included instance in all necessary descriptions

Signed-off-by: Erik Sommer <ersotech@posteo.de>
---
 .../prometheus-mixin/dashboards.libsonnet        | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/documentation/prometheus-mixin/dashboards.libsonnet b/documentation/prometheus-mixin/dashboards.libsonnet
index 2c508e6e0..efe53dbac 100644
--- a/documentation/prometheus-mixin/dashboards.libsonnet
+++ b/documentation/prometheus-mixin/dashboards.libsonnet
@@ -37,12 +37,12 @@ local template = grafana.template;
         g.row('Discovery')
         .addPanel(
           g.panel('Target Sync') +
-          g.queryPanel('sum(rate(prometheus_target_sync_length_seconds_sum{cluster=~"$cluster",job=~"$job",instance=~"$instance"}[5m])) by (cluster, job, scrape_job, instance) * 1e3', '{{cluster}}:{{job}}:{{scrape_job}}') +
+          g.queryPanel('sum(rate(prometheus_target_sync_length_seconds_sum{cluster=~"$cluster",job=~"$job",instance=~"$instance"}[5m])) by (cluster, job, scrape_job, instance) * 1e3', '{{cluster}}:{{job}}:{{instance}}:{{scrape_job}}') +
           { yaxes: g.yaxes('ms') }
         )
         .addPanel(
           g.panel('Targets') +
-          g.queryPanel('sum by (cluster, job, instance) (prometheus_sd_discovered_targets{cluster=~"$cluster", job=~"$job",instance=~"$instance"})', '{{cluster}}:{{job}}') +
+          g.queryPanel('sum by (cluster, job, instance) (prometheus_sd_discovered_targets{cluster=~"$cluster", job=~"$job",instance=~"$instance"})', '{{cluster}}:{{job}}:{{instance}}') +
           g.stack
         )
       )
@@ -50,7 +50,7 @@ local template = grafana.template;
         g.row('Retrieval')
         .addPanel(
           g.panel('Average Scrape Interval Duration') +
-          g.queryPanel('rate(prometheus_target_interval_length_seconds_sum{cluster=~"$cluster", job=~"$job",instance=~"$instance"}[5m]) / rate(prometheus_target_interval_length_seconds_count{cluster=~"$cluster", job=~"$job",instance=~"$instance"}[5m]) * 1e3', '{{cluster}}:{{job}} {{interval}} configured') +
+          g.queryPanel('rate(prometheus_target_interval_length_seconds_sum{cluster=~"$cluster", job=~"$job",instance=~"$instance"}[5m]) / rate(prometheus_target_interval_length_seconds_count{cluster=~"$cluster", job=~"$job",instance=~"$instance"}[5m]) * 1e3', '{{cluster}}:{{job}}:{{instance}} {{interval}} configured') +
           { yaxes: g.yaxes('ms') }
         )
         .addPanel(
@@ -62,11 +62,11 @@ local template = grafana.template;
             'sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_out_of_bounds_total{cluster=~"$cluster",job=~"$job",instance=~"$instance"}[1m]))',
             'sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_out_of_order_total{cluster=~"$cluster",job=~"$job",instance=~"$instance"}[1m]))',
           ], [
-            'exceeded body size limit: {{cluster}} {{job}}',
-            'exceeded sample limit: {{cluster}} {{job}}',
-            'duplicate timestamp: {{cluster}} {{job}}',
-            'out of bounds: {{cluster}} {{job}}',
-            'out of order: {{cluster}} {{job}}',
+            'exceeded body size limit: {{cluster}} {{job}} {{instance}}',
+            'exceeded sample limit: {{cluster}} {{job}} {{instance}}',
+            'duplicate timestamp: {{cluster}} {{job}} {{instance}}',
+            'out of bounds: {{cluster}} {{job}} {{instance}}',
+            'out of order: {{cluster}} {{job}} {{instance}}',
           ]) +
           g.stack
         )

From 9b759135d15ceb4c9aab0e39be260543051e8e92 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?=
 <giedrius.statkevicius@vinted.com>
Date: Thu, 11 Jan 2024 12:07:54 +0200
Subject: [PATCH 079/115] tsdb/compact: fix passing merge func
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Fixing a very small logical problem I've introduced :(.

Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
---
 tsdb/compact.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tsdb/compact.go b/tsdb/compact.go
index c5bd2ed2a..e9639f1e0 100644
--- a/tsdb/compact.go
+++ b/tsdb/compact.go
@@ -178,8 +178,8 @@ func NewLeveledCompactorWithOptions(ctx context.Context, r prometheus.Registerer
 	if l == nil {
 		l = log.NewNopLogger()
 	}
-	var mergeFunc storage.VerticalChunkSeriesMergeFunc
-	if opts.MergeFunc == nil {
+	mergeFunc := opts.MergeFunc
+	if mergeFunc == nil {
 		mergeFunc = storage.NewCompactingChunkSeriesMerger(storage.ChainedSeriesMerge)
 	}
 	var maxBlockChunkSegmentSize int64

From 3a48adc54f1d77dd3956cc0f12c33b8f3454c6af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?=
 <giedrius.statkevicius@vinted.com>
Date: Fri, 12 Jan 2024 09:56:44 +0200
Subject: [PATCH 080/115] tsdb: add enable overlapping compaction
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This functionality is needed in downstream projects because they have a
separate component that does compaction.

Upstreaming
https://github.com/grafana/mimir-prometheus/blob/7c8e9a2a76fc729e9078889782928b2fdfe240e9/tsdb/compact.go#L323-L325.

Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
---
 tsdb/compact.go | 48 +++++++++++++++++++++++++++++-------------------
 1 file changed, 29 insertions(+), 19 deletions(-)

diff --git a/tsdb/compact.go b/tsdb/compact.go
index c5bd2ed2a..b4c194de9 100644
--- a/tsdb/compact.go
+++ b/tsdb/compact.go
@@ -75,14 +75,15 @@ type Compactor interface {
 
 // LeveledCompactor implements the Compactor interface.
 type LeveledCompactor struct {
-	metrics                  *CompactorMetrics
-	logger                   log.Logger
-	ranges                   []int64
-	chunkPool                chunkenc.Pool
-	ctx                      context.Context
-	maxBlockChunkSegmentSize int64
-	mergeFunc                storage.VerticalChunkSeriesMergeFunc
-	postingsEncoder          index.PostingsEncoder
+	metrics                     *CompactorMetrics
+	logger                      log.Logger
+	ranges                      []int64
+	chunkPool                   chunkenc.Pool
+	ctx                         context.Context
+	maxBlockChunkSegmentSize    int64
+	mergeFunc                   storage.VerticalChunkSeriesMergeFunc
+	postingsEncoder             index.PostingsEncoder
+	enableOverlappingCompaction bool
 }
 
 type CompactorMetrics struct {
@@ -153,18 +154,23 @@ type LeveledCompactorOptions struct {
 	MaxBlockChunkSegmentSize int64
 	// MergeFunc is used for merging series together in vertical compaction. By default storage.NewCompactingChunkSeriesMerger(storage.ChainedSeriesMerge) is used.
 	MergeFunc storage.VerticalChunkSeriesMergeFunc
+	// EnableOverlappingCompaction enables compaction of overlapping blocks. In Prometheus it is always enabled.
+	// It is useful for downstream projects like Mimir, Cortex, Thanos where they have a separate component that does compaction.
+	EnableOverlappingCompaction bool
 }
 
 func NewLeveledCompactorWithChunkSize(ctx context.Context, r prometheus.Registerer, l log.Logger, ranges []int64, pool chunkenc.Pool, maxBlockChunkSegmentSize int64, mergeFunc storage.VerticalChunkSeriesMergeFunc) (*LeveledCompactor, error) {
 	return NewLeveledCompactorWithOptions(ctx, r, l, ranges, pool, LeveledCompactorOptions{
-		MaxBlockChunkSegmentSize: maxBlockChunkSegmentSize,
-		MergeFunc:                mergeFunc,
+		MaxBlockChunkSegmentSize:    maxBlockChunkSegmentSize,
+		MergeFunc:                   mergeFunc,
+		EnableOverlappingCompaction: true,
 	})
 }
 
 func NewLeveledCompactor(ctx context.Context, r prometheus.Registerer, l log.Logger, ranges []int64, pool chunkenc.Pool, mergeFunc storage.VerticalChunkSeriesMergeFunc) (*LeveledCompactor, error) {
 	return NewLeveledCompactorWithOptions(ctx, r, l, ranges, pool, LeveledCompactorOptions{
-		MergeFunc: mergeFunc,
+		MergeFunc:                   mergeFunc,
+		EnableOverlappingCompaction: true,
 	})
 }
 
@@ -191,14 +197,15 @@ func NewLeveledCompactorWithOptions(ctx context.Context, r prometheus.Registerer
 		pe = index.EncodePostingsRaw
 	}
 	return &LeveledCompactor{
-		ranges:                   ranges,
-		chunkPool:                pool,
-		logger:                   l,
-		metrics:                  newCompactorMetrics(r),
-		ctx:                      ctx,
-		maxBlockChunkSegmentSize: maxBlockChunkSegmentSize,
-		mergeFunc:                mergeFunc,
-		postingsEncoder:          pe,
+		ranges:                      ranges,
+		chunkPool:                   pool,
+		logger:                      l,
+		metrics:                     newCompactorMetrics(r),
+		ctx:                         ctx,
+		maxBlockChunkSegmentSize:    maxBlockChunkSegmentSize,
+		mergeFunc:                   mergeFunc,
+		postingsEncoder:             pe,
+		enableOverlappingCompaction: opts.EnableOverlappingCompaction,
 	}, nil
 }
 
@@ -317,6 +324,9 @@ func (c *LeveledCompactor) selectDirs(ds []dirMeta) []dirMeta {
 // selectOverlappingDirs returns all dirs with overlapping time ranges.
 // It expects sorted input by mint and returns the overlapping dirs in the same order as received.
 func (c *LeveledCompactor) selectOverlappingDirs(ds []dirMeta) []string {
+	if !c.enableOverlappingCompaction {
+		return nil
+	}
 	if len(ds) < 2 {
 		return nil
 	}

From ace9c8a3da7a9c90af43ee871ad23aacbcb84877 Mon Sep 17 00:00:00 2001
From: Ayoub Mrini <ayoubmrini424@gmail.com>
Date: Mon, 15 Jan 2024 11:29:53 +0100
Subject: [PATCH 081/115] promtool: allow setting multiple matchers to
 "promtool tsdb dump" command. (#13296)

Conditions are ANDed inside the same matcher but matchers are ORed

Including unit tests for "promtool tsdb dump".

Refactor some matchers scraping utils.

Signed-off-by: machine424 <ayoubmrini424@gmail.com>
---
 cmd/promtool/main.go                   |   2 +-
 cmd/promtool/testdata/dump-test-1.prom |  15 ++++
 cmd/promtool/testdata/dump-test-2.prom |  10 +++
 cmd/promtool/testdata/dump-test-3.prom |   2 +
 cmd/promtool/tsdb.go                   |  16 +++-
 cmd/promtool/tsdb_test.go              | 107 +++++++++++++++++++++++++
 docs/command-line/promtool.md          |   2 +-
 promql/parser/parse.go                 |  14 ++++
 web/api/v1/api.go                      |  10 +--
 web/federate.go                        |  12 +--
 10 files changed, 170 insertions(+), 20 deletions(-)
 create mode 100644 cmd/promtool/testdata/dump-test-1.prom
 create mode 100644 cmd/promtool/testdata/dump-test-2.prom
 create mode 100644 cmd/promtool/testdata/dump-test-3.prom

diff --git a/cmd/promtool/main.go b/cmd/promtool/main.go
index 61dcabdc6..5a7f21d3b 100644
--- a/cmd/promtool/main.go
+++ b/cmd/promtool/main.go
@@ -236,7 +236,7 @@ func main() {
 	dumpPath := tsdbDumpCmd.Arg("db path", "Database path (default is "+defaultDBPath+").").Default(defaultDBPath).String()
 	dumpMinTime := tsdbDumpCmd.Flag("min-time", "Minimum timestamp to dump.").Default(strconv.FormatInt(math.MinInt64, 10)).Int64()
 	dumpMaxTime := tsdbDumpCmd.Flag("max-time", "Maximum timestamp to dump.").Default(strconv.FormatInt(math.MaxInt64, 10)).Int64()
-	dumpMatch := tsdbDumpCmd.Flag("match", "Series selector.").Default("{__name__=~'(?s:.*)'}").String()
+	dumpMatch := tsdbDumpCmd.Flag("match", "Series selector. Can be specified multiple times.").Default("{__name__=~'(?s:.*)'}").Strings()
 
 	importCmd := tsdbCmd.Command("create-blocks-from", "[Experimental] Import samples from input and produce TSDB blocks. Please refer to the storage docs for more details.")
 	importHumanReadable := importCmd.Flag("human-readable", "Print human readable values.").Short('r').Bool()
diff --git a/cmd/promtool/testdata/dump-test-1.prom b/cmd/promtool/testdata/dump-test-1.prom
new file mode 100644
index 000000000..878cdecab
--- /dev/null
+++ b/cmd/promtool/testdata/dump-test-1.prom
@@ -0,0 +1,15 @@
+{__name__="heavy_metric", foo="bar"} 5 0
+{__name__="heavy_metric", foo="bar"} 4 60000
+{__name__="heavy_metric", foo="bar"} 3 120000
+{__name__="heavy_metric", foo="bar"} 2 180000
+{__name__="heavy_metric", foo="bar"} 1 240000
+{__name__="heavy_metric", foo="foo"} 5 0
+{__name__="heavy_metric", foo="foo"} 4 60000
+{__name__="heavy_metric", foo="foo"} 3 120000
+{__name__="heavy_metric", foo="foo"} 2 180000
+{__name__="heavy_metric", foo="foo"} 1 240000
+{__name__="metric", baz="abc", foo="bar"} 1 0
+{__name__="metric", baz="abc", foo="bar"} 2 60000
+{__name__="metric", baz="abc", foo="bar"} 3 120000
+{__name__="metric", baz="abc", foo="bar"} 4 180000
+{__name__="metric", baz="abc", foo="bar"} 5 240000
diff --git a/cmd/promtool/testdata/dump-test-2.prom b/cmd/promtool/testdata/dump-test-2.prom
new file mode 100644
index 000000000..4ac2ffa5a
--- /dev/null
+++ b/cmd/promtool/testdata/dump-test-2.prom
@@ -0,0 +1,10 @@
+{__name__="heavy_metric", foo="foo"} 5 0
+{__name__="heavy_metric", foo="foo"} 4 60000
+{__name__="heavy_metric", foo="foo"} 3 120000
+{__name__="heavy_metric", foo="foo"} 2 180000
+{__name__="heavy_metric", foo="foo"} 1 240000
+{__name__="metric", baz="abc", foo="bar"} 1 0
+{__name__="metric", baz="abc", foo="bar"} 2 60000
+{__name__="metric", baz="abc", foo="bar"} 3 120000
+{__name__="metric", baz="abc", foo="bar"} 4 180000
+{__name__="metric", baz="abc", foo="bar"} 5 240000
diff --git a/cmd/promtool/testdata/dump-test-3.prom b/cmd/promtool/testdata/dump-test-3.prom
new file mode 100644
index 000000000..faa278101
--- /dev/null
+++ b/cmd/promtool/testdata/dump-test-3.prom
@@ -0,0 +1,2 @@
+{__name__="metric", baz="abc", foo="bar"} 2 60000
+{__name__="metric", baz="abc", foo="bar"} 3 120000
diff --git a/cmd/promtool/tsdb.go b/cmd/promtool/tsdb.go
index e6df9b78c..a9239d937 100644
--- a/cmd/promtool/tsdb.go
+++ b/cmd/promtool/tsdb.go
@@ -706,7 +706,7 @@ func analyzeCompaction(ctx context.Context, block tsdb.BlockReader, indexr tsdb.
 	return nil
 }
 
-func dumpSamples(ctx context.Context, path string, mint, maxt int64, match string) (err error) {
+func dumpSamples(ctx context.Context, path string, mint, maxt int64, match []string) (err error) {
 	db, err := tsdb.OpenDBReadOnly(path, nil)
 	if err != nil {
 		return err
@@ -720,11 +720,21 @@ func dumpSamples(ctx context.Context, path string, mint, maxt int64, match strin
 	}
 	defer q.Close()
 
-	matchers, err := parser.ParseMetricSelector(match)
+	matcherSets, err := parser.ParseMetricSelectors(match)
 	if err != nil {
 		return err
 	}
-	ss := q.Select(ctx, false, nil, matchers...)
+
+	var ss storage.SeriesSet
+	if len(matcherSets) > 1 {
+		var sets []storage.SeriesSet
+		for _, mset := range matcherSets {
+			sets = append(sets, q.Select(ctx, true, nil, mset...))
+		}
+		ss = storage.NewMergeSeriesSet(sets, storage.ChainedSeriesMerge)
+	} else {
+		ss = q.Select(ctx, false, nil, matcherSets[0]...)
+	}
 
 	for ss.Next() {
 		series := ss.At()
diff --git a/cmd/promtool/tsdb_test.go b/cmd/promtool/tsdb_test.go
index 0f0040cd3..aeb51a07e 100644
--- a/cmd/promtool/tsdb_test.go
+++ b/cmd/promtool/tsdb_test.go
@@ -14,9 +14,18 @@
 package main
 
 import (
+	"bytes"
+	"context"
+	"io"
+	"math"
+	"os"
+	"runtime"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/prometheus/promql"
 )
 
 func TestGenerateBucket(t *testing.T) {
@@ -41,3 +50,101 @@ func TestGenerateBucket(t *testing.T) {
 		require.Equal(t, tc.step, step)
 	}
 }
+
+// getDumpedSamples dumps samples and returns them.
+func getDumpedSamples(t *testing.T, path string, mint, maxt int64, match []string) string {
+	t.Helper()
+
+	oldStdout := os.Stdout
+	r, w, _ := os.Pipe()
+	os.Stdout = w
+
+	err := dumpSamples(
+		context.Background(),
+		path,
+		mint,
+		maxt,
+		match,
+	)
+	require.NoError(t, err)
+
+	w.Close()
+	os.Stdout = oldStdout
+
+	var buf bytes.Buffer
+	io.Copy(&buf, r)
+	return buf.String()
+}
+
+func TestTSDBDump(t *testing.T) {
+	storage := promql.LoadedStorage(t, `
+		load 1m
+			metric{foo="bar", baz="abc"} 1 2 3 4 5
+			heavy_metric{foo="bar"} 5 4 3 2 1
+			heavy_metric{foo="foo"} 5 4 3 2 1
+	`)
+
+	tests := []struct {
+		name         string
+		mint         int64
+		maxt         int64
+		match        []string
+		expectedDump string
+	}{
+		{
+			name:         "default match",
+			mint:         math.MinInt64,
+			maxt:         math.MaxInt64,
+			match:        []string{"{__name__=~'(?s:.*)'}"},
+			expectedDump: "testdata/dump-test-1.prom",
+		},
+		{
+			name:         "same matcher twice",
+			mint:         math.MinInt64,
+			maxt:         math.MaxInt64,
+			match:        []string{"{foo=~'.+'}", "{foo=~'.+'}"},
+			expectedDump: "testdata/dump-test-1.prom",
+		},
+		{
+			name:         "no duplication",
+			mint:         math.MinInt64,
+			maxt:         math.MaxInt64,
+			match:        []string{"{__name__=~'(?s:.*)'}", "{baz='abc'}"},
+			expectedDump: "testdata/dump-test-1.prom",
+		},
+		{
+			name:         "well merged",
+			mint:         math.MinInt64,
+			maxt:         math.MaxInt64,
+			match:        []string{"{__name__='heavy_metric'}", "{baz='abc'}"},
+			expectedDump: "testdata/dump-test-1.prom",
+		},
+		{
+			name:         "multi matchers",
+			mint:         math.MinInt64,
+			maxt:         math.MaxInt64,
+			match:        []string{"{__name__='heavy_metric',foo='foo'}", "{__name__='metric'}"},
+			expectedDump: "testdata/dump-test-2.prom",
+		},
+		{
+			name:         "with reduced mint and maxt",
+			mint:         int64(60000),
+			maxt:         int64(120000),
+			match:        []string{"{__name__='metric'}"},
+			expectedDump: "testdata/dump-test-3.prom",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			dumpedMetrics := getDumpedSamples(t, storage.Dir(), tt.mint, tt.maxt, tt.match)
+			expectedMetrics, err := os.ReadFile(tt.expectedDump)
+			require.NoError(t, err)
+			if strings.Contains(runtime.GOOS, "windows") {
+				// We use "/n" while dumping on windows as well.
+				expectedMetrics = bytes.ReplaceAll(expectedMetrics, []byte("\r\n"), []byte("\n"))
+			}
+			// even though in case of one matcher samples are not sorted, the order in the cases above should stay the same.
+			require.Equal(t, string(expectedMetrics), dumpedMetrics)
+		})
+	}
+}
diff --git a/docs/command-line/promtool.md b/docs/command-line/promtool.md
index e9ee7597e..ba948685b 100644
--- a/docs/command-line/promtool.md
+++ b/docs/command-line/promtool.md
@@ -567,7 +567,7 @@ Dump samples from a TSDB.
 | --- | --- | --- |
 | <code class="text-nowrap">--min-time</code> | Minimum timestamp to dump. | `-9223372036854775808` |
 | <code class="text-nowrap">--max-time</code> | Maximum timestamp to dump. | `9223372036854775807` |
-| <code class="text-nowrap">--match</code> | Series selector. | `{__name__=~'(?s:.*)'}` |
+| <code class="text-nowrap">--match</code> | Series selector. Can be specified multiple times. | `{__name__=~'(?s:.*)'}` |
 
 
 
diff --git a/promql/parser/parse.go b/promql/parser/parse.go
index 122286c55..c2a42ed15 100644
--- a/promql/parser/parse.go
+++ b/promql/parser/parse.go
@@ -208,6 +208,20 @@ func ParseMetricSelector(input string) (m []*labels.Matcher, err error) {
 	return m, err
 }
 
+// ParseMetricSelectors parses a list of provided textual metric selectors into lists of
+// label matchers.
+func ParseMetricSelectors(matchers []string) (m [][]*labels.Matcher, err error) {
+	var matcherSets [][]*labels.Matcher
+	for _, s := range matchers {
+		matchers, err := ParseMetricSelector(s)
+		if err != nil {
+			return nil, err
+		}
+		matcherSets = append(matcherSets, matchers)
+	}
+	return matcherSets, nil
+}
+
 // SequenceValue is an omittable value in a sequence of time series values.
 type SequenceValue struct {
 	Value     float64
diff --git a/web/api/v1/api.go b/web/api/v1/api.go
index dd35d1fe9..d9d4cfd1d 100644
--- a/web/api/v1/api.go
+++ b/web/api/v1/api.go
@@ -1848,13 +1848,9 @@ func parseDuration(s string) (time.Duration, error) {
 }
 
 func parseMatchersParam(matchers []string) ([][]*labels.Matcher, error) {
-	var matcherSets [][]*labels.Matcher
-	for _, s := range matchers {
-		matchers, err := parser.ParseMetricSelector(s)
-		if err != nil {
-			return nil, err
-		}
-		matcherSets = append(matcherSets, matchers)
+	matcherSets, err := parser.ParseMetricSelectors(matchers)
+	if err != nil {
+		return nil, err
 	}
 
 OUTER:
diff --git a/web/federate.go b/web/federate.go
index 2e7bac21d..22384a696 100644
--- a/web/federate.go
+++ b/web/federate.go
@@ -65,14 +65,10 @@ func (h *Handler) federation(w http.ResponseWriter, req *http.Request) {
 		return
 	}
 
-	var matcherSets [][]*labels.Matcher
-	for _, s := range req.Form["match[]"] {
-		matchers, err := parser.ParseMetricSelector(s)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusBadRequest)
-			return
-		}
-		matcherSets = append(matcherSets, matchers)
+	matcherSets, err := parser.ParseMetricSelectors(req.Form["match[]"])
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
 	}
 
 	var (

From 09d351668f208feda2c56aba9f3e4b81d7480c0f Mon Sep 17 00:00:00 2001
From: bwplotka <bwplotka@gmail.com>
Date: Mon, 15 Jan 2024 11:42:09 +0000
Subject: [PATCH 082/115] Fixed changelog

Signed-off-by: bwplotka <bwplotka@gmail.com>
---
 CHANGELOG.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 116a2594b..abe4b556c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,10 @@
 # Changelog
 
+## unreleased
+
+* [ENHANCEMENT] TSDB: Make the wlog watcher read segments synchronously when not tailing. #13224
+* [BUGFIX] Agent: Participate in notify calls. #13223
+
 ## 2.49.0 / 2024-01-15
 
 * [FEATURE] Promtool: Add `--run` flag promtool test rules command. #12206

From b695e069b812bdcb56af1206b56de26c56d0f188 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?=
 <giedrius.statkevicius@vinted.com>
Date: Mon, 15 Jan 2024 17:42:40 +0200
Subject: [PATCH 083/115] tsdb/main: wire "EnableOverlappingCompaction" to
 tsdb.Options (#13398)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This added the https://github.com/prometheus/prometheus/pull/13393
"EnableOverlappingCompaction" parameter to the compactor code but not to
the tsdb.Options. I forgot about that. Add it to `tsdb.Options` too and
set it to `true` in Prometheus.

Copy/paste the description from
https://github.com/prometheus/prometheus/pull/13393#issuecomment-1891787986

Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
---
 cmd/prometheus/main.go |  1 +
 tsdb/db.go             | 40 ++++++++++++++++++++++++++--------------
 2 files changed, 27 insertions(+), 14 deletions(-)

diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go
index a8bd2f2b3..95a581a74 100644
--- a/cmd/prometheus/main.go
+++ b/cmd/prometheus/main.go
@@ -1646,6 +1646,7 @@ func (opts tsdbOptions) ToTSDBOptions() tsdb.Options {
 		EnableMemorySnapshotOnShutdown: opts.EnableMemorySnapshotOnShutdown,
 		EnableNativeHistograms:         opts.EnableNativeHistograms,
 		OutOfOrderTimeWindow:           opts.OutOfOrderTimeWindow,
+		EnableOverlappingCompaction:    true,
 	}
 }
 
diff --git a/tsdb/db.go b/tsdb/db.go
index bdeb3faee..2436fab2a 100644
--- a/tsdb/db.go
+++ b/tsdb/db.go
@@ -70,19 +70,20 @@ var ErrNotReady = errors.New("TSDB not ready")
 // millisecond precision timestamps.
 func DefaultOptions() *Options {
 	return &Options{
-		WALSegmentSize:            wlog.DefaultSegmentSize,
-		MaxBlockChunkSegmentSize:  chunks.DefaultChunkSegmentSize,
-		RetentionDuration:         int64(15 * 24 * time.Hour / time.Millisecond),
-		MinBlockDuration:          DefaultBlockDuration,
-		MaxBlockDuration:          DefaultBlockDuration,
-		NoLockfile:                false,
-		SamplesPerChunk:           DefaultSamplesPerChunk,
-		WALCompression:            wlog.CompressionNone,
-		StripeSize:                DefaultStripeSize,
-		HeadChunksWriteBufferSize: chunks.DefaultWriteBufferSize,
-		IsolationDisabled:         defaultIsolationDisabled,
-		HeadChunksWriteQueueSize:  chunks.DefaultWriteQueueSize,
-		OutOfOrderCapMax:          DefaultOutOfOrderCapMax,
+		WALSegmentSize:              wlog.DefaultSegmentSize,
+		MaxBlockChunkSegmentSize:    chunks.DefaultChunkSegmentSize,
+		RetentionDuration:           int64(15 * 24 * time.Hour / time.Millisecond),
+		MinBlockDuration:            DefaultBlockDuration,
+		MaxBlockDuration:            DefaultBlockDuration,
+		NoLockfile:                  false,
+		SamplesPerChunk:             DefaultSamplesPerChunk,
+		WALCompression:              wlog.CompressionNone,
+		StripeSize:                  DefaultStripeSize,
+		HeadChunksWriteBufferSize:   chunks.DefaultWriteBufferSize,
+		IsolationDisabled:           defaultIsolationDisabled,
+		HeadChunksWriteQueueSize:    chunks.DefaultWriteQueueSize,
+		OutOfOrderCapMax:            DefaultOutOfOrderCapMax,
+		EnableOverlappingCompaction: true,
 	}
 }
 
@@ -177,6 +178,14 @@ type Options struct {
 	// OutOfOrderCapMax is maximum capacity for OOO chunks (in samples).
 	// If it is <=0, the default value is assumed.
 	OutOfOrderCapMax int64
+
+	// Compaction of overlapping blocks are allowed if EnableOverlappingCompaction is true.
+	// This is an optional flag for overlapping blocks.
+	// The reason why this flag exists is because there are various users of the TSDB
+	// that do not want vertical compaction happening on ingest time. Instead,
+	// they'd rather keep overlapping blocks and let another component do the overlapping compaction later.
+	// For Prometheus, this will always be true.
+	EnableOverlappingCompaction bool
 }
 
 type BlocksToDeleteFunc func(blocks []*Block) map[ulid.ULID]struct{}
@@ -816,7 +825,10 @@ func open(dir string, l log.Logger, r prometheus.Registerer, opts *Options, rngs
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())
-	db.compactor, err = NewLeveledCompactorWithChunkSize(ctx, r, l, rngs, db.chunkPool, opts.MaxBlockChunkSegmentSize, nil)
+	db.compactor, err = NewLeveledCompactorWithOptions(ctx, r, l, rngs, db.chunkPool, LeveledCompactorOptions{
+		MaxBlockChunkSegmentSize:    opts.MaxBlockChunkSegmentSize,
+		EnableOverlappingCompaction: opts.EnableOverlappingCompaction,
+	})
 	if err != nil {
 		cancel()
 		return nil, fmt.Errorf("create leveled compactor: %w", err)

From 72a8f1084b29a826367049fe51362f00c86d8849 Mon Sep 17 00:00:00 2001
From: zenador <zenador@users.noreply.github.com>
Date: Tue, 16 Jan 2024 16:54:16 +0800
Subject: [PATCH 084/115] Restore more efficient version of
 NewPossibleNonCounterInfo annotation (#13022)

Restore more efficient version of NewPossibleNonCounterInfo annotation

Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>

---------

Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
---
 promql/engine.go | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/promql/engine.go b/promql/engine.go
index 2ea37dae6..68d0962cd 100644
--- a/promql/engine.go
+++ b/promql/engine.go
@@ -24,6 +24,7 @@ import (
 	"runtime"
 	"sort"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -1543,6 +1544,18 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, annotations.Annotatio
 				}
 			}
 			ev.samplesStats.UpdatePeak(ev.currentSamples)
+
+			if e.Func.Name == "rate" || e.Func.Name == "increase" {
+				samples := inMatrix[0]
+				metricName := samples.Metric.Get(labels.MetricName)
+				if metricName != "" && len(samples.Floats) > 0 &&
+					!strings.HasSuffix(metricName, "_total") &&
+					!strings.HasSuffix(metricName, "_sum") &&
+					!strings.HasSuffix(metricName, "_count") &&
+					!strings.HasSuffix(metricName, "_bucket") {
+					warnings.Add(annotations.NewPossibleNonCounterInfo(metricName, e.Args[0].PositionRange()))
+				}
+			}
 		}
 		ev.samplesStats.UpdatePeak(ev.currentSamples)
 

From 7852a7c5162a7ebd21a8766e3a75d7de1ac1cf3c Mon Sep 17 00:00:00 2001
From: Marco Pracucci <marco@pracucci.com>
Date: Tue, 16 Jan 2024 12:00:53 +0100
Subject: [PATCH 085/115] Fix regressions introduced by #13242

Signed-off-by: Marco Pracucci <marco@pracucci.com>
---
 tsdb/compact.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tsdb/compact.go b/tsdb/compact.go
index 511c518d9..efe3306c7 100644
--- a/tsdb/compact.go
+++ b/tsdb/compact.go
@@ -188,12 +188,12 @@ func NewLeveledCompactorWithOptions(ctx context.Context, r prometheus.Registerer
 	if mergeFunc == nil {
 		mergeFunc = storage.NewCompactingChunkSeriesMerger(storage.ChainedSeriesMerge)
 	}
-	var maxBlockChunkSegmentSize int64
-	if opts.MaxBlockChunkSegmentSize == 0 {
+	maxBlockChunkSegmentSize := opts.MaxBlockChunkSegmentSize
+	if maxBlockChunkSegmentSize == 0 {
 		maxBlockChunkSegmentSize = chunks.DefaultChunkSegmentSize
 	}
-	var pe index.PostingsEncoder
-	if opts.PE == nil {
+	pe := opts.PE
+	if pe == nil {
 		pe = index.EncodePostingsRaw
 	}
 	return &LeveledCompactor{

From 1fa131ee033b7bced52d818c67351a8021765921 Mon Sep 17 00:00:00 2001
From: tyltr <tylitianrui@126.com>
Date: Tue, 16 Jan 2024 19:06:58 +0800
Subject: [PATCH 086/115] fix slice copy in 1.20 (#13389)

The slices package is added to the standard library in Go 1.21;
we need to import from the exp area to maintain compatibility with Go 1.20.

Signed-off-by: tyltr <tylitianrui@126.com>
---
 model/histogram/generic_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/model/histogram/generic_test.go b/model/histogram/generic_test.go
index b4d6585a8..6a22b6f19 100644
--- a/model/histogram/generic_test.go
+++ b/model/histogram/generic_test.go
@@ -15,10 +15,10 @@ package histogram
 
 import (
 	"math"
-	"slices"
 	"testing"
 
 	"github.com/stretchr/testify/require"
+	"golang.org/x/exp/slices"
 )
 
 func TestGetBound(t *testing.T) {

From 73bbeaafbf7cd0d7dfb0ac80a49a6f5c3685f224 Mon Sep 17 00:00:00 2001
From: Ted Robertson <10043369+tredondo@users.noreply.github.com>
Date: Tue, 16 Jan 2024 06:09:51 -0500
Subject: [PATCH 087/115] Docs: Query Basics: link to rate (#10538)

Co-authored-by: Julien Pivotto <roidelapluie@o11y.eu>
---
 docs/querying/basics.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/querying/basics.md b/docs/querying/basics.md
index fa0d44a69..dac503785 100644
--- a/docs/querying/basics.md
+++ b/docs/querying/basics.md
@@ -212,8 +212,8 @@ While the following would be *incorrect*:
 
     sum(http_requests_total{method="GET"}) offset 5m // INVALID.
 
-The same works for range vectors. This returns the 5-minute rate that
-`http_requests_total` had a week ago:
+The same works for range vectors. This returns the 5-minute [rate](./functions.md#rate)
+that `http_requests_total` had a week ago:
 
     rate(http_requests_total[5m] offset 1w)
 

From 2d01e566954067ac902e83eb4daca7340a492825 Mon Sep 17 00:00:00 2001
From: machine424 <ayoubmrini424@gmail.com>
Date: Tue, 16 Jan 2024 12:06:21 +0100
Subject: [PATCH 088/115] chore(kubernetes): check preconditions earlier and
 avoid unnecessary checks or iterations

Signed-off-by: machine424 <ayoubmrini424@gmail.com>
---
 discovery/kubernetes/endpoints.go     | 28 ++++++++++++++-------------
 discovery/kubernetes/endpointslice.go | 28 ++++++++++++++-------------
 2 files changed, 30 insertions(+), 26 deletions(-)

diff --git a/discovery/kubernetes/endpoints.go b/discovery/kubernetes/endpoints.go
index 512d77552..d8c9689ca 100644
--- a/discovery/kubernetes/endpoints.go
+++ b/discovery/kubernetes/endpoints.go
@@ -369,6 +369,11 @@ func (e *Endpoints) buildEndpoints(eps *apiv1.Endpoints) *targetgroup.Group {
 	// For all seen pods, check all container ports. If they were not covered
 	// by one of the service endpoints, generate targets for them.
 	for _, pe := range seenPods {
+		// PodIP can be empty when a pod is starting or has been evicted.
+		if len(pe.pod.Status.PodIP) == 0 {
+			continue
+		}
+
 		for _, c := range pe.pod.Spec.Containers {
 			for _, cport := range c.Ports {
 				hasSeenPort := func() bool {
@@ -383,21 +388,18 @@ func (e *Endpoints) buildEndpoints(eps *apiv1.Endpoints) *targetgroup.Group {
 					continue
 				}
 
-				// PodIP can be empty when a pod is starting or has been evicted.
-				if len(pe.pod.Status.PodIP) != 0 {
-					a := net.JoinHostPort(pe.pod.Status.PodIP, strconv.FormatUint(uint64(cport.ContainerPort), 10))
-					ports := strconv.FormatUint(uint64(cport.ContainerPort), 10)
+				a := net.JoinHostPort(pe.pod.Status.PodIP, strconv.FormatUint(uint64(cport.ContainerPort), 10))
+				ports := strconv.FormatUint(uint64(cport.ContainerPort), 10)
 
-					target := model.LabelSet{
-						model.AddressLabel:            lv(a),
-						podContainerNameLabel:         lv(c.Name),
-						podContainerImageLabel:        lv(c.Image),
-						podContainerPortNameLabel:     lv(cport.Name),
-						podContainerPortNumberLabel:   lv(ports),
-						podContainerPortProtocolLabel: lv(string(cport.Protocol)),
-					}
-					tg.Targets = append(tg.Targets, target.Merge(podLabels(pe.pod)))
+				target := model.LabelSet{
+					model.AddressLabel:            lv(a),
+					podContainerNameLabel:         lv(c.Name),
+					podContainerImageLabel:        lv(c.Image),
+					podContainerPortNameLabel:     lv(cport.Name),
+					podContainerPortNumberLabel:   lv(ports),
+					podContainerPortProtocolLabel: lv(string(cport.Protocol)),
 				}
+				tg.Targets = append(tg.Targets, target.Merge(podLabels(pe.pod)))
 			}
 		}
 	}
diff --git a/discovery/kubernetes/endpointslice.go b/discovery/kubernetes/endpointslice.go
index 21095df4a..f977ce262 100644
--- a/discovery/kubernetes/endpointslice.go
+++ b/discovery/kubernetes/endpointslice.go
@@ -405,6 +405,11 @@ func (e *EndpointSlice) buildEndpointSlice(eps endpointSliceAdaptor) *targetgrou
 	// For all seen pods, check all container ports. If they were not covered
 	// by one of the service endpoints, generate targets for them.
 	for _, pe := range seenPods {
+		// PodIP can be empty when a pod is starting or has been evicted.
+		if len(pe.pod.Status.PodIP) == 0 {
+			continue
+		}
+
 		for _, c := range pe.pod.Spec.Containers {
 			for _, cport := range c.Ports {
 				hasSeenPort := func() bool {
@@ -422,21 +427,18 @@ func (e *EndpointSlice) buildEndpointSlice(eps endpointSliceAdaptor) *targetgrou
 					continue
 				}
 
-				// PodIP can be empty when a pod is starting or has been evicted.
-				if len(pe.pod.Status.PodIP) != 0 {
-					a := net.JoinHostPort(pe.pod.Status.PodIP, strconv.FormatUint(uint64(cport.ContainerPort), 10))
-					ports := strconv.FormatUint(uint64(cport.ContainerPort), 10)
+				a := net.JoinHostPort(pe.pod.Status.PodIP, strconv.FormatUint(uint64(cport.ContainerPort), 10))
+				ports := strconv.FormatUint(uint64(cport.ContainerPort), 10)
 
-					target := model.LabelSet{
-						model.AddressLabel:            lv(a),
-						podContainerNameLabel:         lv(c.Name),
-						podContainerImageLabel:        lv(c.Image),
-						podContainerPortNameLabel:     lv(cport.Name),
-						podContainerPortNumberLabel:   lv(ports),
-						podContainerPortProtocolLabel: lv(string(cport.Protocol)),
-					}
-					tg.Targets = append(tg.Targets, target.Merge(podLabels(pe.pod)))
+				target := model.LabelSet{
+					model.AddressLabel:            lv(a),
+					podContainerNameLabel:         lv(c.Name),
+					podContainerImageLabel:        lv(c.Image),
+					podContainerPortNameLabel:     lv(cport.Name),
+					podContainerPortNumberLabel:   lv(ports),
+					podContainerPortProtocolLabel: lv(string(cport.Protocol)),
 				}
+				tg.Targets = append(tg.Targets, target.Merge(podLabels(pe.pod)))
 			}
 		}
 	}

From cf9e93403651e792a858f4134dd60cc3490f54e0 Mon Sep 17 00:00:00 2001
From: Ted Robertson <10043369+tredondo@users.noreply.github.com>
Date: Tue, 16 Jan 2024 06:13:52 -0500
Subject: [PATCH 089/115] Examples: link to `rate` for new users (#10535)

* Examples: link to `rate` for new users

Signed-off-by: Ted Robertson 10043369+tredondo@users.noreply.github.com
Co-authored-by: Bryan Boreham <bjboreham@gmail.com>
---
 docs/querying/examples.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/querying/examples.md b/docs/querying/examples.md
index a4fd373bf..957c2f5f9 100644
--- a/docs/querying/examples.md
+++ b/docs/querying/examples.md
@@ -39,7 +39,7 @@ To select all HTTP status codes except 4xx ones, you could run:
 
 ## Subquery
 
-Return the 5-minute rate of the `http_requests_total` metric for the past 30 minutes, with a resolution of 1 minute.
+Return the 5-minute [rate](./functions.md#rate) of the `http_requests_total` metric for the past 30 minutes, with a resolution of 1 minute.
 
     rate(http_requests_total[5m])[30m:1m]
 

From a6b35ff304ca22c8676d8da244eb837f04f64169 Mon Sep 17 00:00:00 2001
From: Ivan Babrou <github@ivan.computer>
Date: Tue, 16 Jan 2024 16:34:09 -0800
Subject: [PATCH 090/115] promql: use natural sort in sort_by_label and
 sort_by_label_desc (#13411)

These functions are intended for humans, as robots can already sort the results
however they please. Humans like things sorted "naturally":

* https://blog.codinghorror.com/sorting-for-humans-natural-sort-order/

A similar thing has been done to Grafana, which is also used by humans:

* https://github.com/grafana/grafana/pull/78024
* https://github.com/grafana/grafana/pull/78494

Signed-off-by: Ivan Babrou <github@ivan.computer>
---
 docs/querying/functions.md     |  4 ++++
 go.mod                         |  1 +
 go.sum                         |  2 ++
 promql/functions.go            | 29 ++++++++++++++--------------
 promql/testdata/functions.test | 35 ++++++++++++++++++++++++++++++++++
 5 files changed, 56 insertions(+), 15 deletions(-)

diff --git a/docs/querying/functions.md b/docs/querying/functions.md
index dda88fccd..607e16e82 100644
--- a/docs/querying/functions.md
+++ b/docs/querying/functions.md
@@ -594,6 +594,8 @@ Same as `sort`, but sorts in descending order.
 
 Please note that the sort by label functions only affect the results of instant queries, as range query results always have a fixed output ordering.
 
+This function uses [natural sort order](https://en.wikipedia.org/wiki/Natural_sort_order).
+
 ## `sort_by_label_desc()`
 
 **This function has to be enabled via the [feature flag](../feature_flags/) `--enable-feature=promql-experimental-functions`.**
@@ -602,6 +604,8 @@ Same as `sort_by_label`, but sorts in descending order.
 
 Please note that the sort by label functions only affect the results of instant queries, as range query results always have a fixed output ordering.
 
+This function uses [natural sort order](https://en.wikipedia.org/wiki/Natural_sort_order).
+
 ## `sqrt()`
 
 `sqrt(v instant-vector)` calculates the square root of all elements in `v`.
diff --git a/go.mod b/go.mod
index 428b6edec..e7598db42 100644
--- a/go.mod
+++ b/go.mod
@@ -19,6 +19,7 @@ require (
 	github.com/edsrzf/mmap-go v1.1.0
 	github.com/envoyproxy/go-control-plane v0.11.1
 	github.com/envoyproxy/protoc-gen-validate v1.0.2
+	github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb
 	github.com/fsnotify/fsnotify v1.7.0
 	github.com/go-kit/log v0.2.1
 	github.com/go-logfmt/logfmt v0.6.0
diff --git a/go.sum b/go.sum
index 8ce4efb5c..fa21d23e5 100644
--- a/go.sum
+++ b/go.sum
@@ -166,6 +166,8 @@ github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBF
 github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE=
 github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=
 github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb h1:IT4JYU7k4ikYg1SCxNI1/Tieq/NFvh6dzLdgi7eu0tM=
+github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb/go.mod h1:bH6Xx7IW64qjjJq8M2u4dxNaBiDfKK+z/3eGDpXEQhc=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
diff --git a/promql/functions.go b/promql/functions.go
index 407a11b50..cb3be0aef 100644
--- a/promql/functions.go
+++ b/promql/functions.go
@@ -21,6 +21,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/facette/natsort"
 	"github.com/grafana/regexp"
 	"github.com/prometheus/common/model"
 	"golang.org/x/exp/slices"
@@ -380,15 +381,16 @@ func funcSortByLabel(vals []parser.Value, args parser.Expressions, enh *EvalNode
 		for _, label := range labels {
 			lv1 := a.Metric.Get(label)
 			lv2 := b.Metric.Get(label)
-			// 0 if a == b, -1 if a < b, and +1 if a > b.
-			switch strings.Compare(lv1, lv2) {
-			case -1:
-				return -1
-			case +1:
-				return +1
-			default:
+
+			if lv1 == lv2 {
 				continue
 			}
+
+			if natsort.Compare(lv1, lv2) {
+				return -1
+			}
+
+			return +1
 		}
 
 		return 0
@@ -409,19 +411,16 @@ func funcSortByLabelDesc(vals []parser.Value, args parser.Expressions, enh *Eval
 		for _, label := range labels {
 			lv1 := a.Metric.Get(label)
 			lv2 := b.Metric.Get(label)
-			// If label values are the same, continue to the next label
+
 			if lv1 == lv2 {
 				continue
 			}
-			// 0 if a == b, -1 if a < b, and +1 if a > b.
-			switch strings.Compare(lv1, lv2) {
-			case -1:
+
+			if natsort.Compare(lv1, lv2) {
 				return +1
-			case +1:
-				return -1
-			default:
-				continue
 			}
+
+			return -1
 		}
 
 		return 0
diff --git a/promql/testdata/functions.test b/promql/testdata/functions.test
index b4547886a..c40a6272b 100644
--- a/promql/testdata/functions.test
+++ b/promql/testdata/functions.test
@@ -482,6 +482,19 @@ load 5m
 	http_requests{job="app-server", instance="0", group="canary"}		0+70x10
 	http_requests{job="app-server", instance="1", group="canary"}		0+80x10
 	http_requests{job="api-server", instance="2", group="production"}	0+10x10
+	cpu_time_total{job="cpu", cpu="0"} 0+10x10
+	cpu_time_total{job="cpu", cpu="1"} 0+10x10
+	cpu_time_total{job="cpu", cpu="2"} 0+10x10
+	cpu_time_total{job="cpu", cpu="3"} 0+10x10
+	cpu_time_total{job="cpu", cpu="10"} 0+10x10
+	cpu_time_total{job="cpu", cpu="11"} 0+10x10
+	cpu_time_total{job="cpu", cpu="12"} 0+10x10
+	cpu_time_total{job="cpu", cpu="20"} 0+10x10
+	cpu_time_total{job="cpu", cpu="21"} 0+10x10
+	cpu_time_total{job="cpu", cpu="100"} 0+10x10
+	node_uname_info{job="node_exporter", instance="4m600", release="1.2.3"} 0+10x10
+	node_uname_info{job="node_exporter", instance="4m5", release="1.11.3"} 0+10x10
+	node_uname_info{job="node_exporter", instance="4m1000", release="1.111.3"} 0+10x10
 
 eval_ordered instant at 50m sort_by_label(http_requests, "instance")
 	http_requests{group="production", instance="0", job="api-server"} 100
@@ -579,6 +592,28 @@ eval_ordered instant at 50m sort_by_label_desc(http_requests, "instance", "group
 	http_requests{group="canary", instance="0", job="app-server"} 700
 	http_requests{group="canary", instance="0", job="api-server"} 300
 
+eval_ordered instant at 50m sort_by_label(cpu_time_total, "cpu")
+	cpu_time_total{job="cpu", cpu="0"} 100
+	cpu_time_total{job="cpu", cpu="1"} 100
+	cpu_time_total{job="cpu", cpu="2"} 100
+	cpu_time_total{job="cpu", cpu="3"} 100
+	cpu_time_total{job="cpu", cpu="10"} 100
+	cpu_time_total{job="cpu", cpu="11"} 100
+	cpu_time_total{job="cpu", cpu="12"} 100
+	cpu_time_total{job="cpu", cpu="20"} 100
+	cpu_time_total{job="cpu", cpu="21"} 100
+	cpu_time_total{job="cpu", cpu="100"} 100
+
+eval_ordered instant at 50m sort_by_label(node_uname_info, "instance")
+	node_uname_info{job="node_exporter", instance="4m5", release="1.11.3"} 100
+	node_uname_info{job="node_exporter", instance="4m600", release="1.2.3"} 100
+	node_uname_info{job="node_exporter", instance="4m1000", release="1.111.3"} 100
+
+eval_ordered instant at 50m sort_by_label(node_uname_info, "release")
+	node_uname_info{job="node_exporter", instance="4m600", release="1.2.3"} 100
+	node_uname_info{job="node_exporter", instance="4m5", release="1.11.3"} 100
+	node_uname_info{job="node_exporter", instance="4m1000", release="1.111.3"} 100
+
 # Tests for holt_winters
 clear
 

From 8598150f48e92fc8065e5a9a983a178c66ef0ba0 Mon Sep 17 00:00:00 2001
From: Arve Knudsen <arve.knudsen@gmail.com>
Date: Wed, 17 Jan 2024 12:51:56 +0100
Subject: [PATCH 091/115] TestLabelValuesWithMatchers: Add test case

Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
---
 tsdb/block_test.go | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/tsdb/block_test.go b/tsdb/block_test.go
index 7858e6b0c..c155b4451 100644
--- a/tsdb/block_test.go
+++ b/tsdb/block_test.go
@@ -235,6 +235,13 @@ func TestLabelValuesWithMatchers(t *testing.T) {
 	require.NoError(t, err)
 	defer func() { require.NoError(t, indexReader.Close()) }()
 
+	var uniqueWithout30s []string
+	for i := 0; i < 100; i++ {
+		if i/10 != 3 {
+			uniqueWithout30s = append(uniqueWithout30s, fmt.Sprintf("value%d", i))
+		}
+	}
+	sort.Strings(uniqueWithout30s)
 	testCases := []struct {
 		name           string
 		labelName      string
@@ -261,6 +268,14 @@ func TestLabelValuesWithMatchers(t *testing.T) {
 			labelName:      "tens",
 			matchers:       []*labels.Matcher{labels.MustNewMatcher(labels.MatchNotEqual, "unique", "")},
 			expectedValues: []string{"value0", "value1", "value2", "value3", "value4", "value5", "value6", "value7", "value8", "value9"},
+		}, {
+			name:      "get unique IDs based on tens not being equal to a certain value, while not empty",
+			labelName: "unique",
+			matchers: []*labels.Matcher{
+				labels.MustNewMatcher(labels.MatchNotEqual, "tens", "value3"),
+				labels.MustNewMatcher(labels.MatchNotEqual, "tens", ""),
+			},
+			expectedValues: uniqueWithout30s,
 		},
 	}
 

From f97fa2736c89947aa91a201f23775a4a47916d65 Mon Sep 17 00:00:00 2001
From: tyltr <tylitianrui@126.com>
Date: Wed, 17 Jan 2024 22:26:32 +0800
Subject: [PATCH 092/115] remove  obsolete build tag

Signed-off-by: tyltr <tylitianrui@126.com>
---
 cmd/prometheus/main_unix_test.go   | 1 -
 config/config_default_test.go      | 1 -
 plugins/generate.go                | 1 -
 promql/fuzz.go                     | 1 -
 promql/fuzz_test.go                | 1 -
 scripts/tools.go                   | 1 -
 tsdb/chunks/head_chunks_other.go   | 1 -
 tsdb/fileutil/dir_unix.go          | 1 -
 tsdb/fileutil/dir_windows.go       | 1 -
 tsdb/fileutil/flock_js.go          | 1 -
 tsdb/fileutil/flock_solaris.go     | 1 -
 tsdb/fileutil/flock_unix.go        | 1 -
 tsdb/fileutil/mmap_386.go          | 1 -
 tsdb/fileutil/mmap_amd64.go        | 1 -
 tsdb/fileutil/mmap_arm64.go        | 1 -
 tsdb/fileutil/mmap_js.go           | 1 -
 tsdb/fileutil/mmap_unix.go         | 1 -
 tsdb/fileutil/preallocate_other.go | 1 -
 tsdb/fileutil/sync.go              | 1 -
 tsdb/fileutil/sync_darwin.go       | 1 -
 tsdb/fileutil/sync_linux.go        | 1 -
 tsdb/goversion/goversion.go        | 1 -
 tsdb/wal_test.go                   | 1 -
 util/runtime/limits_default.go     | 1 -
 util/runtime/limits_windows.go     | 1 -
 util/runtime/statfs.go             | 1 -
 util/runtime/statfs_default.go     | 1 -
 util/runtime/statfs_linux_386.go   | 1 -
 util/runtime/statfs_uint32.go      | 1 -
 util/runtime/uname_default.go      | 1 -
 util/runtime/vmlimits_default.go   | 1 -
 util/runtime/vmlimits_openbsd.go   | 1 -
 web/ui/assets_embed.go             | 1 -
 web/ui/ui.go                       | 1 -
 34 files changed, 34 deletions(-)

diff --git a/cmd/prometheus/main_unix_test.go b/cmd/prometheus/main_unix_test.go
index 7224e25d7..417d062d6 100644
--- a/cmd/prometheus/main_unix_test.go
+++ b/cmd/prometheus/main_unix_test.go
@@ -12,7 +12,6 @@
 // limitations under the License.
 //
 //go:build !windows
-// +build !windows
 
 package main
 
diff --git a/config/config_default_test.go b/config/config_default_test.go
index f5333f4c8..26623590d 100644
--- a/config/config_default_test.go
+++ b/config/config_default_test.go
@@ -12,7 +12,6 @@
 // limitations under the License.
 
 //go:build !windows
-// +build !windows
 
 package config
 
diff --git a/plugins/generate.go b/plugins/generate.go
index 32a32d890..3db6de4ab 100644
--- a/plugins/generate.go
+++ b/plugins/generate.go
@@ -12,7 +12,6 @@
 // limitations under the License.
 
 //go:build plugins
-// +build plugins
 
 package main
 
diff --git a/promql/fuzz.go b/promql/fuzz.go
index aff6eb15b..d2e5925eb 100644
--- a/promql/fuzz.go
+++ b/promql/fuzz.go
@@ -13,7 +13,6 @@
 
 // Only build when go-fuzz is in use
 //go:build gofuzz
-// +build gofuzz
 
 package promql
 
diff --git a/promql/fuzz_test.go b/promql/fuzz_test.go
index 08a04d98b..1f0bbaa66 100644
--- a/promql/fuzz_test.go
+++ b/promql/fuzz_test.go
@@ -13,7 +13,6 @@
 
 // Only build when go-fuzz is in use
 //go:build gofuzz
-// +build gofuzz
 
 package promql
 
diff --git a/scripts/tools.go b/scripts/tools.go
index 52b464911..e1dd80232 100644
--- a/scripts/tools.go
+++ b/scripts/tools.go
@@ -12,7 +12,6 @@
 // limitations under the License.
 
 //go:build tools
-// +build tools
 
 // Package tools tracks dependencies for tools that are required to generate the protobuf code.
 // See https://github.com/golang/go/issues/25922
diff --git a/tsdb/chunks/head_chunks_other.go b/tsdb/chunks/head_chunks_other.go
index 3d00fddc3..f30c5e55e 100644
--- a/tsdb/chunks/head_chunks_other.go
+++ b/tsdb/chunks/head_chunks_other.go
@@ -12,7 +12,6 @@
 // limitations under the License.
 
 //go:build !windows
-// +build !windows
 
 package chunks
 
diff --git a/tsdb/fileutil/dir_unix.go b/tsdb/fileutil/dir_unix.go
index db310014e..2afb2aeab 100644
--- a/tsdb/fileutil/dir_unix.go
+++ b/tsdb/fileutil/dir_unix.go
@@ -13,7 +13,6 @@
 // limitations under the License.
 
 //go:build !windows
-// +build !windows
 
 package fileutil
 
diff --git a/tsdb/fileutil/dir_windows.go b/tsdb/fileutil/dir_windows.go
index f67cc74b3..307077ebc 100644
--- a/tsdb/fileutil/dir_windows.go
+++ b/tsdb/fileutil/dir_windows.go
@@ -13,7 +13,6 @@
 // limitations under the License.
 
 //go:build windows
-// +build windows
 
 package fileutil
 
diff --git a/tsdb/fileutil/flock_js.go b/tsdb/fileutil/flock_js.go
index 697d2e542..6029cdf4d 100644
--- a/tsdb/fileutil/flock_js.go
+++ b/tsdb/fileutil/flock_js.go
@@ -12,7 +12,6 @@
 // limitations under the License.
 
 //go:build js
-// +build js
 
 package fileutil
 
diff --git a/tsdb/fileutil/flock_solaris.go b/tsdb/fileutil/flock_solaris.go
index 21be384d3..8ca919f3b 100644
--- a/tsdb/fileutil/flock_solaris.go
+++ b/tsdb/fileutil/flock_solaris.go
@@ -12,7 +12,6 @@
 // limitations under the License.
 
 //go:build solaris
-// +build solaris
 
 package fileutil
 
diff --git a/tsdb/fileutil/flock_unix.go b/tsdb/fileutil/flock_unix.go
index 9637f073b..25de0ffb2 100644
--- a/tsdb/fileutil/flock_unix.go
+++ b/tsdb/fileutil/flock_unix.go
@@ -12,7 +12,6 @@
 // limitations under the License.
 
 //go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd
-// +build darwin dragonfly freebsd linux netbsd openbsd
 
 package fileutil
 
diff --git a/tsdb/fileutil/mmap_386.go b/tsdb/fileutil/mmap_386.go
index 591d722d5..85c0cce09 100644
--- a/tsdb/fileutil/mmap_386.go
+++ b/tsdb/fileutil/mmap_386.go
@@ -12,7 +12,6 @@
 // limitations under the License.
 
 //go:build windows
-// +build windows
 
 package fileutil
 
diff --git a/tsdb/fileutil/mmap_amd64.go b/tsdb/fileutil/mmap_amd64.go
index 4c9534e93..71fc568bd 100644
--- a/tsdb/fileutil/mmap_amd64.go
+++ b/tsdb/fileutil/mmap_amd64.go
@@ -12,7 +12,6 @@
 // limitations under the License.
 
 //go:build windows
-// +build windows
 
 package fileutil
 
diff --git a/tsdb/fileutil/mmap_arm64.go b/tsdb/fileutil/mmap_arm64.go
index 4c9534e93..71fc568bd 100644
--- a/tsdb/fileutil/mmap_arm64.go
+++ b/tsdb/fileutil/mmap_arm64.go
@@ -12,7 +12,6 @@
 // limitations under the License.
 
 //go:build windows
-// +build windows
 
 package fileutil
 
diff --git a/tsdb/fileutil/mmap_js.go b/tsdb/fileutil/mmap_js.go
index 0c666e8bf..f29106fc1 100644
--- a/tsdb/fileutil/mmap_js.go
+++ b/tsdb/fileutil/mmap_js.go
@@ -12,7 +12,6 @@
 // limitations under the License.
 
 //go:build js
-// +build js
 
 package fileutil
 
diff --git a/tsdb/fileutil/mmap_unix.go b/tsdb/fileutil/mmap_unix.go
index 1fd7f48ff..3d15e1a8c 100644
--- a/tsdb/fileutil/mmap_unix.go
+++ b/tsdb/fileutil/mmap_unix.go
@@ -12,7 +12,6 @@
 // limitations under the License.
 
 //go:build !windows && !plan9 && !js
-// +build !windows,!plan9,!js
 
 package fileutil
 
diff --git a/tsdb/fileutil/preallocate_other.go b/tsdb/fileutil/preallocate_other.go
index 2c46dd490..e7fd937a4 100644
--- a/tsdb/fileutil/preallocate_other.go
+++ b/tsdb/fileutil/preallocate_other.go
@@ -13,7 +13,6 @@
 // limitations under the License.
 
 //go:build !linux && !darwin
-// +build !linux,!darwin
 
 package fileutil
 
diff --git a/tsdb/fileutil/sync.go b/tsdb/fileutil/sync.go
index 5bd18b458..e1a4a7fd3 100644
--- a/tsdb/fileutil/sync.go
+++ b/tsdb/fileutil/sync.go
@@ -13,7 +13,6 @@
 // limitations under the License.
 
 //go:build !linux && !darwin
-// +build !linux,!darwin
 
 package fileutil
 
diff --git a/tsdb/fileutil/sync_darwin.go b/tsdb/fileutil/sync_darwin.go
index 155b94459..d698b896a 100644
--- a/tsdb/fileutil/sync_darwin.go
+++ b/tsdb/fileutil/sync_darwin.go
@@ -13,7 +13,6 @@
 // limitations under the License.
 
 //go:build darwin
-// +build darwin
 
 package fileutil
 
diff --git a/tsdb/fileutil/sync_linux.go b/tsdb/fileutil/sync_linux.go
index 1c2091ea3..2b4c620bb 100644
--- a/tsdb/fileutil/sync_linux.go
+++ b/tsdb/fileutil/sync_linux.go
@@ -13,7 +13,6 @@
 // limitations under the License.
 
 //go:build linux
-// +build linux
 
 package fileutil
 
diff --git a/tsdb/goversion/goversion.go b/tsdb/goversion/goversion.go
index 02da2639d..ec23d25f2 100644
--- a/tsdb/goversion/goversion.go
+++ b/tsdb/goversion/goversion.go
@@ -12,7 +12,6 @@
 // limitations under the License.
 
 //go:build go1.12
-// +build go1.12
 
 // Package goversion enforces the go version supported by the tsdb module.
 package goversion
diff --git a/tsdb/wal_test.go b/tsdb/wal_test.go
index 964763d7f..9004f5093 100644
--- a/tsdb/wal_test.go
+++ b/tsdb/wal_test.go
@@ -12,7 +12,6 @@
 // limitations under the License.
 
 //go:build !windows
-// +build !windows
 
 package tsdb
 
diff --git a/util/runtime/limits_default.go b/util/runtime/limits_default.go
index 40af5f14d..0126adb1a 100644
--- a/util/runtime/limits_default.go
+++ b/util/runtime/limits_default.go
@@ -12,7 +12,6 @@
 // limitations under the License.
 
 //go:build !windows
-// +build !windows
 
 package runtime
 
diff --git a/util/runtime/limits_windows.go b/util/runtime/limits_windows.go
index ce514949e..ce82d31e6 100644
--- a/util/runtime/limits_windows.go
+++ b/util/runtime/limits_windows.go
@@ -12,7 +12,6 @@
 // limitations under the License.
 
 //go:build windows
-// +build windows
 
 package runtime
 
diff --git a/util/runtime/statfs.go b/util/runtime/statfs.go
index 2e5a624b6..66bedb5ea 100644
--- a/util/runtime/statfs.go
+++ b/util/runtime/statfs.go
@@ -12,7 +12,6 @@
 // limitations under the License.
 
 //go:build openbsd || windows || netbsd || solaris
-// +build openbsd windows netbsd solaris
 
 package runtime
 
diff --git a/util/runtime/statfs_default.go b/util/runtime/statfs_default.go
index b6f837c47..8d283c1e4 100644
--- a/util/runtime/statfs_default.go
+++ b/util/runtime/statfs_default.go
@@ -12,7 +12,6 @@
 // limitations under the License.
 
 //go:build !windows && !openbsd && !netbsd && !solaris && !386
-// +build !windows,!openbsd,!netbsd,!solaris,!386
 
 package runtime
 
diff --git a/util/runtime/statfs_linux_386.go b/util/runtime/statfs_linux_386.go
index 7494a0adf..a003b2eff 100644
--- a/util/runtime/statfs_linux_386.go
+++ b/util/runtime/statfs_linux_386.go
@@ -12,7 +12,6 @@
 // limitations under the License.
 
 //go:build linux && 386
-// +build linux,386
 
 package runtime
 
diff --git a/util/runtime/statfs_uint32.go b/util/runtime/statfs_uint32.go
index 72d670a39..fbf994ea6 100644
--- a/util/runtime/statfs_uint32.go
+++ b/util/runtime/statfs_uint32.go
@@ -12,7 +12,6 @@
 // limitations under the License.
 
 //go:build (386 && darwin) || (386 && freebsd)
-// +build 386,darwin 386,freebsd
 
 package runtime
 
diff --git a/util/runtime/uname_default.go b/util/runtime/uname_default.go
index 161337121..0052dbab4 100644
--- a/util/runtime/uname_default.go
+++ b/util/runtime/uname_default.go
@@ -12,7 +12,6 @@
 // limitations under the License.
 
 //go:build !linux
-// +build !linux
 
 package runtime
 
diff --git a/util/runtime/vmlimits_default.go b/util/runtime/vmlimits_default.go
index bf35b48fe..aef434106 100644
--- a/util/runtime/vmlimits_default.go
+++ b/util/runtime/vmlimits_default.go
@@ -12,7 +12,6 @@
 // limitations under the License.
 
 //go:build !windows && !openbsd
-// +build !windows,!openbsd
 
 package runtime
 
diff --git a/util/runtime/vmlimits_openbsd.go b/util/runtime/vmlimits_openbsd.go
index e33ab4802..b40f06588 100644
--- a/util/runtime/vmlimits_openbsd.go
+++ b/util/runtime/vmlimits_openbsd.go
@@ -12,7 +12,6 @@
 // limitations under the License.
 
 //go:build openbsd
-// +build openbsd
 
 package runtime
 
diff --git a/web/ui/assets_embed.go b/web/ui/assets_embed.go
index f10d5c0c2..a5f8f5ddf 100644
--- a/web/ui/assets_embed.go
+++ b/web/ui/assets_embed.go
@@ -12,7 +12,6 @@
 // limitations under the License.
 
 //go:build builtinassets
-// +build builtinassets
 
 package ui
 
diff --git a/web/ui/ui.go b/web/ui/ui.go
index 99ae34e5d..2585951d4 100644
--- a/web/ui/ui.go
+++ b/web/ui/ui.go
@@ -12,7 +12,6 @@
 // limitations under the License.
 
 //go:build !builtinassets
-// +build !builtinassets
 
 package ui
 

From c0a66e5540d949fa7a739eac8261478b5424458a Mon Sep 17 00:00:00 2001
From: Israel Blancas <iblancasa@gmail.com>
Date: Wed, 17 Jan 2024 16:43:19 +0100
Subject: [PATCH 093/115] Upgrade some golang dependencies for resty 2.11

Signed-off-by: Israel Blancas <iblancasa@gmail.com>
---
 go.mod | 12 ++++++------
 go.sum | 24 ++++++++++++------------
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/go.mod b/go.mod
index e7598db42..17dd1eedd 100644
--- a/go.mod
+++ b/go.mod
@@ -39,7 +39,7 @@ require (
 	github.com/json-iterator/go v1.1.12
 	github.com/klauspost/compress v1.17.4
 	github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b
-	github.com/linode/linodego v1.25.0
+	github.com/linode/linodego v1.27.1
 	github.com/miekg/dns v1.1.57
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f
@@ -72,10 +72,10 @@ require (
 	go.uber.org/goleak v1.3.0
 	go.uber.org/multierr v1.11.0
 	golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb
-	golang.org/x/net v0.19.0
+	golang.org/x/net v0.20.0
 	golang.org/x/oauth2 v0.15.0
 	golang.org/x/sync v0.5.0
-	golang.org/x/sys v0.15.0
+	golang.org/x/sys v0.16.0
 	golang.org/x/time v0.5.0
 	golang.org/x/tools v0.16.0
 	google.golang.org/api v0.153.0
@@ -123,7 +123,7 @@ require (
 	github.com/go-openapi/spec v0.20.9 // indirect
 	github.com/go-openapi/swag v0.22.4 // indirect
 	github.com/go-openapi/validate v0.22.1 // indirect
-	github.com/go-resty/resty/v2 v2.10.0 // indirect
+	github.com/go-resty/resty/v2 v2.11.0 // indirect
 	github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
 	github.com/golang/glog v1.1.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@@ -177,9 +177,9 @@ require (
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/otel/metric v1.21.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
-	golang.org/x/crypto v0.16.0 // indirect
+	golang.org/x/crypto v0.18.0 // indirect
 	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/term v0.15.0 // indirect
+	golang.org/x/term v0.16.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f // indirect
diff --git a/go.sum b/go.sum
index fa21d23e5..5aaf586b6 100644
--- a/go.sum
+++ b/go.sum
@@ -240,8 +240,8 @@ github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogB
 github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
 github.com/go-openapi/validate v0.22.1 h1:G+c2ub6q47kfX1sOBLwIQwzBVt8qmOAARyo/9Fqs9NU=
 github.com/go-openapi/validate v0.22.1/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUriMu5/zuPSQ1hg=
-github.com/go-resty/resty/v2 v2.10.0 h1:Qla4W/+TMmv0fOeeRqzEpXPLfTUnR5HZ1+lGs+CkiCo=
-github.com/go-resty/resty/v2 v2.10.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A=
+github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8=
+github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
@@ -503,8 +503,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
-github.com/linode/linodego v1.25.0 h1:zYMz0lTasD503jBu3tSRhzEmXHQN1zptCw5o71ibyyU=
-github.com/linode/linodego v1.25.0/go.mod h1:BMZI0pMM/YGjBis7pIXDPbcgYfCZLH0/UvzqtsGtG1c=
+github.com/linode/linodego v1.27.1 h1:KoQm5g2fppw8qIClJqUEL0yKH0+f+7te3Mewagb5QKE=
+github.com/linode/linodego v1.27.1/go.mod h1:5oAsx+uinHtVo6U77nXXXtox7MWzUW6aEkTOKXxA9uo=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
@@ -819,8 +819,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
-golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
-golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
+golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -902,8 +902,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
-golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
-golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
+golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
+golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -995,15 +995,15 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
-golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
-golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
+golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE=
+golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

From df2a0ecf3badc8fd84a14bcb7b13eb6b8893f670 Mon Sep 17 00:00:00 2001
From: Ziqi Zhao <zhaoziqi9146@gmail.com>
Date: Wed, 17 Jan 2024 23:58:54 +0800
Subject: [PATCH 094/115] Native Histograms: support
 `native_histogram_min_bucket_factor` in scrape_config (#13222)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Native Histograms: support native_histogram_min_bucket_factor in scrape_config

---------

Signed-off-by: Ziqi Zhao <zhaoziqi9146@gmail.com>
Signed-off-by: Björn Rabenstein <github@rabenste.in>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
Co-authored-by: Björn Rabenstein <github@rabenste.in>
---
 config/config.go                    |  7 +-
 docs/configuration/configuration.md | 40 ++++++++++++
 scrape/scrape.go                    | 34 +++++++++-
 scrape/scrape_test.go               | 99 +++++++++++++++++++++++++++--
 scrape/target.go                    | 29 +++++++++
 scrape/target_test.go               | 61 ++++++++++++++++++
 6 files changed, 260 insertions(+), 10 deletions(-)

diff --git a/config/config.go b/config/config.go
index c544d7e74..7fa03a144 100644
--- a/config/config.go
+++ b/config/config.go
@@ -610,9 +610,12 @@ type ScrapeConfig struct {
 	// More than this label value length post metric-relabeling will cause the
 	// scrape to fail. 0 means no limit.
 	LabelValueLengthLimit uint `yaml:"label_value_length_limit,omitempty"`
-	// More than this many buckets in a native histogram will cause the scrape to
-	// fail.
+	// If there are more than this many buckets in a native histogram,
+	// buckets will be merged to stay within the limit.
 	NativeHistogramBucketLimit uint `yaml:"native_histogram_bucket_limit,omitempty"`
+	// If the growth factor of one bucket to the next is smaller than this,
+	// buckets will be merged to increase the factor sufficiently.
+	NativeHistogramMinBucketFactor float64 `yaml:"native_histogram_min_bucket_factor,omitempty"`
 	// Keep no more than this many dropped targets per job.
 	// 0 means no limit.
 	KeepDroppedTargets uint `yaml:"keep_dropped_targets,omitempty"`
diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md
index 3a7b71a2f..e62d61b09 100644
--- a/docs/configuration/configuration.md
+++ b/docs/configuration/configuration.md
@@ -451,6 +451,46 @@ metric_relabel_configs:
 # native histogram. If this is exceeded, the entire scrape will be treated as
 # failed. 0 means no limit.
 [ native_histogram_bucket_limit: <int> | default = 0 ]
+
+# Lower limit for the growth factor of one bucket to the next in each native
+# histogram. The resolution of a histogram with a lower growth factor will be
+# reduced until it is within the limit.
+# To set an upper limit for the schema (equivalent to "scale" in OTel's
+# exponential histograms), use the following factor limits:
+# 
+# +----------------------------+----------------------------+
+# |        growth factor       | resulting schema AKA scale |
+# +----------------------------+----------------------------+
+# |          65536             |             -4             |
+# +----------------------------+----------------------------+
+# |            256             |             -3             |
+# +----------------------------+----------------------------+
+# |             16             |             -2             |
+# +----------------------------+----------------------------+
+# |              4             |             -1             |
+# +----------------------------+----------------------------+
+# |              2             |              0             |
+# +----------------------------+----------------------------+
+# |              1.4           |              1             |
+# +----------------------------+----------------------------+
+# |              1.1           |              2             |
+# +----------------------------+----------------------------+
+# |              1.09          |              3             |
+# +----------------------------+----------------------------+
+# |              1.04          |              4             |
+# +----------------------------+----------------------------+
+# |              1.02          |              5             |
+# +----------------------------+----------------------------+
+# |              1.01          |              6             |
+# +----------------------------+----------------------------+
+# |              1.005         |              7             |
+# +----------------------------+----------------------------+
+# |              1.002         |              8             |
+# +----------------------------+----------------------------+
+# 
+# 0 results in the smallest supported factor (which is currently ~1.0027 or
+# schema 8, but might change in the future).
+[ native_histogram_min_bucket_factor: <float> | default = 0 ]
 ```
 
 Where `<job_name>` must be unique across all scrape configurations.
diff --git a/scrape/scrape.go b/scrape/scrape.go
index 2518912a7..dfa945852 100644
--- a/scrape/scrape.go
+++ b/scrape/scrape.go
@@ -99,6 +99,7 @@ type scrapeLoopOptions struct {
 	scraper                  scraper
 	sampleLimit              int
 	bucketLimit              int
+	maxSchema                int32
 	labelLimits              *labelLimits
 	honorLabels              bool
 	honorTimestamps          bool
@@ -165,6 +166,7 @@ func newScrapePool(cfg *config.ScrapeConfig, app storage.Appendable, offsetSeed
 			opts.enableCompression,
 			opts.sampleLimit,
 			opts.bucketLimit,
+			opts.maxSchema,
 			opts.labelLimits,
 			opts.interval,
 			opts.timeout,
@@ -270,6 +272,7 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) error {
 		bodySizeLimit = int64(sp.config.BodySizeLimit)
 		sampleLimit   = int(sp.config.SampleLimit)
 		bucketLimit   = int(sp.config.NativeHistogramBucketLimit)
+		maxSchema     = pickSchema(sp.config.NativeHistogramMinBucketFactor)
 		labelLimits   = &labelLimits{
 			labelLimit:            int(sp.config.LabelLimit),
 			labelNameLengthLimit:  int(sp.config.LabelNameLengthLimit),
@@ -310,6 +313,7 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) error {
 				scraper:                  s,
 				sampleLimit:              sampleLimit,
 				bucketLimit:              bucketLimit,
+				maxSchema:                maxSchema,
 				labelLimits:              labelLimits,
 				honorLabels:              honorLabels,
 				honorTimestamps:          honorTimestamps,
@@ -613,7 +617,7 @@ func mutateReportSampleLabels(lset labels.Labels, target *Target) labels.Labels
 }
 
 // appender returns an appender for ingested samples from the target.
-func appender(app storage.Appender, sampleLimit, bucketLimit int) storage.Appender {
+func appender(app storage.Appender, sampleLimit, bucketLimit int, maxSchema int32) storage.Appender {
 	app = &timeLimitAppender{
 		Appender: app,
 		maxTime:  timestamp.FromTime(time.Now().Add(maxAheadTime)),
@@ -633,6 +637,14 @@ func appender(app storage.Appender, sampleLimit, bucketLimit int) storage.Append
 			limit:    bucketLimit,
 		}
 	}
+
+	if maxSchema < nativeHistogramMaxSchema {
+		app = &maxSchemaAppender{
+			Appender:  app,
+			maxSchema: maxSchema,
+		}
+	}
+
 	return app
 }
 
@@ -786,6 +798,7 @@ type scrapeLoop struct {
 	forcedErrMtx             sync.Mutex
 	sampleLimit              int
 	bucketLimit              int
+	maxSchema                int32
 	labelLimits              *labelLimits
 	interval                 time.Duration
 	timeout                  time.Duration
@@ -1078,6 +1091,7 @@ func newScrapeLoop(ctx context.Context,
 	enableCompression bool,
 	sampleLimit int,
 	bucketLimit int,
+	maxSchema int32,
 	labelLimits *labelLimits,
 	interval time.Duration,
 	timeout time.Duration,
@@ -1128,6 +1142,7 @@ func newScrapeLoop(ctx context.Context,
 		enableCompression:        enableCompression,
 		sampleLimit:              sampleLimit,
 		bucketLimit:              bucketLimit,
+		maxSchema:                maxSchema,
 		labelLimits:              labelLimits,
 		interval:                 interval,
 		timeout:                  timeout,
@@ -1458,7 +1473,7 @@ func (sl *scrapeLoop) append(app storage.Appender, b []byte, contentType string,
 	}
 
 	// Take an appender with limits.
-	app = appender(app, sl.sampleLimit, sl.bucketLimit)
+	app = appender(app, sl.sampleLimit, sl.bucketLimit, sl.maxSchema)
 
 	defer func() {
 		if err != nil {
@@ -1906,3 +1921,18 @@ func TargetFromContext(ctx context.Context) (*Target, bool) {
 	t, ok := ctx.Value(ctxKeyTarget).(*Target)
 	return t, ok
 }
+
+func pickSchema(bucketFactor float64) int32 {
+	if bucketFactor <= 1 {
+		bucketFactor = 1.00271
+	}
+	floor := math.Floor(-math.Log2(math.Log2(bucketFactor)))
+	switch {
+	case floor >= float64(nativeHistogramMaxSchema):
+		return nativeHistogramMaxSchema
+	case floor <= float64(nativeHistogramMinSchema):
+		return nativeHistogramMinSchema
+	default:
+		return int32(floor)
+	}
+}
diff --git a/scrape/scrape_test.go b/scrape/scrape_test.go
index 1a416eeb6..b4bfdc6e4 100644
--- a/scrape/scrape_test.go
+++ b/scrape/scrape_test.go
@@ -513,7 +513,7 @@ func TestScrapePoolAppender(t *testing.T) {
 	appl, ok := loop.(*scrapeLoop)
 	require.True(t, ok, "Expected scrapeLoop but got %T", loop)
 
-	wrapped := appender(appl.appender(context.Background()), 0, 0)
+	wrapped := appender(appl.appender(context.Background()), 0, 0, nativeHistogramMaxSchema)
 
 	tl, ok := wrapped.(*timeLimitAppender)
 	require.True(t, ok, "Expected timeLimitAppender but got %T", wrapped)
@@ -529,7 +529,7 @@ func TestScrapePoolAppender(t *testing.T) {
 	appl, ok = loop.(*scrapeLoop)
 	require.True(t, ok, "Expected scrapeLoop but got %T", loop)
 
-	wrapped = appender(appl.appender(context.Background()), sampleLimit, 0)
+	wrapped = appender(appl.appender(context.Background()), sampleLimit, 0, nativeHistogramMaxSchema)
 
 	sl, ok := wrapped.(*limitAppender)
 	require.True(t, ok, "Expected limitAppender but got %T", wrapped)
@@ -540,7 +540,7 @@ func TestScrapePoolAppender(t *testing.T) {
 	_, ok = tl.Appender.(nopAppender)
 	require.True(t, ok, "Expected base appender but got %T", tl.Appender)
 
-	wrapped = appender(appl.appender(context.Background()), sampleLimit, 100)
+	wrapped = appender(appl.appender(context.Background()), sampleLimit, 100, nativeHistogramMaxSchema)
 
 	bl, ok := wrapped.(*bucketLimitAppender)
 	require.True(t, ok, "Expected bucketLimitAppender but got %T", wrapped)
@@ -553,6 +553,23 @@ func TestScrapePoolAppender(t *testing.T) {
 
 	_, ok = tl.Appender.(nopAppender)
 	require.True(t, ok, "Expected base appender but got %T", tl.Appender)
+
+	wrapped = appender(appl.appender(context.Background()), sampleLimit, 100, 0)
+
+	ml, ok := wrapped.(*maxSchemaAppender)
+	require.True(t, ok, "Expected maxSchemaAppender but got %T", wrapped)
+
+	bl, ok = ml.Appender.(*bucketLimitAppender)
+	require.True(t, ok, "Expected bucketLimitAppender but got %T", wrapped)
+
+	sl, ok = bl.Appender.(*limitAppender)
+	require.True(t, ok, "Expected limitAppender but got %T", bl)
+
+	tl, ok = sl.Appender.(*timeLimitAppender)
+	require.True(t, ok, "Expected timeLimitAppender but got %T", sl.Appender)
+
+	_, ok = tl.Appender.(nopAppender)
+	require.True(t, ok, "Expected base appender but got %T", tl.Appender)
 }
 
 func TestScrapePoolRaces(t *testing.T) {
@@ -653,7 +670,7 @@ func newBasicScrapeLoop(t testing.TB, ctx context.Context, scraper scraper, app
 		true,
 		false,
 		true,
-		0, 0,
+		0, 0, nativeHistogramMaxSchema,
 		nil,
 		interval,
 		time.Hour,
@@ -796,7 +813,7 @@ func TestScrapeLoopRun(t *testing.T) {
 		true,
 		false,
 		true,
-		0, 0,
+		0, 0, nativeHistogramMaxSchema,
 		nil,
 		time.Second,
 		time.Hour,
@@ -942,7 +959,7 @@ func TestScrapeLoopMetadata(t *testing.T) {
 		true,
 		false,
 		true,
-		0, 0,
+		0, 0, nativeHistogramMaxSchema,
 		nil,
 		0,
 		0,
@@ -3465,3 +3482,73 @@ func TestScrapeLoopCompression(t *testing.T) {
 		})
 	}
 }
+
+func TestPickSchema(t *testing.T) {
+	tcs := []struct {
+		factor float64
+		schema int32
+	}{
+		{
+			factor: 65536,
+			schema: -4,
+		},
+		{
+			factor: 256,
+			schema: -3,
+		},
+		{
+			factor: 16,
+			schema: -2,
+		},
+		{
+			factor: 4,
+			schema: -1,
+		},
+		{
+			factor: 2,
+			schema: 0,
+		},
+		{
+			factor: 1.4,
+			schema: 1,
+		},
+		{
+			factor: 1.1,
+			schema: 2,
+		},
+		{
+			factor: 1.09,
+			schema: 3,
+		},
+		{
+			factor: 1.04,
+			schema: 4,
+		},
+		{
+			factor: 1.02,
+			schema: 5,
+		},
+		{
+			factor: 1.01,
+			schema: 6,
+		},
+		{
+			factor: 1.005,
+			schema: 7,
+		},
+		{
+			factor: 1.002,
+			schema: 8,
+		},
+		// The default value of native_histogram_min_bucket_factor
+		{
+			factor: 0,
+			schema: 8,
+		},
+	}
+
+	for _, tc := range tcs {
+		schema := pickSchema(tc.factor)
+		require.Equal(t, tc.schema, schema)
+	}
+}
diff --git a/scrape/target.go b/scrape/target.go
index 0605f5349..c9287f818 100644
--- a/scrape/target.go
+++ b/scrape/target.go
@@ -387,6 +387,35 @@ func (app *bucketLimitAppender) AppendHistogram(ref storage.SeriesRef, lset labe
 	return ref, nil
 }
 
+const (
+	nativeHistogramMaxSchema int32 = 8
+	nativeHistogramMinSchema int32 = -4
+)
+
+type maxSchemaAppender struct {
+	storage.Appender
+
+	maxSchema int32
+}
+
+func (app *maxSchemaAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) {
+	if h != nil {
+		if h.Schema > app.maxSchema {
+			h = h.ReduceResolution(app.maxSchema)
+		}
+	}
+	if fh != nil {
+		if fh.Schema > app.maxSchema {
+			fh = fh.ReduceResolution(app.maxSchema)
+		}
+	}
+	ref, err := app.Appender.AppendHistogram(ref, lset, t, h, fh)
+	if err != nil {
+		return 0, err
+	}
+	return ref, nil
+}
+
 // PopulateLabels builds a label set from the given label set and scrape configuration.
 // It returns a label set before relabeling was applied as the second return value.
 // Returns the original discovered label set found before relabelling was applied if the target is dropped during relabeling.
diff --git a/scrape/target_test.go b/scrape/target_test.go
index f37c75a76..dac502a80 100644
--- a/scrape/target_test.go
+++ b/scrape/target_test.go
@@ -590,3 +590,64 @@ func TestBucketLimitAppender(t *testing.T) {
 		}
 	}
 }
+
+func TestMaxSchemaAppender(t *testing.T) {
+	example := histogram.Histogram{
+		Schema:        0,
+		Count:         21,
+		Sum:           33,
+		ZeroThreshold: 0.001,
+		ZeroCount:     3,
+		PositiveSpans: []histogram.Span{
+			{Offset: 0, Length: 3},
+		},
+		PositiveBuckets: []int64{3, 0, 0},
+		NegativeSpans: []histogram.Span{
+			{Offset: 0, Length: 3},
+		},
+		NegativeBuckets: []int64{3, 0, 0},
+	}
+
+	cases := []struct {
+		h            histogram.Histogram
+		maxSchema    int32
+		expectSchema int32
+	}{
+		{
+			h:            example,
+			maxSchema:    -1,
+			expectSchema: -1,
+		},
+		{
+			h:            example,
+			maxSchema:    0,
+			expectSchema: 0,
+		},
+	}
+
+	resApp := &collectResultAppender{}
+
+	for _, c := range cases {
+		for _, floatHisto := range []bool{true, false} {
+			t.Run(fmt.Sprintf("floatHistogram=%t", floatHisto), func(t *testing.T) {
+				app := &maxSchemaAppender{Appender: resApp, maxSchema: c.maxSchema}
+				ts := int64(10 * time.Minute / time.Millisecond)
+				lbls := labels.FromStrings("__name__", "sparse_histogram_series")
+				var err error
+				if floatHisto {
+					fh := c.h.Copy().ToFloat(nil)
+					_, err = app.AppendHistogram(0, lbls, ts, nil, fh)
+					require.Equal(t, c.expectSchema, fh.Schema)
+					require.NoError(t, err)
+
+				} else {
+					h := c.h.Copy()
+					_, err = app.AppendHistogram(0, lbls, ts, h, nil)
+					require.Equal(t, c.expectSchema, h.Schema)
+					require.NoError(t, err)
+				}
+				require.NoError(t, app.Commit())
+			})
+		}
+	}
+}

From a3ddfbd1ee3ff29821764a700cef52c7bab67236 Mon Sep 17 00:00:00 2001
From: zenador <zenador@users.noreply.github.com>
Date: Thu, 18 Jan 2024 00:06:35 +0800
Subject: [PATCH 095/115] Add warnings for histogramRate applied with isCounter
 not matching counter/gauge histogram (#13392)

Add warnings for histogramRate applied with isCounter not matching counter/gauge histogram

---------

Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
---
 model/histogram/float_histogram.go |  4 ++--
 promql/functions.go                |  8 +++++++-
 util/annotations/annotations.go    | 20 ++++++++++++++++++++
 3 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/model/histogram/float_histogram.go b/model/histogram/float_histogram.go
index 64c7b797a..bf89f2a47 100644
--- a/model/histogram/float_histogram.go
+++ b/model/histogram/float_histogram.go
@@ -469,8 +469,8 @@ func (h *FloatHistogram) DetectReset(previous *FloatHistogram) bool {
 	// is a counter reset or not.
 	// We do the same if the CounterResetHint is GaugeType, which should not happen, but PromQL still
 	// allows the user to apply functions to gauge histograms that are only meant for counter histograms.
-	// In this case, we treat the gauge histograms as a counter histograms
-	// (and we plan to return a warning about it to the user).
+	// In this case, we treat the gauge histograms as counter histograms. A warning should be returned
+	// to the user in this case.
 	if h.Count < previous.Count {
 		return true
 	}
diff --git a/promql/functions.go b/promql/functions.go
index cb3be0aef..9ce577de9 100644
--- a/promql/functions.go
+++ b/promql/functions.go
@@ -187,6 +187,8 @@ func histogramRate(points []HPoint, isCounter bool, metricName string, pos posra
 		minSchema = last.Schema
 	}
 
+	var annos annotations.Annotations
+
 	// First iteration to find out two things:
 	// - What's the smallest relevant schema?
 	// - Are all data points histograms?
@@ -197,10 +199,12 @@ func histogramRate(points []HPoint, isCounter bool, metricName string, pos posra
 		if curr == nil {
 			return nil, annotations.New().Add(annotations.NewMixedFloatsHistogramsWarning(metricName, pos))
 		}
-		// TODO(trevorwhitney): Check if isCounter is consistent with curr.CounterResetHint.
 		if !isCounter {
 			continue
 		}
+		if curr.CounterResetHint == histogram.GaugeType {
+			annos.Add(annotations.NewNativeHistogramNotCounterWarning(metricName, pos))
+		}
 		if curr.Schema < minSchema {
 			minSchema = curr.Schema
 		}
@@ -218,6 +222,8 @@ func histogramRate(points []HPoint, isCounter bool, metricName string, pos posra
 			}
 			prev = curr
 		}
+	} else if points[0].H.CounterResetHint != histogram.GaugeType || points[len(points)-1].H.CounterResetHint != histogram.GaugeType {
+		annos.Add(annotations.NewNativeHistogramNotGaugeWarning(metricName, pos))
 	}
 
 	h.CounterResetHint = histogram.GaugeType
diff --git a/util/annotations/annotations.go b/util/annotations/annotations.go
index 180d40843..29dafeb2e 100644
--- a/util/annotations/annotations.go
+++ b/util/annotations/annotations.go
@@ -107,6 +107,8 @@ var (
 	BadBucketLabelWarning               = fmt.Errorf("%w: bucket label %q is missing or has a malformed value", PromQLWarning, model.BucketLabel)
 	MixedFloatsHistogramsWarning        = fmt.Errorf("%w: encountered a mix of histograms and floats for metric name", PromQLWarning)
 	MixedClassicNativeHistogramsWarning = fmt.Errorf("%w: vector contains a mix of classic and native histograms for metric name", PromQLWarning)
+	NativeHistogramNotCounterWarning    = fmt.Errorf("%w: this native histogram metric is not a counter:", PromQLWarning)
+	NativeHistogramNotGaugeWarning      = fmt.Errorf("%w: this native histogram metric is not a gauge:", PromQLWarning)
 
 	PossibleNonCounterInfo                  = fmt.Errorf("%w: metric might not be a counter, name does not end in _total/_sum/_count/_bucket:", PromQLInfo)
 	HistogramQuantileForcedMonotonicityInfo = fmt.Errorf("%w: input to histogram_quantile needed to be fixed for monotonicity (see https://prometheus.io/docs/prometheus/latest/querying/functions/#histogram_quantile) for metric name", PromQLInfo)
@@ -166,6 +168,24 @@ func NewMixedClassicNativeHistogramsWarning(metricName string, pos posrange.Posi
 	}
 }
 
+// NewNativeHistogramNotCounterWarning is used when histogramRate is called
+// with isCounter set to true on a gauge histogram.
+func NewNativeHistogramNotCounterWarning(metricName string, pos posrange.PositionRange) error {
+	return annoErr{
+		PositionRange: pos,
+		Err:           fmt.Errorf("%w %q", NativeHistogramNotCounterWarning, metricName),
+	}
+}
+
+// NewNativeHistogramNotGaugeWarning is used when histogramRate is called
+// with isCounter set to false on a counter histogram.
+func NewNativeHistogramNotGaugeWarning(metricName string, pos posrange.PositionRange) error {
+	return annoErr{
+		PositionRange: pos,
+		Err:           fmt.Errorf("%w %q", NativeHistogramNotGaugeWarning, metricName),
+	}
+}
+
 // NewPossibleNonCounterInfo is used when a named counter metric with only float samples does not
 // have the suffixes _total, _sum, _count, or _bucket.
 func NewPossibleNonCounterInfo(metricName string, pos posrange.PositionRange) error {

From aee6896c4704f767ea8b0cdc42b26f23aa9ed545 Mon Sep 17 00:00:00 2001
From: Goutham <gouthamve@gmail.com>
Date: Thu, 18 Jan 2024 15:32:06 +0530
Subject: [PATCH 096/115] Minor fixes to otlp vendor update script

Signed-off-by: Goutham <gouthamve@gmail.com>
---
 .../remote/otlptranslator/prometheus/normalize_label.go   | 3 +++
 .../remote/otlptranslator/prometheus/normalize_name.go    | 2 ++
 storage/remote/otlptranslator/prometheus/unit_to_ucum.go  | 2 ++
 .../remote/otlptranslator/prometheusremotewrite/helper.go | 2 +-
 .../otlptranslator/prometheusremotewrite/histograms.go    | 2 +-
 .../prometheusremotewrite/metrics_to_prw.go               | 2 +-
 .../prometheusremotewrite/number_data_points.go           | 2 +-
 storage/remote/otlptranslator/update-copy.sh              | 8 +++++++-
 8 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/storage/remote/otlptranslator/prometheus/normalize_label.go b/storage/remote/otlptranslator/prometheus/normalize_label.go
index b44ba869f..c02800c8b 100644
--- a/storage/remote/otlptranslator/prometheus/normalize_label.go
+++ b/storage/remote/otlptranslator/prometheus/normalize_label.go
@@ -1,3 +1,5 @@
+// DO NOT EDIT. COPIED AS-IS. SEE ../README.md
+
 // Copyright The OpenTelemetry Authors
 // SPDX-License-Identifier: Apache-2.0
 
@@ -25,6 +27,7 @@ var dropSanitizationGate = featuregate.GlobalRegistry().MustRegister(
 //
 // Exception is made for double-underscores which are allowed
 func NormalizeLabel(label string) string {
+
 	// Trivial case
 	if len(label) == 0 {
 		return label
diff --git a/storage/remote/otlptranslator/prometheus/normalize_name.go b/storage/remote/otlptranslator/prometheus/normalize_name.go
index 72fc04cea..7ae233a18 100644
--- a/storage/remote/otlptranslator/prometheus/normalize_name.go
+++ b/storage/remote/otlptranslator/prometheus/normalize_name.go
@@ -1,3 +1,5 @@
+// DO NOT EDIT. COPIED AS-IS. SEE ../README.md
+
 // Copyright The OpenTelemetry Authors
 // SPDX-License-Identifier: Apache-2.0
 
diff --git a/storage/remote/otlptranslator/prometheus/unit_to_ucum.go b/storage/remote/otlptranslator/prometheus/unit_to_ucum.go
index b2f2c4f3a..4a72e683b 100644
--- a/storage/remote/otlptranslator/prometheus/unit_to_ucum.go
+++ b/storage/remote/otlptranslator/prometheus/unit_to_ucum.go
@@ -1,3 +1,5 @@
+// DO NOT EDIT. COPIED AS-IS. SEE ../README.md
+
 // Copyright The OpenTelemetry Authors
 // SPDX-License-Identifier: Apache-2.0
 
diff --git a/storage/remote/otlptranslator/prometheusremotewrite/helper.go b/storage/remote/otlptranslator/prometheusremotewrite/helper.go
index 49ad5672b..09b39d29d 100644
--- a/storage/remote/otlptranslator/prometheusremotewrite/helper.go
+++ b/storage/remote/otlptranslator/prometheusremotewrite/helper.go
@@ -1,4 +1,4 @@
-// DO NOT EDIT. COPIED AS-IS. SEE README.md
+// DO NOT EDIT. COPIED AS-IS. SEE ../README.md
 
 // Copyright The OpenTelemetry Authors
 // SPDX-License-Identifier: Apache-2.0
diff --git a/storage/remote/otlptranslator/prometheusremotewrite/histograms.go b/storage/remote/otlptranslator/prometheusremotewrite/histograms.go
index 3c7494a6b..0a528f607 100644
--- a/storage/remote/otlptranslator/prometheusremotewrite/histograms.go
+++ b/storage/remote/otlptranslator/prometheusremotewrite/histograms.go
@@ -1,4 +1,4 @@
-// DO NOT EDIT. COPIED AS-IS. SEE README.md
+// DO NOT EDIT. COPIED AS-IS. SEE ../README.md
 
 // Copyright The OpenTelemetry Authors
 // SPDX-License-Identifier: Apache-2.0
diff --git a/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go b/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go
index 6a5a65604..6e8f42fae 100644
--- a/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go
+++ b/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go
@@ -1,4 +1,4 @@
-// DO NOT EDIT. COPIED AS-IS. SEE README.md
+// DO NOT EDIT. COPIED AS-IS. SEE ../README.md
 
 // Copyright The OpenTelemetry Authors
 // SPDX-License-Identifier: Apache-2.0
diff --git a/storage/remote/otlptranslator/prometheusremotewrite/number_data_points.go b/storage/remote/otlptranslator/prometheusremotewrite/number_data_points.go
index c8e59694b..85684ad6d 100644
--- a/storage/remote/otlptranslator/prometheusremotewrite/number_data_points.go
+++ b/storage/remote/otlptranslator/prometheusremotewrite/number_data_points.go
@@ -1,4 +1,4 @@
-// DO NOT EDIT. COPIED AS-IS. SEE README.md
+// DO NOT EDIT. COPIED AS-IS. SEE ../README.md
 
 // Copyright The OpenTelemetry Authors
 // SPDX-License-Identifier: Apache-2.0
diff --git a/storage/remote/otlptranslator/update-copy.sh b/storage/remote/otlptranslator/update-copy.sh
index 36ad0cc35..44b3a3245 100755
--- a/storage/remote/otlptranslator/update-copy.sh
+++ b/storage/remote/otlptranslator/update-copy.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
+set -xe
 
 OTEL_VERSION=v0.88.0
 
@@ -6,11 +7,16 @@ git clone https://github.com/open-telemetry/opentelemetry-collector-contrib ./tm
 cd ./tmp
 git checkout $OTEL_VERSION
 cd ..
+
 rm -rf ./prometheusremotewrite/*
 cp -r ./tmp/pkg/translator/prometheusremotewrite/*.go ./prometheusremotewrite
+rm -rf ./prometheusremotewrite/*_test.go
+
+rm -rf ./prometheus/*
 cp -r ./tmp/pkg/translator/prometheus/*.go ./prometheus
 rm -rf ./prometheus/*_test.go
+
 rm -rf ./tmp
 
 sed -i '' 's#github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus#github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus#g' ./prometheusremotewrite/*.go
-sed -i '' '1s#^#// DO NOT EDIT. COPIED AS-IS. SEE README.md\n\n#g' ./prometheusremotewrite/*.go
+sed -i '' '1s#^#// DO NOT EDIT. COPIED AS-IS. SEE ../README.md\n\n#g' ./prometheusremotewrite/*.go ./prometheus/*.go

From 110ba0a2c7e80aab13e5a215fce3ea3917b1e116 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 18 Jan 2024 10:58:49 +0000
Subject: [PATCH 097/115] build(deps): bump
 github.com/hetznercloud/hcloud-go/v2

Bumps [github.com/hetznercloud/hcloud-go/v2](https://github.com/hetznercloud/hcloud-go) from 2.4.0 to 2.6.0.
- [Release notes](https://github.com/hetznercloud/hcloud-go/releases)
- [Changelog](https://github.com/hetznercloud/hcloud-go/blob/main/CHANGELOG.md)
- [Commits](https://github.com/hetznercloud/hcloud-go/compare/v2.4.0...v2.6.0)

---
updated-dependencies:
- dependency-name: github.com/hetznercloud/hcloud-go/v2
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 17dd1eedd..c34d3db2e 100644
--- a/go.mod
+++ b/go.mod
@@ -34,7 +34,7 @@ require (
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0
 	github.com/hashicorp/consul/api v1.26.1
 	github.com/hashicorp/nomad/api v0.0.0-20230721134942-515895c7690c // Not upgrading this for now due to https://github.com/prometheus/prometheus/pull/13255#issuecomment-1845237409
-	github.com/hetznercloud/hcloud-go/v2 v2.4.0
+	github.com/hetznercloud/hcloud-go/v2 v2.6.0
 	github.com/ionos-cloud/sdk-go/v6 v6.1.10
 	github.com/json-iterator/go v1.1.12
 	github.com/klauspost/compress v1.17.4
diff --git a/go.sum b/go.sum
index 5aaf586b6..3593e39da 100644
--- a/go.sum
+++ b/go.sum
@@ -441,8 +441,8 @@ github.com/hashicorp/nomad/api v0.0.0-20230721134942-515895c7690c/go.mod h1:O23q
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY=
 github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4=
-github.com/hetznercloud/hcloud-go/v2 v2.4.0 h1:MqlAE+w125PLvJRCpAJmEwrIxoVdUdOyuFUhE/Ukbok=
-github.com/hetznercloud/hcloud-go/v2 v2.4.0/go.mod h1:l7fA5xsncFBzQTyw29/dw5Yr88yEGKKdc6BHf24ONS0=
+github.com/hetznercloud/hcloud-go/v2 v2.6.0 h1:RJOA2hHZ7rD1pScA4O1NF6qhkHyUdbbxjHgFNot8928=
+github.com/hetznercloud/hcloud-go/v2 v2.6.0/go.mod h1:4J1cSE57+g0WS93IiHLV7ubTHItcp+awzeBp5bM9mfA=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=

From 102fd8cc88be0bfce83ec6dfc527be2894ae8524 Mon Sep 17 00:00:00 2001
From: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
Date: Thu, 18 Jan 2024 18:49:16 +0400
Subject: [PATCH 098/115] Enhanced visibility for `promtool test rules` with
 JSON colored formatting (#13342)

* Added diff flag for unit test to improvise readability & debugging

Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>

* Removed blank spaces

Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>

* Fixed linting error

Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>

* Added cli flags to documentation

Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>

* Revert unrrelated linting fixes

Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>

* Fixed review suggestions

Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>

* Cleanup

Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>

* Updated flag description

Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>

* Updated flag description

Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>

---------

Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
---
 cmd/promtool/main.go          |  2 ++
 cmd/promtool/unittest.go      | 52 ++++++++++++++++++++++++++++++-----
 cmd/promtool/unittest_test.go |  4 +--
 docs/command-line/promtool.md |  7 +++--
 go.mod                        |  1 +
 go.sum                        |  2 ++
 6 files changed, 56 insertions(+), 12 deletions(-)

diff --git a/cmd/promtool/main.go b/cmd/promtool/main.go
index 5a7f21d3b..0332c33ea 100644
--- a/cmd/promtool/main.go
+++ b/cmd/promtool/main.go
@@ -210,6 +210,7 @@ func main() {
 		"test-rule-file",
 		"The unit test file.",
 	).Required().ExistingFiles()
+	testRulesDiff := testRulesCmd.Flag("diff", "[Experimental] Print colored differential output between expected & received output.").Default("false").Bool()
 
 	defaultDBPath := "data/"
 	tsdbCmd := app.Command("tsdb", "Run tsdb commands.")
@@ -375,6 +376,7 @@ func main() {
 				EnableNegativeOffset: true,
 			},
 			*testRulesRun,
+			*testRulesDiff,
 			*testRulesFiles...),
 		)
 
diff --git a/cmd/promtool/unittest.go b/cmd/promtool/unittest.go
index a25a8596d..a89288c44 100644
--- a/cmd/promtool/unittest.go
+++ b/cmd/promtool/unittest.go
@@ -15,6 +15,7 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
@@ -27,6 +28,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/grafana/regexp"
+	"github.com/nsf/jsondiff"
 	"github.com/prometheus/common/model"
 	"gopkg.in/yaml.v2"
 
@@ -40,7 +42,7 @@ import (
 
 // RulesUnitTest does unit testing of rules based on the unit testing files provided.
 // More info about the file format can be found in the docs.
-func RulesUnitTest(queryOpts promql.LazyLoaderOpts, runStrings []string, files ...string) int {
+func RulesUnitTest(queryOpts promql.LazyLoaderOpts, runStrings []string, diffFlag bool, files ...string) int {
 	failed := false
 
 	var run *regexp.Regexp
@@ -49,7 +51,7 @@ func RulesUnitTest(queryOpts promql.LazyLoaderOpts, runStrings []string, files .
 	}
 
 	for _, f := range files {
-		if errs := ruleUnitTest(f, queryOpts, run); errs != nil {
+		if errs := ruleUnitTest(f, queryOpts, run, diffFlag); errs != nil {
 			fmt.Fprintln(os.Stderr, "  FAILED:")
 			for _, e := range errs {
 				fmt.Fprintln(os.Stderr, e.Error())
@@ -67,7 +69,7 @@ func RulesUnitTest(queryOpts promql.LazyLoaderOpts, runStrings []string, files .
 	return successExitCode
 }
 
-func ruleUnitTest(filename string, queryOpts promql.LazyLoaderOpts, run *regexp.Regexp) []error {
+func ruleUnitTest(filename string, queryOpts promql.LazyLoaderOpts, run *regexp.Regexp, diffFlag bool) []error {
 	fmt.Println("Unit Testing: ", filename)
 
 	b, err := os.ReadFile(filename)
@@ -109,7 +111,7 @@ func ruleUnitTest(filename string, queryOpts promql.LazyLoaderOpts, run *regexp.
 		if t.Interval == 0 {
 			t.Interval = unitTestInp.EvaluationInterval
 		}
-		ers := t.test(evalInterval, groupOrderMap, queryOpts, unitTestInp.RuleFiles...)
+		ers := t.test(evalInterval, groupOrderMap, queryOpts, diffFlag, unitTestInp.RuleFiles...)
 		if ers != nil {
 			errs = append(errs, ers...)
 		}
@@ -173,7 +175,7 @@ type testGroup struct {
 }
 
 // test performs the unit tests.
-func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, queryOpts promql.LazyLoaderOpts, ruleFiles ...string) []error {
+func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, queryOpts promql.LazyLoaderOpts, diffFlag bool, ruleFiles ...string) []error {
 	// Setup testing suite.
 	suite, err := promql.NewLazyLoader(nil, tg.seriesLoadingString(), queryOpts)
 	if err != nil {
@@ -345,8 +347,44 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
 					}
 					expString := indentLines(expAlerts.String(), "            ")
 					gotString := indentLines(gotAlerts.String(), "            ")
-					errs = append(errs, fmt.Errorf("%s    alertname: %s, time: %s, \n        exp:%v, \n        got:%v",
-						testName, testcase.Alertname, testcase.EvalTime.String(), expString, gotString))
+					if diffFlag {
+						// If empty, populates an empty value
+						if gotAlerts.Len() == 0 {
+							gotAlerts = append(gotAlerts, labelAndAnnotation{
+								Labels:      labels.Labels{},
+								Annotations: labels.Labels{},
+							})
+						}
+						// If empty, populates an empty value
+						if expAlerts.Len() == 0 {
+							expAlerts = append(expAlerts, labelAndAnnotation{
+								Labels:      labels.Labels{},
+								Annotations: labels.Labels{},
+							})
+						}
+
+						diffOpts := jsondiff.DefaultConsoleOptions()
+						expAlertsJSON, err := json.Marshal(expAlerts)
+						if err != nil {
+							errs = append(errs, fmt.Errorf("error marshaling expected %s alert: [%s]", tg.TestGroupName, err.Error()))
+							continue
+						}
+
+						gotAlertsJSON, err := json.Marshal(gotAlerts)
+						if err != nil {
+							errs = append(errs, fmt.Errorf("error marshaling received %s alert: [%s]", tg.TestGroupName, err.Error()))
+							continue
+						}
+
+						res, diff := jsondiff.Compare(expAlertsJSON, gotAlertsJSON, &diffOpts)
+						if res != jsondiff.FullMatch {
+							errs = append(errs, fmt.Errorf("%s    alertname: %s, time: %s, \n        diff: %v",
+								testName, testcase.Alertname, testcase.EvalTime.String(), indentLines(diff, "            ")))
+						}
+					} else {
+						errs = append(errs, fmt.Errorf("%s    alertname: %s, time: %s, \n        exp:%v, \n        got:%v",
+							testName, testcase.Alertname, testcase.EvalTime.String(), expString, gotString))
+					}
 				}
 			}
 
diff --git a/cmd/promtool/unittest_test.go b/cmd/promtool/unittest_test.go
index fb4012e3c..b8170d784 100644
--- a/cmd/promtool/unittest_test.go
+++ b/cmd/promtool/unittest_test.go
@@ -125,7 +125,7 @@ func TestRulesUnitTest(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := RulesUnitTest(tt.queryOpts, nil, tt.args.files...); got != tt.want {
+			if got := RulesUnitTest(tt.queryOpts, nil, false, tt.args.files...); got != tt.want {
 				t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want)
 			}
 		})
@@ -178,7 +178,7 @@ func TestRulesUnitTestRun(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := RulesUnitTest(tt.queryOpts, tt.args.run, tt.args.files...); got != tt.want {
+			if got := RulesUnitTest(tt.queryOpts, tt.args.run, false, tt.args.files...); got != tt.want {
 				t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want)
 			}
 		})
diff --git a/docs/command-line/promtool.md b/docs/command-line/promtool.md
index ba948685b..863bc068c 100644
--- a/docs/command-line/promtool.md
+++ b/docs/command-line/promtool.md
@@ -450,9 +450,10 @@ Unit tests for rules.
 
 ###### Flags
 
-| Flag | Description |
-| --- | --- |
-| <code class="text-nowrap">--run</code> | If set, will only run test groups whose names match the regular expression. Can be specified multiple times. |
+| Flag | Description | Default |
+| --- | --- | --- |
+| <code class="text-nowrap">--run</code> | If set, will only run test groups whose names match the regular expression. Can be specified multiple times. |  |
+| <code class="text-nowrap">--diff</code> | [Experimental] Print colored differential output between expected & received output. | `false` |
 
 
 
diff --git a/go.mod b/go.mod
index c34d3db2e..4833f9c11 100644
--- a/go.mod
+++ b/go.mod
@@ -43,6 +43,7 @@ require (
 	github.com/miekg/dns v1.1.57
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f
+	github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1
 	github.com/oklog/run v1.1.0
 	github.com/oklog/ulid v1.3.1
 	github.com/ovh/go-ovh v1.4.3
diff --git a/go.sum b/go.sum
index 3593e39da..daac5e320 100644
--- a/go.sum
+++ b/go.sum
@@ -579,6 +579,8 @@ github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxzi
 github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1 h1:dOYG7LS/WK00RWZc8XGgcUTlTxpp3mKhdR2Q9z9HbXM=
+github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1/go.mod h1:mpRZBD8SJ55OIICQ3iWH0Yz3cjzA61JdqMLoWXeB2+8=
 github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs=
 github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
 github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA=

From 63cdd6dbe1fa2c113b2f50d88a1634244a89a60a Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Thu, 18 Jan 2024 17:42:18 +0000
Subject: [PATCH 099/115] storage: skip merging when no remote storage
 configured

Prometheus is hard-coded to use a fanout storage between TSDB and
a remote storage which by default is empty.
This change detects the empty storage and skips merging between
result sets, which would make `Select()` sort results.

Bottom line: we skip a sort unless there really is some remote storage
configured.

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
---
 storage/fanout.go | 4 +++-
 storage/merge.go  | 3 +++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/storage/fanout.go b/storage/fanout.go
index a9a3f904b..e52342bc7 100644
--- a/storage/fanout.go
+++ b/storage/fanout.go
@@ -89,7 +89,9 @@ func (f *fanout) Querier(mint, maxt int64) (Querier, error) {
 			}
 			return nil, errs.Err()
 		}
-		secondaries = append(secondaries, querier)
+		if _, ok := querier.(noopQuerier); !ok {
+			secondaries = append(secondaries, querier)
+		}
 	}
 	return NewMergeQuerier([]Querier{primary}, secondaries, ChainedSeriesMerge), nil
 }
diff --git a/storage/merge.go b/storage/merge.go
index b4ebb440f..bcb0f66fb 100644
--- a/storage/merge.go
+++ b/storage/merge.go
@@ -46,6 +46,9 @@ type mergeGenericQuerier struct {
 //
 // In case of overlaps between the data given by primaries' and secondaries' Selects, merge function will be used.
 func NewMergeQuerier(primaries, secondaries []Querier, mergeFn VerticalSeriesMergeFunc) Querier {
+	if len(primaries)+len(secondaries) == 0 {
+		return NoopQuerier()
+	}
 	queriers := make([]genericQuerier, 0, len(primaries)+len(secondaries))
 	for _, q := range primaries {
 		if _, ok := q.(noopQuerier); !ok && q != nil {

From 65cf528403b9cd2f6e4b526ee2c29d2e84c76178 Mon Sep 17 00:00:00 2001
From: Chris Marchbanks <csmarchbanks@gmail.com>
Date: Fri, 19 Jan 2024 15:37:12 -0700
Subject: [PATCH 100/115] Remove csmarchbanks from remote write owners (#13432)

I have not had the time to keep up with remote write and have no plans
to work on it in the near future so I am withdrawing my maintainership
of that part of the codebase. I continue to focus on client_python.

Signed-off-by: Chris Marchbanks <csmarchbanks@gmail.com>
---
 .github/CODEOWNERS | 2 +-
 MAINTAINERS.md     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 1aae1fff9..41530d465 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -1,6 +1,6 @@
 /web/ui @juliusv
 /web/ui/module @juliusv @nexucis
-/storage/remote @csmarchbanks @cstyan @bwplotka @tomwilkie
+/storage/remote @cstyan @bwplotka @tomwilkie
 /storage/remote/otlptranslator @gouthamve @jesusvazquez
 /discovery/kubernetes @brancz
 /tsdb @jesusvazquez
diff --git a/MAINTAINERS.md b/MAINTAINERS.md
index 902e9a6e9..e2fa9c294 100644
--- a/MAINTAINERS.md
+++ b/MAINTAINERS.md
@@ -9,7 +9,7 @@ Julien Pivotto (<roidelapluie@prometheus.io> / @roidelapluie) and Levi Harrison
 * `documentation`
   * `prometheus-mixin`: Matthias Loibl (<mail@matthiasloibl.com> / @metalmatze)
 * `storage`
-  * `remote`: Chris Marchbanks (<csmarchbanks@gmail.com> / @csmarchbanks), Callum Styan (<callumstyan@gmail.com> / @cstyan), Bartłomiej Płotka (<bwplotka@gmail.com> / @bwplotka), Tom Wilkie (<tom.wilkie@gmail.com> / @tomwilkie)
+  * `remote`: Callum Styan (<callumstyan@gmail.com> / @cstyan), Bartłomiej Płotka (<bwplotka@gmail.com> / @bwplotka), Tom Wilkie (<tom.wilkie@gmail.com> / @tomwilkie)
 * `tsdb`: Ganesh Vernekar (<ganesh@grafana.com> / @codesome), Bartłomiej Płotka (<bwplotka@gmail.com> / @bwplotka), Jesús Vázquez (<jesus.vazquez@grafana.com> / @jesusvazquez)
   * `agent`: Robert Fratto (<robert.fratto@grafana.com> / @rfratto)
 * `web`

From d778591fd3b98f779d5194fd30d07dd813c0ae9b Mon Sep 17 00:00:00 2001
From: Ben Ye <benye@amazon.com>
Date: Sun, 21 Jan 2024 14:19:39 -0800
Subject: [PATCH 101/115] add more context cancellation check at evaluation
 time

Signed-off-by: Ben Ye <benye@amazon.com>
---
 promql/engine.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/promql/engine.go b/promql/engine.go
index 6349ddbbe..7165631e0 100644
--- a/promql/engine.go
+++ b/promql/engine.go
@@ -1471,6 +1471,9 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, annotations.Annotatio
 		it := storage.NewBuffer(selRange)
 		var chkIter chunkenc.Iterator
 		for i, s := range selVS.Series {
+			if err := contextDone(ev.ctx, "expression evaluation"); err != nil {
+				ev.error(err)
+			}
 			ev.currentSamples -= len(floats) + totalHPointSize(histograms)
 			if floats != nil {
 				floats = floats[:0]
@@ -1690,6 +1693,9 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, annotations.Annotatio
 		it := storage.NewMemoizedEmptyIterator(durationMilliseconds(ev.lookbackDelta))
 		var chkIter chunkenc.Iterator
 		for i, s := range e.Series {
+			if err := contextDone(ev.ctx, "expression evaluation"); err != nil {
+				ev.error(err)
+			}
 			chkIter = s.Iterator(chkIter)
 			it.Reset(chkIter)
 			ss := Series{

From ed172a666737a11a09e56d512078b548fe9edb18 Mon Sep 17 00:00:00 2001
From: Oleg Zaytsev <mail@olegzaytsev.com>
Date: Tue, 23 Jan 2024 11:40:21 +0100
Subject: [PATCH 102/115] Optimize label values with matchers by taking
 shortcuts (#13426)

Don't calculate postings beforehand: we may not need them. If all
matchers are for the requested label, we can just filter its values.

Also, if there are no values at all, no need to run any kind of
logic.

Also add more labelValuesWithMatchers benchmarks

Signed-off-by: Oleg Zaytsev <mail@olegzaytsev.com>
---
 tsdb/ooo_head_read_test.go |  4 ++--
 tsdb/querier.go            | 21 ++++++++++++++++-----
 tsdb/querier_bench_test.go | 13 ++++++++++++-
 3 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/tsdb/ooo_head_read_test.go b/tsdb/ooo_head_read_test.go
index b50a268b7..36289056b 100644
--- a/tsdb/ooo_head_read_test.go
+++ b/tsdb/ooo_head_read_test.go
@@ -410,7 +410,7 @@ func TestOOOHeadChunkReader_LabelValues(t *testing.T) {
 			queryMinT:  math.MinInt64,
 			queryMaxT:  math.MaxInt64,
 			expValues1: []string{"bar1"},
-			expValues2: []string{},
+			expValues2: nil,
 			expValues3: []string{"bar1", "bar2"},
 			expValues4: []string{"bar1", "bar2"},
 		},
@@ -419,7 +419,7 @@ func TestOOOHeadChunkReader_LabelValues(t *testing.T) {
 			queryMinT:  90,
 			queryMaxT:  90,
 			expValues1: []string{"bar1"},
-			expValues2: []string{},
+			expValues2: nil,
 			expValues3: []string{"bar1", "bar2"},
 			expValues4: []string{"bar1", "bar2"},
 		},
diff --git a/tsdb/querier.go b/tsdb/querier.go
index f88e4415e..68aa95746 100644
--- a/tsdb/querier.go
+++ b/tsdb/querier.go
@@ -440,11 +440,6 @@ func inversePostingsForMatcher(ctx context.Context, ix IndexReader, m *labels.Ma
 }
 
 func labelValuesWithMatchers(ctx context.Context, r IndexReader, name string, matchers ...*labels.Matcher) ([]string, error) {
-	p, err := PostingsForMatchers(ctx, r, matchers...)
-	if err != nil {
-		return nil, fmt.Errorf("fetching postings for matchers: %w", err)
-	}
-
 	allValues, err := r.LabelValues(ctx, name)
 	if err != nil {
 		return nil, fmt.Errorf("fetching values of label %s: %w", name, err)
@@ -453,8 +448,10 @@ func labelValuesWithMatchers(ctx context.Context, r IndexReader, name string, ma
 	// If we have a matcher for the label name, we can filter out values that don't match
 	// before we fetch postings. This is especially useful for labels with many values.
 	// e.g. __name__ with a selector like {__name__="xyz"}
+	hasMatchersForOtherLabels := false
 	for _, m := range matchers {
 		if m.Name != name {
+			hasMatchersForOtherLabels = true
 			continue
 		}
 
@@ -469,6 +466,20 @@ func labelValuesWithMatchers(ctx context.Context, r IndexReader, name string, ma
 		allValues = filteredValues
 	}
 
+	if len(allValues) == 0 {
+		return nil, nil
+	}
+
+	// If we don't have any matchers for other labels, then we're done.
+	if !hasMatchersForOtherLabels {
+		return allValues, nil
+	}
+
+	p, err := PostingsForMatchers(ctx, r, matchers...)
+	if err != nil {
+		return nil, fmt.Errorf("fetching postings for matchers: %w", err)
+	}
+
 	valuesPostings := make([]index.Postings, len(allValues))
 	for i, value := range allValues {
 		valuesPostings[i], err = r.Postings(ctx, name, value)
diff --git a/tsdb/querier_bench_test.go b/tsdb/querier_bench_test.go
index 7b839ca34..9a8230242 100644
--- a/tsdb/querier_bench_test.go
+++ b/tsdb/querier_bench_test.go
@@ -182,13 +182,16 @@ func benchmarkPostingsForMatchers(b *testing.B, ir IndexReader) {
 
 func benchmarkLabelValuesWithMatchers(b *testing.B, ir IndexReader) {
 	i1 := labels.MustNewMatcher(labels.MatchEqual, "i", "1")
+	i1Plus := labels.MustNewMatcher(labels.MatchRegexp, "i", "1.+")
+	i1PostingsBenchSuffix := labels.MustNewMatcher(labels.MatchEqual, "i", "1"+postingsBenchSuffix)
+	iSuffix := labels.MustNewMatcher(labels.MatchRegexp, "i", ".+ddd")
 	iStar := labels.MustNewMatcher(labels.MatchRegexp, "i", "^.*$")
 	jNotFoo := labels.MustNewMatcher(labels.MatchNotEqual, "j", "foo")
 	jXXXYYY := labels.MustNewMatcher(labels.MatchRegexp, "j", "XXX|YYY")
 	jXplus := labels.MustNewMatcher(labels.MatchRegexp, "j", "X.+")
 	n1 := labels.MustNewMatcher(labels.MatchEqual, "n", "1"+postingsBenchSuffix)
 	nX := labels.MustNewMatcher(labels.MatchNotEqual, "n", "X"+postingsBenchSuffix)
-	nPlus := labels.MustNewMatcher(labels.MatchRegexp, "i", "^.+$")
+	nPlus := labels.MustNewMatcher(labels.MatchRegexp, "n", "^.+$")
 
 	ctx := context.Background()
 
@@ -197,6 +200,7 @@ func benchmarkLabelValuesWithMatchers(b *testing.B, ir IndexReader) {
 		labelName string
 		matchers  []*labels.Matcher
 	}{
+		// i is never "1", this isn't matching anything.
 		{`i with i="1"`, "i", []*labels.Matcher{i1}},
 		// i has 100k values.
 		{`i with n="1"`, "i", []*labels.Matcher{n1}},
@@ -206,9 +210,16 @@ func benchmarkLabelValuesWithMatchers(b *testing.B, ir IndexReader) {
 		{`i with n="1",j=~"XXX|YYY"`, "i", []*labels.Matcher{n1, jXXXYYY}},
 		{`i with n="X",j!="foo"`, "i", []*labels.Matcher{nX, jNotFoo}},
 		{`i with n="1",i=~"^.*$",j!="foo"`, "i", []*labels.Matcher{n1, iStar, jNotFoo}},
+		// matchers on i itself
+		{`i with i="1aaa...ddd"`, "i", []*labels.Matcher{i1PostingsBenchSuffix}},
+		{`i with i=~"1.+"`, "i", []*labels.Matcher{i1Plus}},
+		{`i with i=~"1.+",i=~".+ddd"`, "i", []*labels.Matcher{i1Plus, iSuffix}},
 		// n has 10 values.
 		{`n with j!="foo"`, "n", []*labels.Matcher{jNotFoo}},
 		{`n with i="1"`, "n", []*labels.Matcher{i1}},
+		{`n with i=~"1.+"`, "n", []*labels.Matcher{i1Plus}},
+		// there's no label "none"
+		{`none with i=~"1"`, "none", []*labels.Matcher{i1Plus}},
 	}
 
 	for _, c := range cases {

From 2c0f9d1eee1ee68d830b4c2c070b3bf2b111c83e Mon Sep 17 00:00:00 2001
From: SuperQ <superq@gmail.com>
Date: Mon, 8 Jan 2024 19:10:12 +0100
Subject: [PATCH 103/115] Add automatic memory limit handling

Enable automatic detection of memory limits and configure GOMEMLIMIT to
match.
* Also includes a flag to allow controlling the reserved ratio.

Signed-off-by: SuperQ <superq@gmail.com>
---
 cmd/prometheus/main.go          | 32 ++++++++++++++++++++++++++++++--
 docs/command-line/prometheus.md |  3 ++-
 docs/feature_flags.md           |  8 ++++++++
 go.mod                          |  7 +++++++
 go.sum                          | 15 +++++++++++++++
 5 files changed, 62 insertions(+), 3 deletions(-)

diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go
index 95a581a74..d73652fba 100644
--- a/cmd/prometheus/main.go
+++ b/cmd/prometheus/main.go
@@ -33,6 +33,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/KimMachineGun/automemlimit/memlimit"
 	"github.com/alecthomas/kingpin/v2"
 	"github.com/alecthomas/units"
 	"github.com/go-kit/log"
@@ -147,13 +148,15 @@ type flagConfig struct {
 	queryMaxSamples     int
 	RemoteFlushDeadline model.Duration
 
-	featureList []string
+	featureList   []string
+	memlimitRatio float64
 	// These options are extracted from featureList
 	// for ease of use.
 	enableExpandExternalLabels bool
 	enableNewSDManager         bool
 	enablePerStepStats         bool
 	enableAutoGOMAXPROCS       bool
+	enableAutoGOMEMLIMIT       bool
 
 	prometheusURL   string
 	corsRegexString string
@@ -197,6 +200,9 @@ func (c *flagConfig) setFeatureListOptions(logger log.Logger) error {
 			case "auto-gomaxprocs":
 				c.enableAutoGOMAXPROCS = true
 				level.Info(logger).Log("msg", "Automatically set GOMAXPROCS to match Linux container CPU quota")
+			case "auto-gomemlimit":
+				c.enableAutoGOMEMLIMIT = true
+				level.Info(logger).Log("msg", "Automatically set GOMEMLIMIT to match Linux container or system memory limit")
 			case "no-default-scrape-port":
 				c.scrape.NoDefaultPort = true
 				level.Info(logger).Log("msg", "No default port will be appended to scrape targets' addresses.")
@@ -262,6 +268,9 @@ func main() {
 	a.Flag("web.listen-address", "Address to listen on for UI, API, and telemetry.").
 		Default("0.0.0.0:9090").StringVar(&cfg.web.ListenAddress)
 
+	a.Flag("auto-gomemlimit.ratio", "The ratio of reserved GOMEMLIMIT memory to the detected maximum container or system memory").
+		Default("0.9").FloatVar(&cfg.memlimitRatio)
+
 	webConfig := a.Flag(
 		"web.config.file",
 		"[EXPERIMENTAL] Path to configuration file that can enable TLS or authentication.",
@@ -429,7 +438,7 @@ func main() {
 	a.Flag("scrape.discovery-reload-interval", "Interval used by scrape manager to throttle target groups updates.").
 		Hidden().Default("5s").SetValue(&cfg.scrape.DiscoveryReloadInterval)
 
-	a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: agent, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-at-modifier, promql-negative-offset, promql-per-step-stats, promql-experimental-functions, remote-write-receiver (DEPRECATED), extra-scrape-metrics, new-service-discovery-manager, auto-gomaxprocs, no-default-scrape-port, native-histograms, otlp-write-receiver. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details.").
+	a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: agent, auto-gomemlimit, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-at-modifier, promql-negative-offset, promql-per-step-stats, promql-experimental-functions, remote-write-receiver (DEPRECATED), extra-scrape-metrics, new-service-discovery-manager, auto-gomaxprocs, no-default-scrape-port, native-histograms, otlp-write-receiver. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details.").
 		Default("").StringsVar(&cfg.featureList)
 
 	promlogflag.AddFlags(a, &cfg.promlogConfig)
@@ -467,6 +476,11 @@ func main() {
 		os.Exit(3)
 	}
 
+	if cfg.memlimitRatio <= 0.0 || cfg.memlimitRatio > 1.0 {
+		fmt.Fprintf(os.Stderr, "--auto-gomemlimit.ratio must be greater than 0 and less than or equal to 1.")
+		os.Exit(1)
+	}
+
 	localStoragePath := cfg.serverStoragePath
 	if agentMode {
 		localStoragePath = cfg.agentStoragePath
@@ -695,6 +709,20 @@ func main() {
 		}
 	}
 
+	if cfg.enableAutoGOMEMLIMIT {
+		if _, err := memlimit.SetGoMemLimitWithOpts(
+			memlimit.WithRatio(cfg.memlimitRatio),
+			memlimit.WithProvider(
+				memlimit.ApplyFallback(
+					memlimit.FromCgroup,
+					memlimit.FromSystem,
+				),
+			),
+		); err != nil {
+			level.Warn(logger).Log("component", "automemlimit", "msg", "Failed to set GOMEMLIMIT automatically", "err", err)
+		}
+	}
+
 	if !agentMode {
 		opts := promql.EngineOpts{
 			Logger:                   log.With(logger, "component", "query engine"),
diff --git a/docs/command-line/prometheus.md b/docs/command-line/prometheus.md
index cd6dac555..747457de1 100644
--- a/docs/command-line/prometheus.md
+++ b/docs/command-line/prometheus.md
@@ -16,6 +16,7 @@ The Prometheus monitoring server
 | <code class="text-nowrap">--version</code> | Show application version. |  |
 | <code class="text-nowrap">--config.file</code> | Prometheus configuration file path. | `prometheus.yml` |
 | <code class="text-nowrap">--web.listen-address</code> | Address to listen on for UI, API, and telemetry. | `0.0.0.0:9090` |
+| <code class="text-nowrap">--auto-gomemlimit.ratio</code> | The ratio of reserved GOMEMLIMIT memory to the detected maximum container or system memory | `0.9` |
 | <code class="text-nowrap">--web.config.file</code> | [EXPERIMENTAL] Path to configuration file that can enable TLS or authentication. |  |
 | <code class="text-nowrap">--web.read-timeout</code> | Maximum duration before timing out read of the request, and closing idle connections. | `5m` |
 | <code class="text-nowrap">--web.max-connections</code> | Maximum number of simultaneous connections. | `512` |
@@ -52,7 +53,7 @@ The Prometheus monitoring server
 | <code class="text-nowrap">--query.timeout</code> | Maximum time a query may take before being aborted. Use with server mode only. | `2m` |
 | <code class="text-nowrap">--query.max-concurrency</code> | Maximum number of queries executed concurrently. Use with server mode only. | `20` |
 | <code class="text-nowrap">--query.max-samples</code> | Maximum number of samples a single query can load into memory. Note that queries will fail if they try to load more samples than this into memory, so this also limits the number of samples a query can return. Use with server mode only. | `50000000` |
-| <code class="text-nowrap">--enable-feature</code> | Comma separated feature names to enable. Valid options: agent, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-at-modifier, promql-negative-offset, promql-per-step-stats, promql-experimental-functions, remote-write-receiver (DEPRECATED), extra-scrape-metrics, new-service-discovery-manager, auto-gomaxprocs, no-default-scrape-port, native-histograms, otlp-write-receiver. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details. |  |
+| <code class="text-nowrap">--enable-feature</code> | Comma separated feature names to enable. Valid options: agent, auto-gomemlimit, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-at-modifier, promql-negative-offset, promql-per-step-stats, promql-experimental-functions, remote-write-receiver (DEPRECATED), extra-scrape-metrics, new-service-discovery-manager, auto-gomaxprocs, no-default-scrape-port, native-histograms, otlp-write-receiver. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details. |  |
 | <code class="text-nowrap">--log.level</code> | Only log messages with the given severity or above. One of: [debug, info, warn, error] | `info` |
 | <code class="text-nowrap">--log.format</code> | Output format of log messages. One of: [logfmt, json] | `logfmt` |
 
diff --git a/docs/feature_flags.md b/docs/feature_flags.md
index bcf8309b5..adefaad4b 100644
--- a/docs/feature_flags.md
+++ b/docs/feature_flags.md
@@ -95,6 +95,14 @@ computed at all.
 
 When enabled, GOMAXPROCS variable is automatically set to match Linux container CPU quota.
 
+## Auto GOMEMLIMIT
+
+`--enable-feature=auto-gomemlimit`
+
+When enabled, the GOMEMLIMIT variable is automatically set to match the Linux container memory limit. If there is no container limit, or the process is runing outside of containers, the system memory total is used.
+
+There is also an additional tuning flag, `--auto-gomemlimit.ratio`, which allows controling how much of the memory is used for Prometheus. The remainder is reserved for memory outside the process. For example, kernel page cache. Page cache is important for Prometheus TSDB query performance. The default is `0.9`, which means 90% of the memory limit will be used for Prometheus.
+
 ## No default scrape port
 
 `--enable-feature=no-default-scrape-port`
diff --git a/go.mod b/go.mod
index 4833f9c11..fba6a7739 100644
--- a/go.mod
+++ b/go.mod
@@ -8,6 +8,7 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.3.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0
 	github.com/Code-Hex/go-generics-cache v1.3.1
+	github.com/KimMachineGun/automemlimit v0.5.0
 	github.com/alecthomas/kingpin/v2 v2.4.0
 	github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9
 	github.com/aws/aws-sdk-go v1.48.14
@@ -102,7 +103,9 @@ require (
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
+	github.com/cilium/ebpf v0.11.0 // indirect
 	github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 // indirect
+	github.com/containerd/cgroups/v3 v3.0.3 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/docker/distribution v2.8.2+incompatible // indirect
@@ -125,6 +128,7 @@ require (
 	github.com/go-openapi/swag v0.22.4 // indirect
 	github.com/go-openapi/validate v0.22.1 // indirect
 	github.com/go-resty/resty/v2 v2.11.0 // indirect
+	github.com/godbus/dbus/v5 v5.0.4 // indirect
 	github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
 	github.com/golang/glog v1.1.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@@ -167,10 +171,13 @@ require (
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.2 // indirect
+	github.com/opencontainers/runtime-spec v1.0.2 // indirect
+	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/objx v0.5.0 // indirect
 	github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
diff --git a/go.sum b/go.sum
index daac5e320..e5c269f47 100644
--- a/go.sum
+++ b/go.sum
@@ -55,6 +55,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
 github.com/Code-Hex/go-generics-cache v1.3.1 h1:i8rLwyhoyhaerr7JpjtYjJZUcCbWOdiYO3fZXLiEC4g=
 github.com/Code-Hex/go-generics-cache v1.3.1/go.mod h1:qxcC9kRVrct9rHeiYpFWSoW1vxyillCVzX13KZG8dl4=
 github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
+github.com/KimMachineGun/automemlimit v0.5.0 h1:BeOe+BbJc8L5chL3OwzVYjVzyvPALdd5wxVVOWuUZmQ=
+github.com/KimMachineGun/automemlimit v0.5.0/go.mod h1:di3GCKiu9Y+1fs92erCbUvKzPkNyViN3mA0vti/ykEQ=
 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
 github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
 github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
@@ -110,6 +112,8 @@ github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/cilium/ebpf v0.11.0 h1:V8gS/bTCCjX9uUnkUFUpPsksM8n1lXBAvHcpiFk1X2Y=
+github.com/cilium/ebpf v0.11.0/go.mod h1:WE7CZAnqOL2RouJ4f1uyNhqr2P4CCvXFIqdRDUgWsVs=
 github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
 github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
 github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
@@ -119,6 +123,8 @@ github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+g
 github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
+github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0=
+github.com/containerd/cgroups/v3 v3.0.3/go.mod h1:8HBe7V3aWGLFPd/k03swSIsGjZhHI2WzJmticMgVuz0=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
@@ -177,6 +183,7 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
 github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
+github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
@@ -271,6 +278,7 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe
 github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
 github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
 github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
+github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -599,6 +607,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=
 github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0=
+github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis=
 github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
 github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
@@ -613,6 +623,8 @@ github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIw
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
+github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
 github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
@@ -700,6 +712,8 @@ github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
+github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
@@ -990,6 +1004,7 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

From 7bbae70b40e7201979ccbcabb9abb1437e3b4dc2 Mon Sep 17 00:00:00 2001
From: Matthieu MOREL <matthieu.morel35@gmail.com>
Date: Tue, 23 Jan 2024 16:08:22 +0100
Subject: [PATCH 104/115] Update OSSF badge link (#13433)

Provide a more user friendly interface

Signed-off-by: Matthieu MOREL <matthieu.morel35@gmail.com>
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 5fa6cc49e..0042793ff 100644
--- a/README.md
+++ b/README.md
@@ -14,7 +14,7 @@ examples and guides.</p>
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/486/badge)](https://bestpractices.coreinfrastructure.org/projects/486)
 [![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/prometheus/prometheus)
 [![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/prometheus.svg)](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:prometheus)
-[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/prometheus/prometheus/badge)](https://api.securityscorecards.dev/projects/github.com/prometheus/prometheus)
+[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/prometheus/prometheus/badge)](https://securityscorecards.dev/viewer/?uri=github.com/prometheus/prometheus)
 
 </div>
 

From 78411d5e8b803b590ba10bb563caab52292a11a2 Mon Sep 17 00:00:00 2001
From: Paulin Todev <paulin.todev@gmail.com>
Date: Tue, 23 Jan 2024 15:53:55 +0000
Subject: [PATCH 105/115] SD Managers taking over responsibility for
 registration of debug metrics (#13375)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

SD Managers take over responsibility for SD metrics registration

---------

Signed-off-by: Paulin Todev <paulin.todev@gmail.com>
Signed-off-by: Björn Rabenstein <github@rabenste.in>
Co-authored-by: Björn Rabenstein <github@rabenste.in>
---
 cmd/prometheus/main.go                        | 15 ++-
 cmd/promtool/sd.go                            | 17 +++-
 discovery/aws/ec2.go                          | 28 ++++--
 discovery/aws/lightsail.go                    | 29 ++++--
 discovery/aws/metrics_ec2.go                  | 32 ++++++
 discovery/aws/metrics_lightsail.go            | 32 ++++++
 discovery/azure/azure.go                      | 67 +++++++------
 discovery/azure/metrics.go                    | 64 ++++++++++++
 discovery/consul/consul.go                    | 89 ++++++-----------
 discovery/consul/consul_test.go               | 37 +++++--
 discovery/consul/metrics.go                   | 73 ++++++++++++++
 discovery/digitalocean/digitalocean.go        | 26 +++--
 discovery/digitalocean/digitalocean_test.go   | 12 ++-
 discovery/digitalocean/metrics.go             | 32 ++++++
 discovery/discoverer_metrics_noop.go          | 28 ++++++
 discovery/discovery.go                        | 47 +++++++--
 discovery/dns/dns.go                          | 53 +++++-----
 discovery/dns/dns_test.go                     | 11 ++-
 discovery/dns/metrics.go                      | 66 +++++++++++++
 discovery/eureka/eureka.go                    | 27 +++--
 discovery/eureka/eureka_test.go               | 13 ++-
 discovery/eureka/metrics.go                   | 32 ++++++
 discovery/file/file.go                        | 66 +++++--------
 discovery/file/file_test.go                   | 24 +++--
 discovery/file/metrics.go                     | 76 ++++++++++++++
 discovery/gce/gce.go                          | 26 +++--
 discovery/gce/metrics.go                      | 32 ++++++
 discovery/hetzner/hetzner.go                  | 26 +++--
 discovery/hetzner/metrics.go                  | 32 ++++++
 discovery/http/http.go                        | 45 +++++----
 discovery/http/http_test.go                   | 44 +++++++--
 discovery/http/metrics.go                     | 57 +++++++++++
 discovery/ionos/ionos.go                      | 30 ++++--
 discovery/ionos/metrics.go                    | 32 ++++++
 discovery/kubernetes/kubernetes.go            | 65 ++++--------
 discovery/kubernetes/kubernetes_test.go       | 25 +++--
 discovery/kubernetes/metrics.go               | 75 ++++++++++++++
 discovery/legacymanager/manager.go            | 12 ++-
 discovery/legacymanager/manager_test.go       | 58 +++++++++--
 discovery/linode/linode.go                    | 37 ++++---
 discovery/linode/linode_test.go               | 12 ++-
 discovery/linode/metrics.go                   | 57 +++++++++++
 discovery/manager.go                          | 26 ++++-
 discovery/manager_test.go                     | 98 ++++++++++++++++---
 discovery/marathon/marathon.go                | 26 +++--
 discovery/marathon/marathon_test.go           | 26 ++++-
 discovery/marathon/metrics.go                 | 32 ++++++
 discovery/metrics.go                          |  2 +-
 discovery/metrics_refresh.go                  | 75 ++++++++++++++
 discovery/moby/docker.go                      | 26 +++--
 discovery/moby/docker_test.go                 | 11 ++-
 discovery/moby/dockerswarm.go                 | 26 +++--
 discovery/moby/metrics_docker.go              | 32 ++++++
 discovery/moby/metrics_dockerswarm.go         | 32 ++++++
 discovery/moby/nodes_test.go                  | 11 ++-
 discovery/moby/services_test.go               | 20 +++-
 discovery/moby/tasks_test.go                  | 11 ++-
 discovery/nomad/metrics.go                    | 57 +++++++++++
 discovery/nomad/nomad.go                      | 37 ++++---
 discovery/nomad/nomad_test.go                 | 22 ++++-
 discovery/openstack/metrics.go                | 32 ++++++
 discovery/openstack/openstack.go              | 26 +++--
 discovery/ovhcloud/metrics.go                 | 32 ++++++
 discovery/ovhcloud/ovhcloud.go                | 26 +++--
 discovery/ovhcloud/ovhcloud_test.go           | 12 ++-
 discovery/puppetdb/metrics.go                 | 32 ++++++
 discovery/puppetdb/puppetdb.go                | 26 +++--
 discovery/puppetdb/puppetdb_test.go           | 47 ++++++++-
 discovery/refresh/refresh.go                  | 56 +++--------
 discovery/refresh/refresh_test.go             | 16 ++-
 discovery/registry.go                         | 23 +++++
 discovery/scaleway/metrics.go                 | 32 ++++++
 discovery/scaleway/scaleway.go                | 26 +++--
 discovery/triton/metrics.go                   | 32 ++++++
 discovery/triton/triton.go                    | 26 +++--
 discovery/triton/triton_test.go               | 43 ++++++--
 discovery/uyuni/metrics.go                    | 32 ++++++
 discovery/uyuni/uyuni.go                      | 26 +++--
 discovery/uyuni/uyuni_test.go                 | 22 ++++-
 discovery/vultr/metrics.go                    | 32 ++++++
 discovery/vultr/vultr.go                      | 26 +++--
 discovery/vultr/vultr_test.go                 | 12 ++-
 discovery/xds/kuma.go                         | 44 +++------
 discovery/xds/kuma_test.go                    | 16 ++-
 discovery/xds/metrics.go                      | 73 ++++++++++++++
 discovery/xds/xds.go                          | 21 +---
 discovery/xds/xds_test.go                     | 91 ++++++++++-------
 discovery/zookeeper/zookeeper.go              | 11 +++
 .../examples/custom-sd/adapter-usage/main.go  | 13 ++-
 .../examples/custom-sd/adapter/adapter.go     |  4 +-
 .../custom-sd/adapter/adapter_test.go         | 10 +-
 91 files changed, 2522 insertions(+), 629 deletions(-)
 create mode 100644 discovery/aws/metrics_ec2.go
 create mode 100644 discovery/aws/metrics_lightsail.go
 create mode 100644 discovery/azure/metrics.go
 create mode 100644 discovery/consul/metrics.go
 create mode 100644 discovery/digitalocean/metrics.go
 create mode 100644 discovery/discoverer_metrics_noop.go
 create mode 100644 discovery/dns/metrics.go
 create mode 100644 discovery/eureka/metrics.go
 create mode 100644 discovery/file/metrics.go
 create mode 100644 discovery/gce/metrics.go
 create mode 100644 discovery/hetzner/metrics.go
 create mode 100644 discovery/http/metrics.go
 create mode 100644 discovery/ionos/metrics.go
 create mode 100644 discovery/kubernetes/metrics.go
 create mode 100644 discovery/linode/metrics.go
 create mode 100644 discovery/marathon/metrics.go
 create mode 100644 discovery/metrics_refresh.go
 create mode 100644 discovery/moby/metrics_docker.go
 create mode 100644 discovery/moby/metrics_dockerswarm.go
 create mode 100644 discovery/nomad/metrics.go
 create mode 100644 discovery/openstack/metrics.go
 create mode 100644 discovery/ovhcloud/metrics.go
 create mode 100644 discovery/puppetdb/metrics.go
 create mode 100644 discovery/scaleway/metrics.go
 create mode 100644 discovery/triton/metrics.go
 create mode 100644 discovery/uyuni/metrics.go
 create mode 100644 discovery/vultr/metrics.go
 create mode 100644 discovery/xds/metrics.go

diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go
index d73652fba..f7244646e 100644
--- a/cmd/prometheus/main.go
+++ b/cmd/prometheus/main.go
@@ -644,9 +644,16 @@ func main() {
 		level.Error(logger).Log("msg", "failed to register Kubernetes client metrics", "err", err)
 		os.Exit(1)
 	}
+
+	sdMetrics, err := discovery.CreateAndRegisterSDMetrics(prometheus.DefaultRegisterer)
+	if err != nil {
+		level.Error(logger).Log("msg", "failed to register service discovery metrics", "err", err)
+		os.Exit(1)
+	}
+
 	if cfg.enableNewSDManager {
 		{
-			discMgr := discovery.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, discovery.Name("scrape"))
+			discMgr := discovery.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, sdMetrics, discovery.Name("scrape"))
 			if discMgr == nil {
 				level.Error(logger).Log("msg", "failed to create a discovery manager scrape")
 				os.Exit(1)
@@ -655,7 +662,7 @@ func main() {
 		}
 
 		{
-			discMgr := discovery.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, discovery.Name("notify"))
+			discMgr := discovery.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, sdMetrics, discovery.Name("notify"))
 			if discMgr == nil {
 				level.Error(logger).Log("msg", "failed to create a discovery manager notify")
 				os.Exit(1)
@@ -664,7 +671,7 @@ func main() {
 		}
 	} else {
 		{
-			discMgr := legacymanager.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, legacymanager.Name("scrape"))
+			discMgr := legacymanager.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, sdMetrics, legacymanager.Name("scrape"))
 			if discMgr == nil {
 				level.Error(logger).Log("msg", "failed to create a discovery manager scrape")
 				os.Exit(1)
@@ -673,7 +680,7 @@ func main() {
 		}
 
 		{
-			discMgr := legacymanager.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, legacymanager.Name("notify"))
+			discMgr := legacymanager.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, sdMetrics, legacymanager.Name("notify"))
 			if discMgr == nil {
 				level.Error(logger).Log("msg", "failed to create a discovery manager notify")
 				os.Exit(1)
diff --git a/cmd/promtool/sd.go b/cmd/promtool/sd.go
index 155152e1a..4892743fc 100644
--- a/cmd/promtool/sd.go
+++ b/cmd/promtool/sd.go
@@ -78,12 +78,25 @@ func CheckSD(sdConfigFiles, sdJobName string, sdTimeout time.Duration, noDefault
 	defer cancel()
 
 	for _, cfg := range scrapeConfig.ServiceDiscoveryConfigs {
-		d, err := cfg.NewDiscoverer(discovery.DiscovererOptions{Logger: logger, Registerer: registerer})
+		reg := prometheus.NewRegistry()
+		refreshMetrics := discovery.NewRefreshMetrics(reg)
+		metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+		err := metrics.Register()
+		if err != nil {
+			fmt.Fprintln(os.Stderr, "Could not register service discovery metrics", err)
+			return failureExitCode
+		}
+
+		d, err := cfg.NewDiscoverer(discovery.DiscovererOptions{Logger: logger, Metrics: metrics})
 		if err != nil {
 			fmt.Fprintln(os.Stderr, "Could not create new discoverer", err)
 			return failureExitCode
 		}
-		go d.Run(ctx, targetGroupChan)
+		go func() {
+			d.Run(ctx, targetGroupChan)
+			metrics.Unregister()
+			refreshMetrics.Unregister()
+		}()
 	}
 
 	var targetGroups []*targetgroup.Group
diff --git a/discovery/aws/ec2.go b/discovery/aws/ec2.go
index 40e6e7cb7..aa79fd9c6 100644
--- a/discovery/aws/ec2.go
+++ b/discovery/aws/ec2.go
@@ -97,12 +97,19 @@ type EC2SDConfig struct {
 	HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*EC2SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &ec2Metrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the EC2 Config.
 func (*EC2SDConfig) Name() string { return "ec2" }
 
 // NewDiscoverer returns a Discoverer for the EC2 Config.
 func (c *EC2SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewEC2Discovery(c, opts.Logger, opts.Registerer), nil
+	return NewEC2Discovery(c, opts.Logger, opts.Metrics)
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface for the EC2 Config.
@@ -148,7 +155,12 @@ type EC2Discovery struct {
 }
 
 // NewEC2Discovery returns a new EC2Discovery which periodically refreshes its targets.
-func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger, reg prometheus.Registerer) *EC2Discovery {
+func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*EC2Discovery, error) {
+	m, ok := metrics.(*ec2Metrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -158,14 +170,14 @@ func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger, reg prometheus.Regist
 	}
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
-			Mech:     "ec2",
-			Interval: time.Duration(d.cfg.RefreshInterval),
-			RefreshF: d.refresh,
-			Registry: reg,
+			Logger:              logger,
+			Mech:                "ec2",
+			Interval:            time.Duration(d.cfg.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
-	return d
+	return d, nil
 }
 
 func (d *EC2Discovery) ec2Client(context.Context) (*ec2.EC2, error) {
diff --git a/discovery/aws/lightsail.go b/discovery/aws/lightsail.go
index 5382ea015..86b138be5 100644
--- a/discovery/aws/lightsail.go
+++ b/discovery/aws/lightsail.go
@@ -80,12 +80,19 @@ type LightsailSDConfig struct {
 	HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*LightsailSDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &lightsailMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Lightsail Config.
 func (*LightsailSDConfig) Name() string { return "lightsail" }
 
 // NewDiscoverer returns a Discoverer for the Lightsail Config.
 func (c *LightsailSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewLightsailDiscovery(c, opts.Logger, opts.Registerer), nil
+	return NewLightsailDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface for the Lightsail Config.
@@ -122,23 +129,29 @@ type LightsailDiscovery struct {
 }
 
 // NewLightsailDiscovery returns a new LightsailDiscovery which periodically refreshes its targets.
-func NewLightsailDiscovery(conf *LightsailSDConfig, logger log.Logger, reg prometheus.Registerer) *LightsailDiscovery {
+func NewLightsailDiscovery(conf *LightsailSDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*LightsailDiscovery, error) {
+	m, ok := metrics.(*lightsailMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
+
 	d := &LightsailDiscovery{
 		cfg: conf,
 	}
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
-			Mech:     "lightsail",
-			Interval: time.Duration(d.cfg.RefreshInterval),
-			RefreshF: d.refresh,
-			Registry: reg,
+			Logger:              logger,
+			Mech:                "lightsail",
+			Interval:            time.Duration(d.cfg.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
-	return d
+	return d, nil
 }
 
 func (d *LightsailDiscovery) lightsailClient() (*lightsail.Lightsail, error) {
diff --git a/discovery/aws/metrics_ec2.go b/discovery/aws/metrics_ec2.go
new file mode 100644
index 000000000..73c061c3d
--- /dev/null
+++ b/discovery/aws/metrics_ec2.go
@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aws
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+type ec2Metrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+var _ discovery.DiscovererMetrics = (*ec2Metrics)(nil)
+
+// Register implements discovery.DiscovererMetrics.
+func (m *ec2Metrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *ec2Metrics) Unregister() {}
diff --git a/discovery/aws/metrics_lightsail.go b/discovery/aws/metrics_lightsail.go
new file mode 100644
index 000000000..69593e701
--- /dev/null
+++ b/discovery/aws/metrics_lightsail.go
@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aws
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+type lightsailMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+var _ discovery.DiscovererMetrics = (*lightsailMetrics)(nil)
+
+// Register implements discovery.DiscovererMetrics.
+func (m *lightsailMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *lightsailMetrics) Unregister() {}
diff --git a/discovery/azure/azure.go b/discovery/azure/azure.go
index 64f02ec7a..e5e6109b2 100644
--- a/discovery/azure/azure.go
+++ b/discovery/azure/azure.go
@@ -120,12 +120,17 @@ type SDConfig struct {
 	HTTPClientConfig config_util.HTTPClientConfig `yaml:",inline"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return newDiscovererMetrics(reg, rmi)
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "azure" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.Registerer)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 func validateAuthParam(param, name string) error {
@@ -168,45 +173,39 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 
 type Discovery struct {
 	*refresh.Discovery
-	logger        log.Logger
-	cfg           *SDConfig
-	port          int
-	cache         *cache.Cache[string, *armnetwork.Interface]
-	failuresCount prometheus.Counter
-	cacheHitCount prometheus.Counter
+	logger  log.Logger
+	cfg     *SDConfig
+	port    int
+	cache   *cache.Cache[string, *armnetwork.Interface]
+	metrics *azureMetrics
 }
 
 // NewDiscovery returns a new AzureDiscovery which periodically refreshes its targets.
-func NewDiscovery(cfg *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(cfg *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*azureMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
 	l := cache.New(cache.AsLRU[string, *armnetwork.Interface](lru.WithCapacity(5000)))
 	d := &Discovery{
-		cfg:    cfg,
-		port:   cfg.Port,
-		logger: logger,
-		cache:  l,
-		failuresCount: prometheus.NewCounter(
-			prometheus.CounterOpts{
-				Name: "prometheus_sd_azure_failures_total",
-				Help: "Number of Azure service discovery refresh failures.",
-			}),
-		cacheHitCount: prometheus.NewCounter(
-			prometheus.CounterOpts{
-				Name: "prometheus_sd_azure_cache_hit_total",
-				Help: "Number of cache hit during refresh.",
-			}),
+		cfg:     cfg,
+		port:    cfg.Port,
+		logger:  logger,
+		cache:   l,
+		metrics: m,
 	}
 
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
-			Mech:     "azure",
-			Interval: time.Duration(cfg.RefreshInterval),
-			RefreshF: d.refresh,
-			Registry: reg,
-			Metrics:  []prometheus.Collector{d.failuresCount, d.cacheHitCount},
+			Logger:              logger,
+			Mech:                "azure",
+			Interval:            time.Duration(cfg.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
 
@@ -333,14 +332,14 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 
 	client, err := createAzureClient(*d.cfg)
 	if err != nil {
-		d.failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, fmt.Errorf("could not create Azure client: %w", err)
 	}
 	client.logger = d.logger
 
 	machines, err := client.getVMs(ctx, d.cfg.ResourceGroup)
 	if err != nil {
-		d.failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, fmt.Errorf("could not get virtual machines: %w", err)
 	}
 
@@ -349,14 +348,14 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 	// Load the vms managed by scale sets.
 	scaleSets, err := client.getScaleSets(ctx, d.cfg.ResourceGroup)
 	if err != nil {
-		d.failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, fmt.Errorf("could not get virtual machine scale sets: %w", err)
 	}
 
 	for _, scaleSet := range scaleSets {
 		scaleSetVms, err := client.getScaleSetVMs(ctx, scaleSet)
 		if err != nil {
-			d.failuresCount.Inc()
+			d.metrics.failuresCount.Inc()
 			return nil, fmt.Errorf("could not get virtual machine scale set vms: %w", err)
 		}
 		machines = append(machines, scaleSetVms...)
@@ -407,7 +406,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 				var networkInterface *armnetwork.Interface
 				if v, ok := d.getFromCache(nicID); ok {
 					networkInterface = v
-					d.cacheHitCount.Add(1)
+					d.metrics.cacheHitCount.Add(1)
 				} else {
 					if vm.ScaleSet == "" {
 						networkInterface, err = client.getVMNetworkInterfaceByID(ctx, nicID)
@@ -480,7 +479,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 	var tg targetgroup.Group
 	for tgt := range ch {
 		if tgt.err != nil {
-			d.failuresCount.Inc()
+			d.metrics.failuresCount.Inc()
 			return nil, fmt.Errorf("unable to complete Azure service discovery: %w", tgt.err)
 		}
 		if tgt.labelSet != nil {
diff --git a/discovery/azure/metrics.go b/discovery/azure/metrics.go
new file mode 100644
index 000000000..3e3dbdbfb
--- /dev/null
+++ b/discovery/azure/metrics.go
@@ -0,0 +1,64 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package azure
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*azureMetrics)(nil)
+
+type azureMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+
+	failuresCount prometheus.Counter
+	cacheHitCount prometheus.Counter
+
+	metricRegisterer discovery.MetricRegisterer
+}
+
+func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	m := &azureMetrics{
+		refreshMetrics: rmi,
+		failuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_azure_failures_total",
+				Help: "Number of Azure service discovery refresh failures.",
+			}),
+		cacheHitCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_azure_cache_hit_total",
+				Help: "Number of cache hit during refresh.",
+			}),
+	}
+
+	m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
+		m.failuresCount,
+		m.cacheHitCount,
+	})
+
+	return m
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *azureMetrics) Register() error {
+	return m.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *azureMetrics) Unregister() {
+	m.metricRegisterer.UnregisterMetrics()
+}
diff --git a/discovery/consul/consul.go b/discovery/consul/consul.go
index 50f171a78..40eed7697 100644
--- a/discovery/consul/consul.go
+++ b/discovery/consul/consul.go
@@ -119,12 +119,17 @@ type SDConfig struct {
 	HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return newDiscovererMetrics(reg, rmi)
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "consul" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.Registerer)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -161,27 +166,28 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 // Discovery retrieves target information from a Consul server
 // and updates them via watches.
 type Discovery struct {
-	client              *consul.Client
-	clientDatacenter    string
-	clientNamespace     string
-	clientPartition     string
-	tagSeparator        string
-	watchedServices     []string // Set of services which will be discovered.
-	watchedTags         []string // Tags used to filter instances of a service.
-	watchedNodeMeta     map[string]string
-	allowStale          bool
-	refreshInterval     time.Duration
-	finalizer           func()
-	logger              log.Logger
-	rpcFailuresCount    prometheus.Counter
-	rpcDuration         *prometheus.SummaryVec
-	servicesRPCDuration prometheus.Observer
-	serviceRPCDuration  prometheus.Observer
-	metricRegisterer    discovery.MetricRegisterer
+	client           *consul.Client
+	clientDatacenter string
+	clientNamespace  string
+	clientPartition  string
+	tagSeparator     string
+	watchedServices  []string // Set of services which will be discovered.
+	watchedTags      []string // Tags used to filter instances of a service.
+	watchedNodeMeta  map[string]string
+	allowStale       bool
+	refreshInterval  time.Duration
+	finalizer        func()
+	logger           log.Logger
+	metrics          *consulMetrics
 }
 
 // NewDiscovery returns a new Discovery for the given config.
-func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*consulMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -219,35 +225,9 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer)
 		clientPartition:  conf.Partition,
 		finalizer:        wrapper.CloseIdleConnections,
 		logger:           logger,
-		rpcFailuresCount: prometheus.NewCounter(
-			prometheus.CounterOpts{
-				Namespace: namespace,
-				Name:      "sd_consul_rpc_failures_total",
-				Help:      "The number of Consul RPC call failures.",
-			}),
-		rpcDuration: prometheus.NewSummaryVec(
-			prometheus.SummaryOpts{
-				Namespace:  namespace,
-				Name:       "sd_consul_rpc_duration_seconds",
-				Help:       "The duration of a Consul RPC call in seconds.",
-				Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
-			},
-			[]string{"endpoint", "call"},
-		),
+		metrics:          m,
 	}
 
-	cd.metricRegisterer = discovery.NewMetricRegisterer(
-		reg,
-		[]prometheus.Collector{
-			cd.rpcFailuresCount,
-			cd.rpcDuration,
-		},
-	)
-
-	// Initialize metric vectors.
-	cd.servicesRPCDuration = cd.rpcDuration.WithLabelValues("catalog", "services")
-	cd.serviceRPCDuration = cd.rpcDuration.WithLabelValues("catalog", "service")
-
 	return cd, nil
 }
 
@@ -303,7 +283,7 @@ func (d *Discovery) getDatacenter() error {
 	info, err := d.client.Agent().Self()
 	if err != nil {
 		level.Error(d.logger).Log("msg", "Error retrieving datacenter name", "err", err)
-		d.rpcFailuresCount.Inc()
+		d.metrics.rpcFailuresCount.Inc()
 		return err
 	}
 
@@ -344,13 +324,6 @@ func (d *Discovery) initialize(ctx context.Context) {
 
 // Run implements the Discoverer interface.
 func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
-	err := d.metricRegisterer.RegisterMetrics()
-	if err != nil {
-		level.Error(d.logger).Log("msg", "Unable to register metrics", "err", err.Error())
-		return
-	}
-	defer d.metricRegisterer.UnregisterMetrics()
-
 	if d.finalizer != nil {
 		defer d.finalizer()
 	}
@@ -399,7 +372,7 @@ func (d *Discovery) watchServices(ctx context.Context, ch chan<- []*targetgroup.
 	t0 := time.Now()
 	srvs, meta, err := catalog.Services(opts.WithContext(ctx))
 	elapsed := time.Since(t0)
-	d.servicesRPCDuration.Observe(elapsed.Seconds())
+	d.metrics.servicesRPCDuration.Observe(elapsed.Seconds())
 
 	// Check the context before in order to exit early.
 	select {
@@ -410,7 +383,7 @@ func (d *Discovery) watchServices(ctx context.Context, ch chan<- []*targetgroup.
 
 	if err != nil {
 		level.Error(d.logger).Log("msg", "Error refreshing service list", "err", err)
-		d.rpcFailuresCount.Inc()
+		d.metrics.rpcFailuresCount.Inc()
 		time.Sleep(retryInterval)
 		return
 	}
@@ -490,8 +463,8 @@ func (d *Discovery) watchService(ctx context.Context, ch chan<- []*targetgroup.G
 		},
 		tagSeparator:       d.tagSeparator,
 		logger:             d.logger,
-		rpcFailuresCount:   d.rpcFailuresCount,
-		serviceRPCDuration: d.serviceRPCDuration,
+		rpcFailuresCount:   d.metrics.rpcFailuresCount,
+		serviceRPCDuration: d.metrics.serviceRPCDuration,
 	}
 
 	go func() {
diff --git a/discovery/consul/consul_test.go b/discovery/consul/consul_test.go
index 97cb8fbc9..040040ae9 100644
--- a/discovery/consul/consul_test.go
+++ b/discovery/consul/consul_test.go
@@ -29,6 +29,7 @@ import (
 	"go.uber.org/goleak"
 	"gopkg.in/yaml.v2"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -36,11 +37,25 @@ func TestMain(m *testing.M) {
 	goleak.VerifyTestMain(m)
 }
 
+// TODO: Add ability to unregister metrics?
+func NewTestMetrics(t *testing.T, conf discovery.Config, reg prometheus.Registerer) discovery.DiscovererMetrics {
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	require.NoError(t, refreshMetrics.Register())
+
+	metrics := conf.NewDiscovererMetrics(prometheus.NewRegistry(), refreshMetrics)
+	require.NoError(t, metrics.Register())
+
+	return metrics
+}
+
 func TestConfiguredService(t *testing.T) {
 	conf := &SDConfig{
 		Services: []string{"configuredServiceName"},
 	}
-	consulDiscovery, err := NewDiscovery(conf, nil, prometheus.NewRegistry())
+
+	metrics := NewTestMetrics(t, conf, prometheus.NewRegistry())
+
+	consulDiscovery, err := NewDiscovery(conf, nil, metrics)
 	if err != nil {
 		t.Errorf("Unexpected error when initializing discovery %v", err)
 	}
@@ -57,7 +72,10 @@ func TestConfiguredServiceWithTag(t *testing.T) {
 		Services:    []string{"configuredServiceName"},
 		ServiceTags: []string{"http"},
 	}
-	consulDiscovery, err := NewDiscovery(conf, nil, prometheus.NewRegistry())
+
+	metrics := NewTestMetrics(t, conf, prometheus.NewRegistry())
+
+	consulDiscovery, err := NewDiscovery(conf, nil, metrics)
 	if err != nil {
 		t.Errorf("Unexpected error when initializing discovery %v", err)
 	}
@@ -152,7 +170,9 @@ func TestConfiguredServiceWithTags(t *testing.T) {
 	}
 
 	for _, tc := range cases {
-		consulDiscovery, err := NewDiscovery(tc.conf, nil, prometheus.NewRegistry())
+		metrics := NewTestMetrics(t, tc.conf, prometheus.NewRegistry())
+
+		consulDiscovery, err := NewDiscovery(tc.conf, nil, metrics)
 		if err != nil {
 			t.Errorf("Unexpected error when initializing discovery %v", err)
 		}
@@ -160,13 +180,15 @@ func TestConfiguredServiceWithTags(t *testing.T) {
 		if ret != tc.shouldWatch {
 			t.Errorf("Expected should watch? %t, got %t. Watched service and tags: %s %+v, input was %s %+v", tc.shouldWatch, ret, tc.conf.Services, tc.conf.ServiceTags, tc.serviceName, tc.serviceTags)
 		}
-
 	}
 }
 
 func TestNonConfiguredService(t *testing.T) {
 	conf := &SDConfig{}
-	consulDiscovery, err := NewDiscovery(conf, nil, prometheus.NewRegistry())
+
+	metrics := NewTestMetrics(t, conf, prometheus.NewRegistry())
+
+	consulDiscovery, err := NewDiscovery(conf, nil, metrics)
 	if err != nil {
 		t.Errorf("Unexpected error when initializing discovery %v", err)
 	}
@@ -263,7 +285,10 @@ func newServer(t *testing.T) (*httptest.Server, *SDConfig) {
 
 func newDiscovery(t *testing.T, config *SDConfig) *Discovery {
 	logger := log.NewNopLogger()
-	d, err := NewDiscovery(config, logger, prometheus.NewRegistry())
+
+	metrics := NewTestMetrics(t, config, prometheus.NewRegistry())
+
+	d, err := NewDiscovery(config, logger, metrics)
 	require.NoError(t, err)
 	return d
 }
diff --git a/discovery/consul/metrics.go b/discovery/consul/metrics.go
new file mode 100644
index 000000000..8266e7cc6
--- /dev/null
+++ b/discovery/consul/metrics.go
@@ -0,0 +1,73 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package consul
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*consulMetrics)(nil)
+
+type consulMetrics struct {
+	rpcFailuresCount prometheus.Counter
+	rpcDuration      *prometheus.SummaryVec
+
+	servicesRPCDuration prometheus.Observer
+	serviceRPCDuration  prometheus.Observer
+
+	metricRegisterer discovery.MetricRegisterer
+}
+
+func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	m := &consulMetrics{
+		rpcFailuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      "sd_consul_rpc_failures_total",
+				Help:      "The number of Consul RPC call failures.",
+			}),
+		rpcDuration: prometheus.NewSummaryVec(
+			prometheus.SummaryOpts{
+				Namespace:  namespace,
+				Name:       "sd_consul_rpc_duration_seconds",
+				Help:       "The duration of a Consul RPC call in seconds.",
+				Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+			},
+			[]string{"endpoint", "call"},
+		),
+	}
+
+	m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
+		m.rpcFailuresCount,
+		m.rpcDuration,
+	})
+
+	// Initialize metric vectors.
+	m.servicesRPCDuration = m.rpcDuration.WithLabelValues("catalog", "services")
+	m.serviceRPCDuration = m.rpcDuration.WithLabelValues("catalog", "service")
+
+	return m
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *consulMetrics) Register() error {
+	return m.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *consulMetrics) Unregister() {
+	m.metricRegisterer.UnregisterMetrics()
+}
diff --git a/discovery/digitalocean/digitalocean.go b/discovery/digitalocean/digitalocean.go
index 970258de0..18380b729 100644
--- a/discovery/digitalocean/digitalocean.go
+++ b/discovery/digitalocean/digitalocean.go
@@ -63,6 +63,13 @@ func init() {
 	discovery.RegisterConfig(&SDConfig{})
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &digitaloceanMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // SDConfig is the configuration for DigitalOcean based service discovery.
 type SDConfig struct {
 	HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
@@ -76,7 +83,7 @@ func (*SDConfig) Name() string { return "digitalocean" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.Registerer)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -104,7 +111,12 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*digitaloceanMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	d := &Discovery{
 		port: conf.Port,
 	}
@@ -127,11 +139,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer)
 
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
-			Mech:     "digitalocean",
-			Interval: time.Duration(conf.RefreshInterval),
-			RefreshF: d.refresh,
-			Registry: reg,
+			Logger:              logger,
+			Mech:                "digitalocean",
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
 	return d, nil
diff --git a/discovery/digitalocean/digitalocean_test.go b/discovery/digitalocean/digitalocean_test.go
index a959b312c..841b5ef97 100644
--- a/discovery/digitalocean/digitalocean_test.go
+++ b/discovery/digitalocean/digitalocean_test.go
@@ -23,6 +23,8 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/prometheus/discovery"
 )
 
 type DigitalOceanSDTestSuite struct {
@@ -47,7 +49,15 @@ func TestDigitalOceanSDRefresh(t *testing.T) {
 
 	cfg := DefaultSDConfig
 	cfg.HTTPClientConfig.BearerToken = tokenID
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 	endpoint, err := url.Parse(sdmock.Mock.Endpoint())
 	require.NoError(t, err)
diff --git a/discovery/digitalocean/metrics.go b/discovery/digitalocean/metrics.go
new file mode 100644
index 000000000..91cfd9bf3
--- /dev/null
+++ b/discovery/digitalocean/metrics.go
@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package digitalocean
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*digitaloceanMetrics)(nil)
+
+type digitaloceanMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *digitaloceanMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *digitaloceanMetrics) Unregister() {}
diff --git a/discovery/discoverer_metrics_noop.go b/discovery/discoverer_metrics_noop.go
new file mode 100644
index 000000000..638317ace
--- /dev/null
+++ b/discovery/discoverer_metrics_noop.go
@@ -0,0 +1,28 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package discovery
+
+// Create a dummy metrics struct, because this SD doesn't have any metrics.
+type NoopDiscovererMetrics struct{}
+
+var _ DiscovererMetrics = (*NoopDiscovererMetrics)(nil)
+
+// Register implements discovery.DiscovererMetrics.
+func (*NoopDiscovererMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (*NoopDiscovererMetrics) Unregister() {
+}
diff --git a/discovery/discovery.go b/discovery/discovery.go
index acc4c1efe..a5826f817 100644
--- a/discovery/discovery.go
+++ b/discovery/discovery.go
@@ -39,24 +39,47 @@ type Discoverer interface {
 	Run(ctx context.Context, up chan<- []*targetgroup.Group)
 }
 
+// Internal metrics of service discovery mechanisms.
+type DiscovererMetrics interface {
+	Register() error
+	Unregister()
+}
+
 // DiscovererOptions provides options for a Discoverer.
 type DiscovererOptions struct {
 	Logger log.Logger
 
-	// A registerer for the Discoverer's metrics.
-	// Some Discoverers may ignore this registerer and use the global one instead.
-	// For now this will work, because the Prometheus `main` function uses the global registry.
-	// However, in the future the Prometheus `main` function will be updated to not use the global registry.
-	// Hence, if a discoverer wants its metrics to be visible via the Prometheus executable's
-	// `/metrics` endpoint, it should use this explicit registerer.
-	// TODO(ptodev): Update this comment once the Prometheus `main` function does not use the global registry.
-	Registerer prometheus.Registerer
+	Metrics DiscovererMetrics
 
 	// Extra HTTP client options to expose to Discoverers. This field may be
 	// ignored; Discoverer implementations must opt-in to reading it.
 	HTTPClientOptions []config.HTTPClientOption
 }
 
+// Metrics used by the "refresh" package.
+// We define them here in the "discovery" package in order to avoid a cyclic dependency between
+// "discovery" and "refresh".
+type RefreshMetrics struct {
+	Failures prometheus.Counter
+	Duration prometheus.Observer
+}
+
+// Instantiate the metrics used by the "refresh" package.
+type RefreshMetricsInstantiator interface {
+	Instantiate(mech string) *RefreshMetrics
+}
+
+// An interface for registering, unregistering, and instantiating metrics for the "refresh" package.
+// Refresh metrics are registered and unregistered outside of the service discovery mechanism.
+// This is so that the same metrics can be reused across different service discovery mechanisms.
+// To manage refresh metrics inside the SD mechanism, we'd need to use const labels which are
+// specific to that SD. However, doing so would also expose too many unused metrics on
+// the Prometheus /metrics endpoint.
+type RefreshMetricsManager interface {
+	DiscovererMetrics
+	RefreshMetricsInstantiator
+}
+
 // A Config provides the configuration and constructor for a Discoverer.
 type Config interface {
 	// Name returns the name of the discovery mechanism.
@@ -65,6 +88,9 @@ type Config interface {
 	// NewDiscoverer returns a Discoverer for the Config
 	// with the given DiscovererOptions.
 	NewDiscoverer(DiscovererOptions) (Discoverer, error)
+
+	// NewDiscovererMetrics returns the metrics used by the service discovery.
+	NewDiscovererMetrics(prometheus.Registerer, RefreshMetricsInstantiator) DiscovererMetrics
 }
 
 // Configs is a slice of Config values that uses custom YAML marshaling and unmarshaling
@@ -119,6 +145,11 @@ func (c StaticConfig) NewDiscoverer(DiscovererOptions) (Discoverer, error) {
 	return staticDiscoverer(c), nil
 }
 
+// No metrics are needed for this service discovery mechanism.
+func (c StaticConfig) NewDiscovererMetrics(prometheus.Registerer, RefreshMetricsInstantiator) DiscovererMetrics {
+	return &NoopDiscovererMetrics{}
+}
+
 type staticDiscoverer []*targetgroup.Group
 
 func (c staticDiscoverer) Run(ctx context.Context, up chan<- []*targetgroup.Group) {
diff --git a/discovery/dns/dns.go b/discovery/dns/dns.go
index 9b6bd6741..cf56a2ad0 100644
--- a/discovery/dns/dns.go
+++ b/discovery/dns/dns.go
@@ -67,12 +67,17 @@ type SDConfig struct {
 	Port            int            `yaml:"port"` // Ignored for SRV records
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return newDiscovererMetrics(reg, rmi)
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "dns" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(*c, opts.Logger, opts.Registerer)
+	return NewDiscovery(*c, opts.Logger, opts.Metrics)
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
@@ -102,18 +107,22 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 // the Discoverer interface.
 type Discovery struct {
 	*refresh.Discovery
-	names                    []string
-	port                     int
-	qtype                    uint16
-	logger                   log.Logger
-	dnsSDLookupsCount        prometheus.Counter
-	dnsSDLookupFailuresCount prometheus.Counter
+	names   []string
+	port    int
+	qtype   uint16
+	logger  log.Logger
+	metrics *dnsMetrics
 
 	lookupFn func(name string, qtype uint16, logger log.Logger) (*dns.Msg, error)
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(conf SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*dnsMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -137,28 +146,16 @@ func NewDiscovery(conf SDConfig, logger log.Logger, reg prometheus.Registerer) (
 		port:     conf.Port,
 		logger:   logger,
 		lookupFn: lookupWithSearchPath,
-		dnsSDLookupsCount: prometheus.NewCounter(
-			prometheus.CounterOpts{
-				Namespace: namespace,
-				Name:      "sd_dns_lookups_total",
-				Help:      "The number of DNS-SD lookups.",
-			}),
-		dnsSDLookupFailuresCount: prometheus.NewCounter(
-			prometheus.CounterOpts{
-				Namespace: namespace,
-				Name:      "sd_dns_lookup_failures_total",
-				Help:      "The number of DNS-SD lookup failures.",
-			}),
+		metrics:  m,
 	}
 
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
-			Mech:     "dns",
-			Interval: time.Duration(conf.RefreshInterval),
-			RefreshF: d.refresh,
-			Registry: prometheus.NewRegistry(),
-			Metrics:  []prometheus.Collector{d.dnsSDLookupsCount, d.dnsSDLookupFailuresCount},
+			Logger:              logger,
+			Mech:                "dns",
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
 
@@ -195,9 +192,9 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 
 func (d *Discovery) refreshOne(ctx context.Context, name string, ch chan<- *targetgroup.Group) error {
 	response, err := d.lookupFn(name, d.qtype, d.logger)
-	d.dnsSDLookupsCount.Inc()
+	d.metrics.dnsSDLookupsCount.Inc()
 	if err != nil {
-		d.dnsSDLookupFailuresCount.Inc()
+		d.metrics.dnsSDLookupFailuresCount.Inc()
 		return err
 	}
 
diff --git a/discovery/dns/dns_test.go b/discovery/dns/dns_test.go
index b8dd2efaa..33a976827 100644
--- a/discovery/dns/dns_test.go
+++ b/discovery/dns/dns_test.go
@@ -28,6 +28,7 @@ import (
 	"go.uber.org/goleak"
 	"gopkg.in/yaml.v2"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -253,13 +254,21 @@ func TestDNS(t *testing.T) {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
-			sd, err := NewDiscovery(tc.config, nil, prometheus.NewRegistry())
+
+			reg := prometheus.NewRegistry()
+			refreshMetrics := discovery.NewRefreshMetrics(reg)
+			metrics := tc.config.NewDiscovererMetrics(reg, refreshMetrics)
+			require.NoError(t, metrics.Register())
+
+			sd, err := NewDiscovery(tc.config, nil, metrics)
 			require.NoError(t, err)
 			sd.lookupFn = tc.lookup
 
 			tgs, err := sd.refresh(context.Background())
 			require.NoError(t, err)
 			require.Equal(t, tc.expected, tgs)
+
+			metrics.Unregister()
 		})
 	}
 }
diff --git a/discovery/dns/metrics.go b/discovery/dns/metrics.go
new file mode 100644
index 000000000..27c96b53e
--- /dev/null
+++ b/discovery/dns/metrics.go
@@ -0,0 +1,66 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dns
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*dnsMetrics)(nil)
+
+type dnsMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+
+	dnsSDLookupsCount        prometheus.Counter
+	dnsSDLookupFailuresCount prometheus.Counter
+
+	metricRegisterer discovery.MetricRegisterer
+}
+
+func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	m := &dnsMetrics{
+		refreshMetrics: rmi,
+		dnsSDLookupsCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      "sd_dns_lookups_total",
+				Help:      "The number of DNS-SD lookups.",
+			}),
+		dnsSDLookupFailuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      "sd_dns_lookup_failures_total",
+				Help:      "The number of DNS-SD lookup failures.",
+			}),
+	}
+
+	m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
+		m.dnsSDLookupsCount,
+		m.dnsSDLookupFailuresCount,
+	})
+
+	return m
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *dnsMetrics) Register() error {
+	return m.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *dnsMetrics) Unregister() {
+	m.metricRegisterer.UnregisterMetrics()
+}
diff --git a/discovery/eureka/eureka.go b/discovery/eureka/eureka.go
index d3e4084e5..deed89f6f 100644
--- a/discovery/eureka/eureka.go
+++ b/discovery/eureka/eureka.go
@@ -16,6 +16,7 @@ package eureka
 import (
 	"context"
 	"errors"
+	"fmt"
 	"net"
 	"net/http"
 	"net/url"
@@ -76,12 +77,19 @@ type SDConfig struct {
 	RefreshInterval  model.Duration          `yaml:"refresh_interval,omitempty"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &eurekaMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "eureka" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.Registerer)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -118,7 +126,12 @@ type Discovery struct {
 }
 
 // NewDiscovery creates a new Eureka discovery for the given role.
-func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*eurekaMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "eureka_sd")
 	if err != nil {
 		return nil, err
@@ -130,11 +143,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer)
 	}
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
-			Mech:     "eureka",
-			Interval: time.Duration(conf.RefreshInterval),
-			RefreshF: d.refresh,
-			Registry: reg,
+			Logger:              logger,
+			Mech:                "eureka",
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
 	return d, nil
diff --git a/discovery/eureka/eureka_test.go b/discovery/eureka/eureka_test.go
index 1fe3c710e..b499410bf 100644
--- a/discovery/eureka/eureka_test.go
+++ b/discovery/eureka/eureka_test.go
@@ -24,6 +24,7 @@ import (
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -36,7 +37,17 @@ func testUpdateServices(respHandler http.HandlerFunc) ([]*targetgroup.Group, err
 		Server: ts.URL,
 	}
 
-	md, err := NewDiscovery(&conf, nil, prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := conf.NewDiscovererMetrics(reg, refreshMetrics)
+	err := metrics.Register()
+	if err != nil {
+		return nil, err
+	}
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	md, err := NewDiscovery(&conf, nil, metrics)
 	if err != nil {
 		return nil, err
 	}
diff --git a/discovery/eureka/metrics.go b/discovery/eureka/metrics.go
new file mode 100644
index 000000000..6d6463ccd
--- /dev/null
+++ b/discovery/eureka/metrics.go
@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package eureka
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*eurekaMetrics)(nil)
+
+type eurekaMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *eurekaMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *eurekaMetrics) Unregister() {}
diff --git a/discovery/file/file.go b/discovery/file/file.go
index ef6ed1f5e..e7e9d0870 100644
--- a/discovery/file/file.go
+++ b/discovery/file/file.go
@@ -57,12 +57,17 @@ type SDConfig struct {
 	RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return newDiscovererMetrics(reg, rmi)
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "file" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.Registerer)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -94,6 +99,9 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 const fileSDFilepathLabel = model.MetaLabelPrefix + "filepath"
 
 // TimestampCollector is a Custom Collector for Timestamps of the files.
+// TODO(ptodev): Now that each file SD has its own TimestampCollector
+// inside discovery/file/metrics.go, we can refactor this collector
+// (or get rid of it) as each TimestampCollector instance will only use one discoverer.
 type TimestampCollector struct {
 	Description *prometheus.Desc
 	discoverers map[*Discovery]struct{}
@@ -169,16 +177,16 @@ type Discovery struct {
 	lastRefresh map[string]int
 	logger      log.Logger
 
-	fileSDReadErrorsCount  prometheus.Counter
-	fileSDScanDuration     prometheus.Summary
-	fileWatcherErrorsCount prometheus.Counter
-	fileSDTimeStamp        *TimestampCollector
-
-	metricRegisterer discovery.MetricRegisterer
+	metrics *fileMetrics
 }
 
 // NewDiscovery returns a new file discovery for the given paths.
-func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	fm, ok := metrics.(*fileMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -188,33 +196,10 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer)
 		interval:   time.Duration(conf.RefreshInterval),
 		timestamps: make(map[string]float64),
 		logger:     logger,
-		fileSDReadErrorsCount: prometheus.NewCounter(
-			prometheus.CounterOpts{
-				Name: "prometheus_sd_file_read_errors_total",
-				Help: "The number of File-SD read errors.",
-			}),
-		fileSDScanDuration: prometheus.NewSummary(
-			prometheus.SummaryOpts{
-				Name:       "prometheus_sd_file_scan_duration_seconds",
-				Help:       "The duration of the File-SD scan in seconds.",
-				Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
-			}),
-		fileWatcherErrorsCount: prometheus.NewCounter(
-			prometheus.CounterOpts{
-				Name: "prometheus_sd_file_watcher_errors_total",
-				Help: "The number of File-SD errors caused by filesystem watch failures.",
-			}),
-		fileSDTimeStamp: NewTimestampCollector(),
+		metrics:    fm,
 	}
 
-	disc.fileSDTimeStamp.addDiscoverer(disc)
-
-	disc.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
-		disc.fileSDReadErrorsCount,
-		disc.fileSDScanDuration,
-		disc.fileWatcherErrorsCount,
-		disc.fileSDTimeStamp,
-	})
+	fm.init(disc)
 
 	return disc, nil
 }
@@ -253,17 +238,10 @@ func (d *Discovery) watchFiles() {
 
 // Run implements the Discoverer interface.
 func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
-	err := d.metricRegisterer.RegisterMetrics()
-	if err != nil {
-		level.Error(d.logger).Log("msg", "Unable to register metrics", "err", err.Error())
-		return
-	}
-	defer d.metricRegisterer.UnregisterMetrics()
-
 	watcher, err := fsnotify.NewWatcher()
 	if err != nil {
 		level.Error(d.logger).Log("msg", "Error adding file watcher", "err", err)
-		d.fileWatcherErrorsCount.Inc()
+		d.metrics.fileWatcherErrorsCount.Inc()
 		return
 	}
 	d.watcher = watcher
@@ -327,7 +305,7 @@ func (d *Discovery) stop() {
 	done := make(chan struct{})
 	defer close(done)
 
-	d.fileSDTimeStamp.removeDiscoverer(d)
+	d.metrics.fileSDTimeStamp.removeDiscoverer(d)
 
 	// Closing the watcher will deadlock unless all events and errors are drained.
 	go func() {
@@ -353,13 +331,13 @@ func (d *Discovery) stop() {
 func (d *Discovery) refresh(ctx context.Context, ch chan<- []*targetgroup.Group) {
 	t0 := time.Now()
 	defer func() {
-		d.fileSDScanDuration.Observe(time.Since(t0).Seconds())
+		d.metrics.fileSDScanDuration.Observe(time.Since(t0).Seconds())
 	}()
 	ref := map[string]int{}
 	for _, p := range d.listFiles() {
 		tgroups, err := d.readFile(p)
 		if err != nil {
-			d.fileSDReadErrorsCount.Inc()
+			d.metrics.fileSDReadErrorsCount.Inc()
 
 			level.Error(d.logger).Log("msg", "Error reading file", "path", p, "err", err)
 			// Prevent deletion down below.
diff --git a/discovery/file/file_test.go b/discovery/file/file_test.go
index 731611832..8a5057089 100644
--- a/discovery/file/file_test.go
+++ b/discovery/file/file_test.go
@@ -29,6 +29,7 @@ import (
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -144,19 +145,28 @@ func (t *testRunner) run(files ...string) {
 	ctx, cancel := context.WithCancel(context.Background())
 	t.cancelSD = cancel
 	go func() {
+		conf := &SDConfig{
+			Files: files,
+			// Setting a high refresh interval to make sure that the tests only
+			// rely on file watches.
+			RefreshInterval: model.Duration(1 * time.Hour),
+		}
+
+		reg := prometheus.NewRegistry()
+		refreshMetrics := discovery.NewRefreshMetrics(reg)
+		metrics := conf.NewDiscovererMetrics(reg, refreshMetrics)
+		require.NoError(t, metrics.Register())
+
 		d, err := NewDiscovery(
-			&SDConfig{
-				Files: files,
-				// Setting a high refresh interval to make sure that the tests only
-				// rely on file watches.
-				RefreshInterval: model.Duration(1 * time.Hour),
-			},
+			conf,
 			nil,
-			prometheus.NewRegistry(),
+			metrics,
 		)
 		require.NoError(t, err)
 
 		d.Run(ctx, t.ch)
+
+		metrics.Unregister()
 	}()
 }
 
diff --git a/discovery/file/metrics.go b/discovery/file/metrics.go
new file mode 100644
index 000000000..c01501e4e
--- /dev/null
+++ b/discovery/file/metrics.go
@@ -0,0 +1,76 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package file
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*fileMetrics)(nil)
+
+type fileMetrics struct {
+	fileSDReadErrorsCount  prometheus.Counter
+	fileSDScanDuration     prometheus.Summary
+	fileWatcherErrorsCount prometheus.Counter
+	fileSDTimeStamp        *TimestampCollector
+
+	metricRegisterer discovery.MetricRegisterer
+}
+
+func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	fm := &fileMetrics{
+		fileSDReadErrorsCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_file_read_errors_total",
+				Help: "The number of File-SD read errors.",
+			}),
+		fileSDScanDuration: prometheus.NewSummary(
+			prometheus.SummaryOpts{
+				Name:       "prometheus_sd_file_scan_duration_seconds",
+				Help:       "The duration of the File-SD scan in seconds.",
+				Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+			}),
+		fileWatcherErrorsCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_file_watcher_errors_total",
+				Help: "The number of File-SD errors caused by filesystem watch failures.",
+			}),
+		fileSDTimeStamp: NewTimestampCollector(),
+	}
+
+	fm.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
+		fm.fileSDReadErrorsCount,
+		fm.fileSDScanDuration,
+		fm.fileWatcherErrorsCount,
+		fm.fileSDTimeStamp,
+	})
+
+	return fm
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (fm *fileMetrics) Register() error {
+	return fm.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (fm *fileMetrics) Unregister() {
+	fm.metricRegisterer.UnregisterMetrics()
+}
+
+func (fm *fileMetrics) init(disc *Discovery) {
+	fm.fileSDTimeStamp.addDiscoverer(disc)
+}
diff --git a/discovery/gce/gce.go b/discovery/gce/gce.go
index 21a95ee39..15f32dd24 100644
--- a/discovery/gce/gce.go
+++ b/discovery/gce/gce.go
@@ -82,12 +82,19 @@ type SDConfig struct {
 	TagSeparator    string         `yaml:"tag_separator,omitempty"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &gceMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "gce" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(*c, opts.Logger, opts.Registerer)
+	return NewDiscovery(*c, opts.Logger, opts.Metrics)
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
@@ -122,7 +129,12 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(conf SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*gceMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	d := &Discovery{
 		project:      conf.Project,
 		zone:         conf.Zone,
@@ -143,11 +155,11 @@ func NewDiscovery(conf SDConfig, logger log.Logger, reg prometheus.Registerer) (
 
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
-			Mech:     "gce",
-			Interval: time.Duration(conf.RefreshInterval),
-			RefreshF: d.refresh,
-			Registry: reg,
+			Logger:              logger,
+			Mech:                "gce",
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
 	return d, nil
diff --git a/discovery/gce/metrics.go b/discovery/gce/metrics.go
new file mode 100644
index 000000000..f53ea5a8c
--- /dev/null
+++ b/discovery/gce/metrics.go
@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gce
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*gceMetrics)(nil)
+
+type gceMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *gceMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *gceMetrics) Unregister() {}
diff --git a/discovery/hetzner/hetzner.go b/discovery/hetzner/hetzner.go
index 9d3e6aa65..69c823d38 100644
--- a/discovery/hetzner/hetzner.go
+++ b/discovery/hetzner/hetzner.go
@@ -63,12 +63,19 @@ type SDConfig struct {
 	robotEndpoint   string         // For tests only.
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &hetznerMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "hetzner" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.Registerer)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 type refresher interface {
@@ -128,7 +135,12 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*refresh.Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*refresh.Discovery, error) {
+	m, ok := metrics.(*hetznerMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	r, err := newRefresher(conf, logger)
 	if err != nil {
 		return nil, err
@@ -136,11 +148,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer)
 
 	return refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
-			Mech:     "hetzner",
-			Interval: time.Duration(conf.RefreshInterval),
-			RefreshF: r.refresh,
-			Registry: reg,
+			Logger:              logger,
+			Mech:                "hetzner",
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            r.refresh,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	), nil
 }
diff --git a/discovery/hetzner/metrics.go b/discovery/hetzner/metrics.go
new file mode 100644
index 000000000..27361ee75
--- /dev/null
+++ b/discovery/hetzner/metrics.go
@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hetzner
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*hetznerMetrics)(nil)
+
+type hetznerMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *hetznerMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *hetznerMetrics) Unregister() {}
diff --git a/discovery/http/http.go b/discovery/http/http.go
index c12fdb26d..8dd21ec9e 100644
--- a/discovery/http/http.go
+++ b/discovery/http/http.go
@@ -58,12 +58,17 @@ type SDConfig struct {
 	URL              string                  `yaml:"url"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return newDiscovererMetrics(reg, rmi)
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "http" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.HTTPClientOptions, opts.Registerer)
+	return NewDiscovery(c, opts.Logger, opts.HTTPClientOptions, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -105,11 +110,16 @@ type Discovery struct {
 	client          *http.Client
 	refreshInterval time.Duration
 	tgLastLength    int
-	failuresCount   prometheus.Counter
+	metrics         *httpMetrics
 }
 
 // NewDiscovery returns a new HTTP discovery for the given config.
-func NewDiscovery(conf *SDConfig, logger log.Logger, clientOpts []config.HTTPClientOption, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, clientOpts []config.HTTPClientOption, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*httpMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -124,21 +134,16 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, clientOpts []config.HTTPCli
 		url:             conf.URL,
 		client:          client,
 		refreshInterval: time.Duration(conf.RefreshInterval), // Stored to be sent as headers.
-		failuresCount: prometheus.NewCounter(
-			prometheus.CounterOpts{
-				Name: "prometheus_sd_http_failures_total",
-				Help: "Number of HTTP service discovery refresh failures.",
-			}),
+		metrics:         m,
 	}
 
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
-			Mech:     "http",
-			Interval: time.Duration(conf.RefreshInterval),
-			RefreshF: d.Refresh,
-			Registry: reg,
-			Metrics:  []prometheus.Collector{d.failuresCount},
+			Logger:              logger,
+			Mech:                "http",
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            d.Refresh,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
 	return d, nil
@@ -155,7 +160,7 @@ func (d *Discovery) Refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 
 	resp, err := d.client.Do(req.WithContext(ctx))
 	if err != nil {
-		d.failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, err
 	}
 	defer func() {
@@ -164,31 +169,31 @@ func (d *Discovery) Refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 	}()
 
 	if resp.StatusCode != http.StatusOK {
-		d.failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, fmt.Errorf("server returned HTTP status %s", resp.Status)
 	}
 
 	if !matchContentType.MatchString(strings.TrimSpace(resp.Header.Get("Content-Type"))) {
-		d.failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, fmt.Errorf("unsupported content type %q", resp.Header.Get("Content-Type"))
 	}
 
 	b, err := io.ReadAll(resp.Body)
 	if err != nil {
-		d.failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, err
 	}
 
 	var targetGroups []*targetgroup.Group
 
 	if err := json.Unmarshal(b, &targetGroups); err != nil {
-		d.failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, err
 	}
 
 	for i, tg := range targetGroups {
 		if tg == nil {
-			d.failuresCount.Inc()
+			d.metrics.failuresCount.Inc()
 			err = errors.New("nil target group item found")
 			return nil, err
 		}
diff --git a/discovery/http/http_test.go b/discovery/http/http_test.go
index 164719e90..0cafe035d 100644
--- a/discovery/http/http_test.go
+++ b/discovery/http/http_test.go
@@ -28,6 +28,7 @@ import (
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -41,7 +42,14 @@ func TestHTTPValidRefresh(t *testing.T) {
 		RefreshInterval:  model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	defer refreshMetrics.Unregister()
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()
@@ -63,7 +71,7 @@ func TestHTTPValidRefresh(t *testing.T) {
 		},
 	}
 	require.Equal(t, expectedTargets, tgs)
-	require.Equal(t, 0.0, getFailureCount(d.failuresCount))
+	require.Equal(t, 0.0, getFailureCount(d.metrics.failuresCount))
 }
 
 func TestHTTPInvalidCode(t *testing.T) {
@@ -79,13 +87,20 @@ func TestHTTPInvalidCode(t *testing.T) {
 		RefreshInterval:  model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	defer refreshMetrics.Unregister()
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()
 	_, err = d.Refresh(ctx)
 	require.EqualError(t, err, "server returned HTTP status 400 Bad Request")
-	require.Equal(t, 1.0, getFailureCount(d.failuresCount))
+	require.Equal(t, 1.0, getFailureCount(d.metrics.failuresCount))
 }
 
 func TestHTTPInvalidFormat(t *testing.T) {
@@ -101,13 +116,20 @@ func TestHTTPInvalidFormat(t *testing.T) {
 		RefreshInterval:  model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	defer refreshMetrics.Unregister()
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()
 	_, err = d.Refresh(ctx)
 	require.EqualError(t, err, `unsupported content type "text/plain; charset=utf-8"`)
-	require.Equal(t, 1.0, getFailureCount(d.failuresCount))
+	require.Equal(t, 1.0, getFailureCount(d.metrics.failuresCount))
 }
 
 func getFailureCount(failuresCount prometheus.Counter) float64 {
@@ -412,7 +434,15 @@ func TestSourceDisappeared(t *testing.T) {
 		URL:              ts.URL,
 		RefreshInterval:  model.Duration(1 * time.Second),
 	}
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	defer refreshMetrics.Unregister()
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, metrics)
 	require.NoError(t, err)
 	for _, test := range cases {
 		ctx := context.Background()
diff --git a/discovery/http/metrics.go b/discovery/http/metrics.go
new file mode 100644
index 000000000..b1f8b8443
--- /dev/null
+++ b/discovery/http/metrics.go
@@ -0,0 +1,57 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package http
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*httpMetrics)(nil)
+
+type httpMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+
+	failuresCount prometheus.Counter
+
+	metricRegisterer discovery.MetricRegisterer
+}
+
+func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	m := &httpMetrics{
+		refreshMetrics: rmi,
+		failuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_http_failures_total",
+				Help: "Number of HTTP service discovery refresh failures.",
+			}),
+	}
+
+	m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
+		m.failuresCount,
+	})
+
+	return m
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *httpMetrics) Register() error {
+	return m.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *httpMetrics) Unregister() {
+	m.metricRegisterer.UnregisterMetrics()
+}
diff --git a/discovery/ionos/ionos.go b/discovery/ionos/ionos.go
index 36623745a..c8b4f7f8e 100644
--- a/discovery/ionos/ionos.go
+++ b/discovery/ionos/ionos.go
@@ -15,16 +15,16 @@ package ionos
 
 import (
 	"errors"
+	"fmt"
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
 	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/refresh"
-
-	"github.com/prometheus/client_golang/prometheus"
 )
 
 const (
@@ -43,7 +43,12 @@ func init() {
 type Discovery struct{}
 
 // NewDiscovery returns a new refresh.Discovery for IONOS Cloud.
-func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*refresh.Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*refresh.Discovery, error) {
+	m, ok := metrics.(*ionosMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if conf.ionosEndpoint == "" {
 		conf.ionosEndpoint = "https://api.ionos.com"
 	}
@@ -55,11 +60,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer)
 
 	return refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
-			Mech:     "ionos",
-			Interval: time.Duration(conf.RefreshInterval),
-			RefreshF: d.refresh,
-			Registry: reg,
+			Logger:              logger,
+			Mech:                "ionos",
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	), nil
 }
@@ -84,6 +89,13 @@ type SDConfig struct {
 	ionosEndpoint string // For tests only.
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &ionosMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the IONOS Cloud service discovery.
 func (c SDConfig) Name() string {
 	return "ionos"
@@ -91,7 +103,7 @@ func (c SDConfig) Name() string {
 
 // NewDiscoverer returns a new discovery.Discoverer for IONOS Cloud.
 func (c SDConfig) NewDiscoverer(options discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(&c, options.Logger, options.Registerer)
+	return NewDiscovery(&c, options.Logger, options.Metrics)
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
diff --git a/discovery/ionos/metrics.go b/discovery/ionos/metrics.go
new file mode 100644
index 000000000..88e465acf
--- /dev/null
+++ b/discovery/ionos/metrics.go
@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ionos
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*ionosMetrics)(nil)
+
+type ionosMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *ionosMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *ionosMetrics) Unregister() {}
diff --git a/discovery/kubernetes/kubernetes.go b/discovery/kubernetes/kubernetes.go
index 5c5f3dfb6..362e05326 100644
--- a/discovery/kubernetes/kubernetes.go
+++ b/discovery/kubernetes/kubernetes.go
@@ -123,12 +123,17 @@ type SDConfig struct {
 	AttachMetadata     AttachMetadataConfig    `yaml:"attach_metadata,omitempty"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return newDiscovererMetrics(reg, rmi)
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "kubernetes" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return New(opts.Logger, opts.Registerer, c)
+	return New(opts.Logger, opts.Metrics, c)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -265,8 +270,7 @@ type Discovery struct {
 	selectors          roleSelector
 	ownNamespace       string
 	attachMetadata     AttachMetadataConfig
-	eventCount         *prometheus.CounterVec
-	metricRegisterer   discovery.MetricRegisterer
+	metrics            *kubernetesMetrics
 }
 
 func (d *Discovery) getNamespaces() []string {
@@ -285,7 +289,12 @@ func (d *Discovery) getNamespaces() []string {
 }
 
 // New creates a new Kubernetes discovery for the given role.
-func New(l log.Logger, reg prometheus.Registerer, conf *SDConfig) (*Discovery, error) {
+func New(l log.Logger, metrics discovery.DiscovererMetrics, conf *SDConfig) (*Discovery, error) {
+	m, ok := metrics.(*kubernetesMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if l == nil {
 		l = log.NewNopLogger()
 	}
@@ -348,34 +357,7 @@ func New(l log.Logger, reg prometheus.Registerer, conf *SDConfig) (*Discovery, e
 		selectors:          mapSelector(conf.Selectors),
 		ownNamespace:       ownNamespace,
 		attachMetadata:     conf.AttachMetadata,
-		eventCount: prometheus.NewCounterVec(
-			prometheus.CounterOpts{
-				Namespace: discovery.KubernetesMetricsNamespace,
-				Name:      "events_total",
-				Help:      "The number of Kubernetes events handled.",
-			},
-			[]string{"role", "event"},
-		),
-	}
-
-	d.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{d.eventCount})
-
-	// Initialize metric vectors.
-	for _, role := range []string{
-		RoleEndpointSlice.String(),
-		RoleEndpoint.String(),
-		RoleNode.String(),
-		RolePod.String(),
-		RoleService.String(),
-		RoleIngress.String(),
-	} {
-		for _, evt := range []string{
-			MetricLabelRoleAdd,
-			MetricLabelRoleDelete,
-			MetricLabelRoleUpdate,
-		} {
-			d.eventCount.WithLabelValues(role, evt)
-		}
+		metrics:            m,
 	}
 
 	return d, nil
@@ -415,13 +397,6 @@ const resyncDisabled = 0
 func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 	d.Lock()
 
-	err := d.metricRegisterer.RegisterMetrics()
-	if err != nil {
-		level.Error(d.logger).Log("msg", "Unable to register metrics", "err", err.Error())
-		return
-	}
-	defer d.metricRegisterer.UnregisterMetrics()
-
 	namespaces := d.getNamespaces()
 
 	switch d.role {
@@ -513,7 +488,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 				cache.NewSharedInformer(slw, &apiv1.Service{}, resyncDisabled),
 				cache.NewSharedInformer(plw, &apiv1.Pod{}, resyncDisabled),
 				nodeInf,
-				d.eventCount,
+				d.metrics.eventCount,
 			)
 			d.discoverers = append(d.discoverers, eps)
 			go eps.endpointSliceInf.Run(ctx.Done())
@@ -573,7 +548,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 				cache.NewSharedInformer(slw, &apiv1.Service{}, resyncDisabled),
 				cache.NewSharedInformer(plw, &apiv1.Pod{}, resyncDisabled),
 				nodeInf,
-				d.eventCount,
+				d.metrics.eventCount,
 			)
 			d.discoverers = append(d.discoverers, eps)
 			go eps.endpointsInf.Run(ctx.Done())
@@ -605,7 +580,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 				log.With(d.logger, "role", "pod"),
 				d.newPodsByNodeInformer(plw),
 				nodeInformer,
-				d.eventCount,
+				d.metrics.eventCount,
 			)
 			d.discoverers = append(d.discoverers, pod)
 			go pod.podInf.Run(ctx.Done())
@@ -628,7 +603,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 			svc := NewService(
 				log.With(d.logger, "role", "service"),
 				cache.NewSharedInformer(slw, &apiv1.Service{}, resyncDisabled),
-				d.eventCount,
+				d.metrics.eventCount,
 			)
 			d.discoverers = append(d.discoverers, svc)
 			go svc.informer.Run(ctx.Done())
@@ -686,14 +661,14 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 			ingress := NewIngress(
 				log.With(d.logger, "role", "ingress"),
 				informer,
-				d.eventCount,
+				d.metrics.eventCount,
 			)
 			d.discoverers = append(d.discoverers, ingress)
 			go ingress.informer.Run(ctx.Done())
 		}
 	case RoleNode:
 		nodeInformer := d.newNodeInformer(ctx)
-		node := NewNode(log.With(d.logger, "role", "node"), nodeInformer, d.eventCount)
+		node := NewNode(log.With(d.logger, "role", "node"), nodeInformer, d.metrics.eventCount)
 		d.discoverers = append(d.discoverers, node)
 		go node.informer.Run(ctx.Done())
 	default:
diff --git a/discovery/kubernetes/kubernetes_test.go b/discovery/kubernetes/kubernetes_test.go
index 71c937e94..4071ebc34 100644
--- a/discovery/kubernetes/kubernetes_test.go
+++ b/discovery/kubernetes/kubernetes_test.go
@@ -51,24 +51,29 @@ func makeDiscoveryWithVersion(role Role, nsDiscovery NamespaceDiscovery, k8sVer
 	fakeDiscovery, _ := clientset.Discovery().(*fakediscovery.FakeDiscovery)
 	fakeDiscovery.FakedServerVersion = &version.Info{GitVersion: k8sVer}
 
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := newDiscovererMetrics(reg, refreshMetrics)
+	err := metrics.Register()
+	if err != nil {
+		panic(err)
+	}
+	// TODO(ptodev): Unregister the metrics at the end of the test.
+
+	kubeMetrics, ok := metrics.(*kubernetesMetrics)
+	if !ok {
+		panic("invalid discovery metrics type")
+	}
+
 	d := &Discovery{
 		client:             clientset,
 		logger:             log.NewNopLogger(),
 		role:               role,
 		namespaceDiscovery: &nsDiscovery,
 		ownNamespace:       "own-ns",
-		eventCount: prometheus.NewCounterVec(
-			prometheus.CounterOpts{
-				Namespace: discovery.KubernetesMetricsNamespace,
-				Name:      "events_total",
-				Help:      "The number of Kubernetes events handled.",
-			},
-			[]string{"role", "event"},
-		),
+		metrics:            kubeMetrics,
 	}
 
-	d.metricRegisterer = discovery.NewMetricRegisterer(prometheus.NewRegistry(), []prometheus.Collector{d.eventCount})
-
 	return d, clientset
 }
 
diff --git a/discovery/kubernetes/metrics.go b/discovery/kubernetes/metrics.go
new file mode 100644
index 000000000..7d384fb96
--- /dev/null
+++ b/discovery/kubernetes/metrics.go
@@ -0,0 +1,75 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kubernetes
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*kubernetesMetrics)(nil)
+
+type kubernetesMetrics struct {
+	eventCount *prometheus.CounterVec
+
+	metricRegisterer discovery.MetricRegisterer
+}
+
+func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	m := &kubernetesMetrics{
+		eventCount: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Namespace: discovery.KubernetesMetricsNamespace,
+				Name:      "events_total",
+				Help:      "The number of Kubernetes events handled.",
+			},
+			[]string{"role", "event"},
+		),
+	}
+
+	m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
+		m.eventCount,
+	})
+
+	// Initialize metric vectors.
+	for _, role := range []string{
+		RoleEndpointSlice.String(),
+		RoleEndpoint.String(),
+		RoleNode.String(),
+		RolePod.String(),
+		RoleService.String(),
+		RoleIngress.String(),
+	} {
+		for _, evt := range []string{
+			MetricLabelRoleAdd,
+			MetricLabelRoleDelete,
+			MetricLabelRoleUpdate,
+		} {
+			m.eventCount.WithLabelValues(role, evt)
+		}
+	}
+
+	return m
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *kubernetesMetrics) Register() error {
+	return m.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *kubernetesMetrics) Unregister() {
+	m.metricRegisterer.UnregisterMetrics()
+}
diff --git a/discovery/legacymanager/manager.go b/discovery/legacymanager/manager.go
index 9c80f305a..6fc61485d 100644
--- a/discovery/legacymanager/manager.go
+++ b/discovery/legacymanager/manager.go
@@ -42,7 +42,7 @@ type provider struct {
 }
 
 // NewManager is the Discovery Manager constructor.
-func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Registerer, options ...func(*Manager)) *Manager {
+func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Registerer, sdMetrics map[string]discovery.DiscovererMetrics, options ...func(*Manager)) *Manager {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -55,6 +55,7 @@ func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Re
 		updatert:       5 * time.Second,
 		triggerSend:    make(chan struct{}, 1),
 		registerer:     registerer,
+		sdMetrics:      sdMetrics,
 	}
 	for _, option := range options {
 		option(mgr)
@@ -62,7 +63,7 @@ func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Re
 
 	// Register the metrics.
 	// We have to do this after setting all options, so that the name of the Manager is set.
-	if metrics, err := discovery.NewMetrics(registerer, mgr.name); err == nil {
+	if metrics, err := discovery.NewManagerMetrics(registerer, mgr.name); err == nil {
 		mgr.metrics = metrics
 	} else {
 		level.Error(logger).Log("msg", "Failed to create discovery manager metrics", "manager", mgr.name, "err", err)
@@ -108,7 +109,8 @@ type Manager struct {
 	// A registerer for all service discovery metrics.
 	registerer prometheus.Registerer
 
-	metrics *discovery.Metrics
+	metrics   *discovery.Metrics
+	sdMetrics map[string]discovery.DiscovererMetrics
 }
 
 // Run starts the background processing.
@@ -283,8 +285,8 @@ func (m *Manager) registerProviders(cfgs discovery.Configs, setName string) int
 		}
 		typ := cfg.Name()
 		d, err := cfg.NewDiscoverer(discovery.DiscovererOptions{
-			Logger:     log.With(m.logger, "discovery", typ, "config", setName),
-			Registerer: m.registerer,
+			Logger:  log.With(m.logger, "discovery", typ, "config", setName),
+			Metrics: m.sdMetrics[typ],
 		})
 		if err != nil {
 			level.Error(m.logger).Log("msg", "Cannot create service discovery", "err", err, "type", typ, "config", setName)
diff --git a/discovery/legacymanager/manager_test.go b/discovery/legacymanager/manager_test.go
index 7a2e8feea..6fbecabc2 100644
--- a/discovery/legacymanager/manager_test.go
+++ b/discovery/legacymanager/manager_test.go
@@ -36,6 +36,13 @@ func TestMain(m *testing.M) {
 	testutil.TolerantVerifyLeak(m)
 }
 
+func newTestMetrics(t *testing.T, reg prometheus.Registerer) (*discovery.RefreshMetricsManager, map[string]discovery.DiscovererMetrics) {
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	sdMetrics, err := discovery.RegisterSDMetrics(reg, refreshMetrics)
+	require.NoError(t, err)
+	return &refreshMetrics, sdMetrics
+}
+
 // TestTargetUpdatesOrder checks that the target updates are received in the expected order.
 func TestTargetUpdatesOrder(t *testing.T) {
 	// The order by which the updates are send is determined by the interval passed to the mock discovery adapter
@@ -665,7 +672,10 @@ func TestTargetUpdatesOrder(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel()
 
-			discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+			reg := prometheus.NewRegistry()
+			_, sdMetrics := newTestMetrics(t, reg)
+
+			discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 			require.NotNil(t, discoveryManager)
 			discoveryManager.updatert = 100 * time.Millisecond
 
@@ -748,7 +758,11 @@ func verifyPresence(t *testing.T, tSets map[poolKey]map[string]*targetgroup.Grou
 func TestTargetSetRecreatesTargetGroupsEveryRun(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := newTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -777,7 +791,11 @@ func TestTargetSetRecreatesTargetGroupsEveryRun(t *testing.T) {
 func TestDiscovererConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := newTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -802,7 +820,11 @@ func TestDiscovererConfigs(t *testing.T) {
 func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := newTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -842,7 +864,11 @@ func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 func TestIdenticalConfigurationsAreCoalesced(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, nil, prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := newTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, nil, reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -874,7 +900,11 @@ func TestApplyConfigDoesNotModifyStaticTargets(t *testing.T) {
 	}
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := newTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -897,10 +927,19 @@ func (e errorConfig) NewDiscoverer(discovery.DiscovererOptions) (discovery.Disco
 	return nil, e.err
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (errorConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &discovery.NoopDiscovererMetrics{}
+}
+
 func TestGaugeFailedConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := newTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -1057,7 +1096,10 @@ func TestCoordinationWithReceiver(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel()
 
-			mgr := NewManager(ctx, nil, prometheus.NewRegistry())
+			reg := prometheus.NewRegistry()
+			_, sdMetrics := newTestMetrics(t, reg)
+
+			mgr := NewManager(ctx, nil, reg, sdMetrics)
 			require.NotNil(t, mgr)
 			mgr.updatert = updateDelay
 			go mgr.Run()
diff --git a/discovery/linode/linode.go b/discovery/linode/linode.go
index 38a5cdad4..94f0a63bb 100644
--- a/discovery/linode/linode.go
+++ b/discovery/linode/linode.go
@@ -87,12 +87,17 @@ type SDConfig struct {
 	TagSeparator    string         `yaml:"tag_separator,omitempty"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return newDiscovererMetrics(reg, rmi)
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "linode" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.Registerer)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -122,22 +127,23 @@ type Discovery struct {
 	pollCount            int
 	lastResults          []*targetgroup.Group
 	eventPollingEnabled  bool
-	failuresCount        prometheus.Counter
+	metrics              *linodeMetrics
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*linodeMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	d := &Discovery{
 		port:                 conf.Port,
 		tagSeparator:         conf.TagSeparator,
 		pollCount:            0,
 		lastRefreshTimestamp: time.Now().UTC(),
 		eventPollingEnabled:  true,
-		failuresCount: prometheus.NewCounter(
-			prometheus.CounterOpts{
-				Name: "prometheus_sd_linode_failures_total",
-				Help: "Number of Linode service discovery refresh failures.",
-			}),
+		metrics:              m,
 	}
 
 	rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "linode_sd")
@@ -156,12 +162,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer)
 
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
-			Mech:     "linode",
-			Interval: time.Duration(conf.RefreshInterval),
-			RefreshF: d.refresh,
-			Registry: reg,
-			Metrics:  []prometheus.Collector{d.failuresCount},
+			Logger:              logger,
+			Mech:                "linode",
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
 	return d, nil
@@ -223,14 +228,14 @@ func (d *Discovery) refreshData(ctx context.Context) ([]*targetgroup.Group, erro
 	// Gather all linode instances.
 	instances, err := d.client.ListInstances(ctx, &linodego.ListOptions{PageSize: 500})
 	if err != nil {
-		d.failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, err
 	}
 
 	// Gather detailed IP address info for all IPs on all linode instances.
 	detailedIPs, err := d.client.ListIPAddresses(ctx, &linodego.ListOptions{PageSize: 500})
 	if err != nil {
-		d.failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, err
 	}
 
diff --git a/discovery/linode/linode_test.go b/discovery/linode/linode_test.go
index 536b12090..a6a16d82a 100644
--- a/discovery/linode/linode_test.go
+++ b/discovery/linode/linode_test.go
@@ -24,6 +24,8 @@ import (
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/prometheus/discovery"
 )
 
 type LinodeSDTestSuite struct {
@@ -53,7 +55,15 @@ func TestLinodeSDRefresh(t *testing.T) {
 		Credentials: tokenID,
 		Type:        "Bearer",
 	}
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 	endpoint, err := url.Parse(sdmock.Mock.Endpoint())
 	require.NoError(t, err)
diff --git a/discovery/linode/metrics.go b/discovery/linode/metrics.go
new file mode 100644
index 000000000..8f8138922
--- /dev/null
+++ b/discovery/linode/metrics.go
@@ -0,0 +1,57 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package linode
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*linodeMetrics)(nil)
+
+type linodeMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+
+	failuresCount prometheus.Counter
+
+	metricRegisterer discovery.MetricRegisterer
+}
+
+func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	m := &linodeMetrics{
+		refreshMetrics: rmi,
+		failuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_linode_failures_total",
+				Help: "Number of Linode service discovery refresh failures.",
+			}),
+	}
+
+	m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
+		m.failuresCount,
+	})
+
+	return m
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *linodeMetrics) Register() error {
+	return m.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *linodeMetrics) Unregister() {
+	m.metricRegisterer.UnregisterMetrics()
+}
diff --git a/discovery/manager.go b/discovery/manager.go
index 67e326c41..e3a263557 100644
--- a/discovery/manager.go
+++ b/discovery/manager.go
@@ -64,8 +64,24 @@ func (p *Provider) Config() interface{} {
 	return p.config
 }
 
+// Registers the metrics needed for SD mechanisms.
+// Does not register the metrics for the Discovery Manager.
+// TODO(ptodev): Add ability to unregister the metrics?
+func CreateAndRegisterSDMetrics(reg prometheus.Registerer) (map[string]DiscovererMetrics, error) {
+	// Some SD mechanisms use the "refresh" package, which has its own metrics.
+	refreshSdMetrics := NewRefreshMetrics(reg)
+
+	// Register the metrics specific for each SD mechanism, and the ones for the refresh package.
+	sdMetrics, err := RegisterSDMetrics(reg, refreshSdMetrics)
+	if err != nil {
+		return nil, fmt.Errorf("failed to register service discovery metrics: %w", err)
+	}
+
+	return sdMetrics, nil
+}
+
 // NewManager is the Discovery Manager constructor.
-func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Registerer, options ...func(*Manager)) *Manager {
+func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Registerer, sdMetrics map[string]DiscovererMetrics, options ...func(*Manager)) *Manager {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -77,6 +93,7 @@ func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Re
 		updatert:    5 * time.Second,
 		triggerSend: make(chan struct{}, 1),
 		registerer:  registerer,
+		sdMetrics:   sdMetrics,
 	}
 	for _, option := range options {
 		option(mgr)
@@ -84,7 +101,7 @@ func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Re
 
 	// Register the metrics.
 	// We have to do this after setting all options, so that the name of the Manager is set.
-	if metrics, err := NewMetrics(registerer, mgr.name); err == nil {
+	if metrics, err := NewManagerMetrics(registerer, mgr.name); err == nil {
 		mgr.metrics = metrics
 	} else {
 		level.Error(logger).Log("msg", "Failed to create discovery manager metrics", "manager", mgr.name, "err", err)
@@ -143,7 +160,8 @@ type Manager struct {
 	// A registerer for all service discovery metrics.
 	registerer prometheus.Registerer
 
-	metrics *Metrics
+	metrics   *Metrics
+	sdMetrics map[string]DiscovererMetrics
 }
 
 // Providers returns the currently configured SD providers.
@@ -402,7 +420,7 @@ func (m *Manager) registerProviders(cfgs Configs, setName string) int {
 		d, err := cfg.NewDiscoverer(DiscovererOptions{
 			Logger:            log.With(m.logger, "discovery", typ, "config", setName),
 			HTTPClientOptions: m.httpOpts,
-			Registerer:        m.registerer,
+			Metrics:           m.sdMetrics[typ],
 		})
 		if err != nil {
 			level.Error(m.logger).Log("msg", "Cannot create service discovery", "err", err, "type", typ, "config", setName)
diff --git a/discovery/manager_test.go b/discovery/manager_test.go
index f22de75a4..2f6e03aa5 100644
--- a/discovery/manager_test.go
+++ b/discovery/manager_test.go
@@ -36,6 +36,13 @@ func TestMain(m *testing.M) {
 	testutil.TolerantVerifyLeak(m)
 }
 
+func NewTestMetrics(t *testing.T, reg prometheus.Registerer) (*RefreshMetricsManager, map[string]DiscovererMetrics) {
+	refreshMetrics := NewRefreshMetrics(reg)
+	sdMetrics, err := RegisterSDMetrics(reg, refreshMetrics)
+	require.NoError(t, err)
+	return &refreshMetrics, sdMetrics
+}
+
 // TestTargetUpdatesOrder checks that the target updates are received in the expected order.
 func TestTargetUpdatesOrder(t *testing.T) {
 	// The order by which the updates are send is determined by the interval passed to the mock discovery adapter
@@ -665,7 +672,10 @@ func TestTargetUpdatesOrder(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel()
 
-			discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+			reg := prometheus.NewRegistry()
+			_, sdMetrics := NewTestMetrics(t, reg)
+
+			discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 			require.NotNil(t, discoveryManager)
 			discoveryManager.updatert = 100 * time.Millisecond
 
@@ -780,7 +790,11 @@ func pk(provider, setName string, n int) poolKey {
 func TestTargetSetTargetGroupsPresentOnConfigReload(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -813,7 +827,11 @@ func TestTargetSetTargetGroupsPresentOnConfigReload(t *testing.T) {
 func TestTargetSetTargetGroupsPresentOnConfigRename(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -849,7 +867,11 @@ func TestTargetSetTargetGroupsPresentOnConfigRename(t *testing.T) {
 func TestTargetSetTargetGroupsPresentOnConfigDuplicateAndDeleteOriginal(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -888,7 +910,11 @@ func TestTargetSetTargetGroupsPresentOnConfigDuplicateAndDeleteOriginal(t *testi
 func TestTargetSetTargetGroupsPresentOnConfigChange(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -950,7 +976,11 @@ func TestTargetSetTargetGroupsPresentOnConfigChange(t *testing.T) {
 func TestTargetSetRecreatesTargetGroupsOnConfigChange(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -990,7 +1020,11 @@ func TestTargetSetRecreatesTargetGroupsOnConfigChange(t *testing.T) {
 func TestDiscovererConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -1023,7 +1057,11 @@ func TestDiscovererConfigs(t *testing.T) {
 func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -1071,7 +1109,11 @@ func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 func TestIdenticalConfigurationsAreCoalesced(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, nil, prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, nil, reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -1108,7 +1150,11 @@ func TestApplyConfigDoesNotModifyStaticTargets(t *testing.T) {
 	}
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -1129,11 +1175,21 @@ type errorConfig struct{ err error }
 func (e errorConfig) Name() string                                        { return "error" }
 func (e errorConfig) NewDiscoverer(DiscovererOptions) (Discoverer, error) { return nil, e.err }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (errorConfig) NewDiscovererMetrics(prometheus.Registerer, RefreshMetricsInstantiator) DiscovererMetrics {
+	return &NoopDiscovererMetrics{}
+}
+
 type lockStaticConfig struct {
 	mu     *sync.Mutex
 	config StaticConfig
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (lockStaticConfig) NewDiscovererMetrics(prometheus.Registerer, RefreshMetricsInstantiator) DiscovererMetrics {
+	return &NoopDiscovererMetrics{}
+}
+
 func (s lockStaticConfig) Name() string { return "lockstatic" }
 func (s lockStaticConfig) NewDiscoverer(DiscovererOptions) (Discoverer, error) {
 	return (lockStaticDiscoverer)(s), nil
@@ -1155,7 +1211,11 @@ func (s lockStaticDiscoverer) Run(ctx context.Context, up chan<- []*targetgroup.
 func TestGaugeFailedConfigs(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -1312,7 +1372,10 @@ func TestCoordinationWithReceiver(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel()
 
-			mgr := NewManager(ctx, nil, prometheus.NewRegistry())
+			reg := prometheus.NewRegistry()
+			_, sdMetrics := NewTestMetrics(t, reg)
+
+			mgr := NewManager(ctx, nil, reg, sdMetrics)
 			require.NotNil(t, mgr)
 			mgr.updatert = updateDelay
 			go mgr.Run()
@@ -1408,7 +1471,11 @@ func (o onceProvider) Run(_ context.Context, ch chan<- []*targetgroup.Group) {
 func TestTargetSetTargetGroupsUpdateDuringApplyConfig(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	discoveryManager := NewManager(ctx, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics)
 	require.NotNil(t, discoveryManager)
 	discoveryManager.updatert = 100 * time.Millisecond
 	go discoveryManager.Run()
@@ -1470,6 +1537,11 @@ func newTestDiscoverer() *testDiscoverer {
 	}
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*testDiscoverer) NewDiscovererMetrics(prometheus.Registerer, RefreshMetricsInstantiator) DiscovererMetrics {
+	return &NoopDiscovererMetrics{}
+}
+
 // Name implements Config.
 func (t *testDiscoverer) Name() string {
 	return "test"
diff --git a/discovery/marathon/marathon.go b/discovery/marathon/marathon.go
index a6a6252fd..ecad108e4 100644
--- a/discovery/marathon/marathon.go
+++ b/discovery/marathon/marathon.go
@@ -79,12 +79,19 @@ type SDConfig struct {
 	HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &marathonMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "marathon" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(*c, opts.Logger, opts.Registerer)
+	return NewDiscovery(*c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -133,7 +140,12 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Marathon Discovery.
-func NewDiscovery(conf SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(conf SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*marathonMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "marathon_sd")
 	if err != nil {
 		return nil, err
@@ -156,11 +168,11 @@ func NewDiscovery(conf SDConfig, logger log.Logger, reg prometheus.Registerer) (
 	}
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
-			Mech:     "marathon",
-			Interval: time.Duration(conf.RefreshInterval),
-			RefreshF: d.refresh,
-			Registry: reg,
+			Logger:              logger,
+			Mech:                "marathon",
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
 	return d, nil
diff --git a/discovery/marathon/marathon_test.go b/discovery/marathon/marathon_test.go
index a1ddce930..c78cc1e3c 100644
--- a/discovery/marathon/marathon_test.go
+++ b/discovery/marathon/marathon_test.go
@@ -23,7 +23,9 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/require"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -37,7 +39,19 @@ func testConfig() SDConfig {
 }
 
 func testUpdateServices(client appListClient) ([]*targetgroup.Group, error) {
-	md, err := NewDiscovery(testConfig(), nil, prometheus.NewRegistry())
+	cfg := testConfig()
+
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	err := metrics.Register()
+	if err != nil {
+		return nil, err
+	}
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	md, err := NewDiscovery(cfg, nil, metrics)
 	if err != nil {
 		return nil, err
 	}
@@ -130,7 +144,15 @@ func TestMarathonSDSendGroup(t *testing.T) {
 }
 
 func TestMarathonSDRemoveApp(t *testing.T) {
-	md, err := NewDiscovery(testConfig(), nil, prometheus.NewRegistry())
+	cfg := testConfig()
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	md, err := NewDiscovery(cfg, nil, metrics)
 	if err != nil {
 		t.Fatalf("%s", err)
 	}
diff --git a/discovery/marathon/metrics.go b/discovery/marathon/metrics.go
new file mode 100644
index 000000000..790c16747
--- /dev/null
+++ b/discovery/marathon/metrics.go
@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package marathon
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*marathonMetrics)(nil)
+
+type marathonMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *marathonMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *marathonMetrics) Unregister() {}
diff --git a/discovery/metrics.go b/discovery/metrics.go
index 6a6060395..a77b86d27 100644
--- a/discovery/metrics.go
+++ b/discovery/metrics.go
@@ -38,7 +38,7 @@ type Metrics struct {
 	SentUpdates       prometheus.Counter
 }
 
-func NewMetrics(registerer prometheus.Registerer, sdManagerName string) (*Metrics, error) {
+func NewManagerMetrics(registerer prometheus.Registerer, sdManagerName string) (*Metrics, error) {
 	m := &Metrics{}
 
 	m.FailedConfigs = prometheus.NewGauge(
diff --git a/discovery/metrics_refresh.go b/discovery/metrics_refresh.go
new file mode 100644
index 000000000..d621165ce
--- /dev/null
+++ b/discovery/metrics_refresh.go
@@ -0,0 +1,75 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package discovery
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Metric vectors for the "refresh" package.
+// We define them here in the "discovery" package in order to avoid a cyclic dependency between
+// "discovery" and "refresh".
+type RefreshMetricsVecs struct {
+	failuresVec *prometheus.CounterVec
+	durationVec *prometheus.SummaryVec
+
+	metricRegisterer MetricRegisterer
+}
+
+var _ RefreshMetricsManager = (*RefreshMetricsVecs)(nil)
+
+func NewRefreshMetrics(reg prometheus.Registerer) RefreshMetricsManager {
+	m := &RefreshMetricsVecs{
+		failuresVec: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_refresh_failures_total",
+				Help: "Number of refresh failures for the given SD mechanism.",
+			},
+			[]string{"mechanism"}),
+		durationVec: prometheus.NewSummaryVec(
+			prometheus.SummaryOpts{
+				Name:       "prometheus_sd_refresh_duration_seconds",
+				Help:       "The duration of a refresh in seconds for the given SD mechanism.",
+				Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+			},
+			[]string{"mechanism"}),
+	}
+
+	// The reason we register metric vectors instead of metrics is so that
+	// the metrics are not visible until they are recorded.
+	m.metricRegisterer = NewMetricRegisterer(reg, []prometheus.Collector{
+		m.failuresVec,
+		m.durationVec,
+	})
+
+	return m
+}
+
+// Instantiate returns metrics out of metric vectors.
+func (m *RefreshMetricsVecs) Instantiate(mech string) *RefreshMetrics {
+	return &RefreshMetrics{
+		Failures: m.failuresVec.WithLabelValues(mech),
+		Duration: m.durationVec.WithLabelValues(mech),
+	}
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *RefreshMetricsVecs) Register() error {
+	return m.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *RefreshMetricsVecs) Unregister() {
+	m.metricRegisterer.UnregisterMetrics()
+}
diff --git a/discovery/moby/docker.go b/discovery/moby/docker.go
index a13bb8704..57fa62bb5 100644
--- a/discovery/moby/docker.go
+++ b/discovery/moby/docker.go
@@ -76,12 +76,19 @@ type DockerSDConfig struct {
 	RefreshInterval model.Duration `yaml:"refresh_interval"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*DockerSDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &dockerMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Config.
 func (*DockerSDConfig) Name() string { return "docker" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *DockerSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDockerDiscovery(c, opts.Logger, opts.Registerer)
+	return NewDockerDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -115,8 +122,11 @@ type DockerDiscovery struct {
 }
 
 // NewDockerDiscovery returns a new DockerDiscovery which periodically refreshes its targets.
-func NewDockerDiscovery(conf *DockerSDConfig, logger log.Logger, reg prometheus.Registerer) (*DockerDiscovery, error) {
-	var err error
+func NewDockerDiscovery(conf *DockerSDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*DockerDiscovery, error) {
+	m, ok := metrics.(*dockerMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
 
 	d := &DockerDiscovery{
 		port:               conf.Port,
@@ -167,11 +177,11 @@ func NewDockerDiscovery(conf *DockerSDConfig, logger log.Logger, reg prometheus.
 
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
-			Mech:     "docker",
-			Interval: time.Duration(conf.RefreshInterval),
-			RefreshF: d.refresh,
-			Registry: reg,
+			Logger:              logger,
+			Mech:                "docker",
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
 	return d, nil
diff --git a/discovery/moby/docker_test.go b/discovery/moby/docker_test.go
index 1a87ad2a1..fec56d3e5 100644
--- a/discovery/moby/docker_test.go
+++ b/discovery/moby/docker_test.go
@@ -23,6 +23,8 @@ import (
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
+
+	"github.com/prometheus/prometheus/discovery"
 )
 
 func TestDockerSDRefresh(t *testing.T) {
@@ -38,7 +40,14 @@ host: %s
 	var cfg DockerSDConfig
 	require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg))
 
-	d, err := NewDockerDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	d, err := NewDockerDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()
diff --git a/discovery/moby/dockerswarm.go b/discovery/moby/dockerswarm.go
index bd87fea5a..b0147467d 100644
--- a/discovery/moby/dockerswarm.go
+++ b/discovery/moby/dockerswarm.go
@@ -70,12 +70,19 @@ type Filter struct {
 	Values []string `yaml:"values"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*DockerSwarmSDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &dockerswarmMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Config.
 func (*DockerSwarmSDConfig) Name() string { return "dockerswarm" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *DockerSwarmSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.Registerer)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -118,8 +125,11 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *DockerSwarmSDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
-	var err error
+func NewDiscovery(conf *DockerSwarmSDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*dockerswarmMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
 
 	d := &Discovery{
 		port: conf.Port,
@@ -170,11 +180,11 @@ func NewDiscovery(conf *DockerSwarmSDConfig, logger log.Logger, reg prometheus.R
 
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
-			Mech:     "dockerswarm",
-			Interval: time.Duration(conf.RefreshInterval),
-			RefreshF: d.refresh,
-			Registry: reg,
+			Logger:              logger,
+			Mech:                "dockerswarm",
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
 	return d, nil
diff --git a/discovery/moby/metrics_docker.go b/discovery/moby/metrics_docker.go
new file mode 100644
index 000000000..457d520a0
--- /dev/null
+++ b/discovery/moby/metrics_docker.go
@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package moby
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*dockerMetrics)(nil)
+
+type dockerMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *dockerMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *dockerMetrics) Unregister() {}
diff --git a/discovery/moby/metrics_dockerswarm.go b/discovery/moby/metrics_dockerswarm.go
new file mode 100644
index 000000000..227ba6c2b
--- /dev/null
+++ b/discovery/moby/metrics_dockerswarm.go
@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package moby
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*dockerswarmMetrics)(nil)
+
+type dockerswarmMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *dockerswarmMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *dockerswarmMetrics) Unregister() {}
diff --git a/discovery/moby/nodes_test.go b/discovery/moby/nodes_test.go
index 512ff7049..4ad1088d1 100644
--- a/discovery/moby/nodes_test.go
+++ b/discovery/moby/nodes_test.go
@@ -23,6 +23,8 @@ import (
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
+
+	"github.com/prometheus/prometheus/discovery"
 )
 
 func TestDockerSwarmNodesSDRefresh(t *testing.T) {
@@ -39,7 +41,14 @@ host: %s
 	var cfg DockerSwarmSDConfig
 	require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg))
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()
diff --git a/discovery/moby/services_test.go b/discovery/moby/services_test.go
index 816586dd7..47ca69e33 100644
--- a/discovery/moby/services_test.go
+++ b/discovery/moby/services_test.go
@@ -23,6 +23,8 @@ import (
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
+
+	"github.com/prometheus/prometheus/discovery"
 )
 
 func TestDockerSwarmSDServicesRefresh(t *testing.T) {
@@ -39,7 +41,14 @@ host: %s
 	var cfg DockerSwarmSDConfig
 	require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg))
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()
@@ -333,7 +342,14 @@ filters:
 	var cfg DockerSwarmSDConfig
 	require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg))
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()
diff --git a/discovery/moby/tasks_test.go b/discovery/moby/tasks_test.go
index 764fda343..ef71bc02f 100644
--- a/discovery/moby/tasks_test.go
+++ b/discovery/moby/tasks_test.go
@@ -23,6 +23,8 @@ import (
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
+
+	"github.com/prometheus/prometheus/discovery"
 )
 
 func TestDockerSwarmTasksSDRefresh(t *testing.T) {
@@ -39,7 +41,14 @@ host: %s
 	var cfg DockerSwarmSDConfig
 	require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg))
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()
diff --git a/discovery/nomad/metrics.go b/discovery/nomad/metrics.go
new file mode 100644
index 000000000..9707153d9
--- /dev/null
+++ b/discovery/nomad/metrics.go
@@ -0,0 +1,57 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nomad
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*nomadMetrics)(nil)
+
+type nomadMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+
+	failuresCount prometheus.Counter
+
+	metricRegisterer discovery.MetricRegisterer
+}
+
+func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	m := &nomadMetrics{
+		refreshMetrics: rmi,
+		failuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "prometheus_sd_nomad_failures_total",
+				Help: "Number of nomad service discovery refresh failures.",
+			}),
+	}
+
+	m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
+		m.failuresCount,
+	})
+
+	return m
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *nomadMetrics) Register() error {
+	return m.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *nomadMetrics) Unregister() {
+	m.metricRegisterer.UnregisterMetrics()
+}
diff --git a/discovery/nomad/nomad.go b/discovery/nomad/nomad.go
index 3fdcf714e..d9c48120a 100644
--- a/discovery/nomad/nomad.go
+++ b/discovery/nomad/nomad.go
@@ -74,12 +74,17 @@ type SDConfig struct {
 	TagSeparator     string                  `yaml:"tag_separator,omitempty"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return newDiscovererMetrics(reg, rmi)
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "nomad" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.Registerer)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -112,11 +117,16 @@ type Discovery struct {
 	region          string
 	server          string
 	tagSeparator    string
-	failuresCount   prometheus.Counter
+	metrics         *nomadMetrics
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*nomadMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	d := &Discovery{
 		allowStale:      conf.AllowStale,
 		namespace:       conf.Namespace,
@@ -124,11 +134,7 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer)
 		region:          conf.Region,
 		server:          conf.Server,
 		tagSeparator:    conf.TagSeparator,
-		failuresCount: prometheus.NewCounter(
-			prometheus.CounterOpts{
-				Name: "prometheus_sd_nomad_failures_total",
-				Help: "Number of nomad service discovery refresh failures.",
-			}),
+		metrics:         m,
 	}
 
 	HTTPClient, err := config.NewClientFromConfig(conf.HTTPClientConfig, "nomad_sd")
@@ -151,12 +157,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer)
 
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
-			Mech:     "nomad",
-			Interval: time.Duration(conf.RefreshInterval),
-			RefreshF: d.refresh,
-			Registry: reg,
-			Metrics:  []prometheus.Collector{d.failuresCount},
+			Logger:              logger,
+			Mech:                "nomad",
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
 	return d, nil
@@ -168,7 +173,7 @@ func (d *Discovery) refresh(context.Context) ([]*targetgroup.Group, error) {
 	}
 	stubs, _, err := d.client.Services().List(opts)
 	if err != nil {
-		d.failuresCount.Inc()
+		d.metrics.failuresCount.Inc()
 		return nil, err
 	}
 
@@ -180,7 +185,7 @@ func (d *Discovery) refresh(context.Context) ([]*targetgroup.Group, error) {
 		for _, service := range stub.Services {
 			instances, _, err := d.client.Services().Get(service.ServiceName, opts)
 			if err != nil {
-				d.failuresCount.Inc()
+				d.metrics.failuresCount.Inc()
 				return nil, fmt.Errorf("failed to fetch services: %w", err)
 			}
 
diff --git a/discovery/nomad/nomad_test.go b/discovery/nomad/nomad_test.go
index ca67a877e..357d4a8e9 100644
--- a/discovery/nomad/nomad_test.go
+++ b/discovery/nomad/nomad_test.go
@@ -25,6 +25,8 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/prometheus/discovery"
 )
 
 type NomadSDTestSuite struct {
@@ -128,8 +130,16 @@ func TestConfiguredService(t *testing.T) {
 	conf := &SDConfig{
 		Server: "http://localhost:4646",
 	}
-	_, err := NewDiscovery(conf, nil, prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := conf.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+
+	_, err := NewDiscovery(conf, nil, metrics)
 	require.NoError(t, err)
+
+	metrics.Unregister()
 }
 
 func TestNomadSDRefresh(t *testing.T) {
@@ -142,7 +152,15 @@ func TestNomadSDRefresh(t *testing.T) {
 
 	cfg := DefaultSDConfig
 	cfg.Server = endpoint.String()
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 
 	tgs, err := d.refresh(context.Background())
diff --git a/discovery/openstack/metrics.go b/discovery/openstack/metrics.go
new file mode 100644
index 000000000..a64c1a673
--- /dev/null
+++ b/discovery/openstack/metrics.go
@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package openstack
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+type openstackMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+var _ discovery.DiscovererMetrics = (*openstackMetrics)(nil)
+
+// Register implements discovery.DiscovererMetrics.
+func (m *openstackMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *openstackMetrics) Unregister() {}
diff --git a/discovery/openstack/openstack.go b/discovery/openstack/openstack.go
index 9544a7c0f..c98f78788 100644
--- a/discovery/openstack/openstack.go
+++ b/discovery/openstack/openstack.go
@@ -66,12 +66,19 @@ type SDConfig struct {
 	Availability                string           `yaml:"availability,omitempty"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &openstackMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "openstack" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.Registerer)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -135,18 +142,23 @@ type refresher interface {
 }
 
 // NewDiscovery returns a new OpenStack Discoverer which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, l log.Logger, reg prometheus.Registerer) (*refresh.Discovery, error) {
+func NewDiscovery(conf *SDConfig, l log.Logger, metrics discovery.DiscovererMetrics) (*refresh.Discovery, error) {
+	m, ok := metrics.(*openstackMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	r, err := newRefresher(conf, l)
 	if err != nil {
 		return nil, err
 	}
 	return refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   l,
-			Mech:     "openstack",
-			Interval: time.Duration(conf.RefreshInterval),
-			RefreshF: r.refresh,
-			Registry: reg,
+			Logger:              l,
+			Mech:                "openstack",
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            r.refresh,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	), nil
 }
diff --git a/discovery/ovhcloud/metrics.go b/discovery/ovhcloud/metrics.go
new file mode 100644
index 000000000..1c5170946
--- /dev/null
+++ b/discovery/ovhcloud/metrics.go
@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ovhcloud
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*ovhcloudMetrics)(nil)
+
+type ovhcloudMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *ovhcloudMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *ovhcloudMetrics) Unregister() {}
diff --git a/discovery/ovhcloud/ovhcloud.go b/discovery/ovhcloud/ovhcloud.go
index eca284a85..988b4482f 100644
--- a/discovery/ovhcloud/ovhcloud.go
+++ b/discovery/ovhcloud/ovhcloud.go
@@ -53,6 +53,13 @@ type SDConfig struct {
 	Service           string         `yaml:"service"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &ovhcloudMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name implements the Discoverer interface.
 func (c SDConfig) Name() string {
 	return "ovhcloud"
@@ -94,7 +101,7 @@ func createClient(config *SDConfig) (*ovh.Client, error) {
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(options discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, options.Logger, options.Registerer)
+	return NewDiscovery(c, options.Logger, options.Metrics)
 }
 
 func init() {
@@ -141,7 +148,12 @@ func newRefresher(conf *SDConfig, logger log.Logger) (refresher, error) {
 }
 
 // NewDiscovery returns a new OVHcloud Discoverer which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*refresh.Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*refresh.Discovery, error) {
+	m, ok := metrics.(*ovhcloudMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	r, err := newRefresher(conf, logger)
 	if err != nil {
 		return nil, err
@@ -149,11 +161,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer)
 
 	return refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
-			Mech:     "ovhcloud",
-			Interval: time.Duration(conf.RefreshInterval),
-			RefreshF: r.refresh,
-			Registry: reg,
+			Logger:              logger,
+			Mech:                "ovhcloud",
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            r.refresh,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	), nil
 }
diff --git a/discovery/ovhcloud/ovhcloud_test.go b/discovery/ovhcloud/ovhcloud_test.go
index 9bd9ea954..53ec9b445 100644
--- a/discovery/ovhcloud/ovhcloud_test.go
+++ b/discovery/ovhcloud/ovhcloud_test.go
@@ -122,9 +122,17 @@ func TestParseIPs(t *testing.T) {
 func TestDiscoverer(t *testing.T) {
 	conf, _ := getMockConf("vps")
 	logger := testutil.NewLogger(t)
+
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := conf.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
 	_, err := conf.NewDiscoverer(discovery.DiscovererOptions{
-		Logger:     logger,
-		Registerer: prometheus.NewRegistry(),
+		Logger:  logger,
+		Metrics: metrics,
 	})
 
 	require.NoError(t, err)
diff --git a/discovery/puppetdb/metrics.go b/discovery/puppetdb/metrics.go
new file mode 100644
index 000000000..2f5faa57c
--- /dev/null
+++ b/discovery/puppetdb/metrics.go
@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package puppetdb
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*puppetdbMetrics)(nil)
+
+type puppetdbMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *puppetdbMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *puppetdbMetrics) Unregister() {}
diff --git a/discovery/puppetdb/puppetdb.go b/discovery/puppetdb/puppetdb.go
index 616f2c61e..4355a7ade 100644
--- a/discovery/puppetdb/puppetdb.go
+++ b/discovery/puppetdb/puppetdb.go
@@ -79,12 +79,19 @@ type SDConfig struct {
 	Port              int                     `yaml:"port"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &puppetdbMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "puppetdb" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.Registerer)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -131,7 +138,12 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new PuppetDB discovery for the given config.
-func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*puppetdbMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -158,11 +170,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer)
 
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
-			Mech:     "http",
-			Interval: time.Duration(conf.RefreshInterval),
-			RefreshF: d.refresh,
-			Registry: reg,
+			Logger:              logger,
+			Mech:                "http",
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
 	return d, nil
diff --git a/discovery/puppetdb/puppetdb_test.go b/discovery/puppetdb/puppetdb_test.go
index edd9b9d04..bf9c7b215 100644
--- a/discovery/puppetdb/puppetdb_test.go
+++ b/discovery/puppetdb/puppetdb_test.go
@@ -28,6 +28,7 @@ import (
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -63,9 +64,17 @@ func TestPuppetSlashInURL(t *testing.T) {
 			Port:             80,
 			RefreshInterval:  model.Duration(30 * time.Second),
 		}
-		d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
+
+		reg := prometheus.NewRegistry()
+		refreshMetrics := discovery.NewRefreshMetrics(reg)
+		metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+		require.NoError(t, metrics.Register())
+
+		d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 		require.NoError(t, err)
 		require.Equal(t, apiURL, d.url)
+
+		metrics.Unregister()
 	}
 }
 
@@ -80,7 +89,12 @@ func TestPuppetDBRefresh(t *testing.T) {
 		RefreshInterval:  model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()
@@ -107,6 +121,8 @@ func TestPuppetDBRefresh(t *testing.T) {
 		},
 	}
 	require.Equal(t, expectedTargets, tgs)
+
+	metrics.Unregister()
 }
 
 func TestPuppetDBRefreshWithParameters(t *testing.T) {
@@ -121,7 +137,12 @@ func TestPuppetDBRefreshWithParameters(t *testing.T) {
 		RefreshInterval:   model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()
@@ -158,6 +179,8 @@ func TestPuppetDBRefreshWithParameters(t *testing.T) {
 		},
 	}
 	require.Equal(t, expectedTargets, tgs)
+
+	metrics.Unregister()
 }
 
 func TestPuppetDBInvalidCode(t *testing.T) {
@@ -173,12 +196,19 @@ func TestPuppetDBInvalidCode(t *testing.T) {
 		RefreshInterval:  model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()
 	_, err = d.refresh(ctx)
 	require.EqualError(t, err, "server returned HTTP status 400 Bad Request")
+
+	metrics.Unregister()
 }
 
 func TestPuppetDBInvalidFormat(t *testing.T) {
@@ -194,10 +224,17 @@ func TestPuppetDBInvalidFormat(t *testing.T) {
 		RefreshInterval:  model.Duration(30 * time.Second),
 	}
 
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 
 	ctx := context.Background()
 	_, err = d.refresh(ctx)
 	require.EqualError(t, err, "unsupported content type text/plain; charset=utf-8")
+
+	metrics.Unregister()
 }
diff --git a/discovery/refresh/refresh.go b/discovery/refresh/refresh.go
index 0b0e5a921..f037a90cf 100644
--- a/discovery/refresh/refresh.go
+++ b/discovery/refresh/refresh.go
@@ -20,19 +20,17 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
-	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
 type Options struct {
-	Logger   log.Logger
-	Mech     string
-	Interval time.Duration
-	RefreshF func(ctx context.Context) ([]*targetgroup.Group, error)
-	Registry prometheus.Registerer
-	Metrics  []prometheus.Collector
+	Logger              log.Logger
+	Mech                string
+	Interval            time.Duration
+	RefreshF            func(ctx context.Context) ([]*targetgroup.Group, error)
+	MetricsInstantiator discovery.RefreshMetricsInstantiator
 }
 
 // Discovery implements the Discoverer interface.
@@ -40,15 +38,13 @@ type Discovery struct {
 	logger   log.Logger
 	interval time.Duration
 	refreshf func(ctx context.Context) ([]*targetgroup.Group, error)
-
-	failures prometheus.Counter
-	duration prometheus.Summary
-
-	metricRegisterer discovery.MetricRegisterer
+	metrics  *discovery.RefreshMetrics
 }
 
 // NewDiscovery returns a Discoverer function that calls a refresh() function at every interval.
 func NewDiscovery(opts Options) *Discovery {
+	m := opts.MetricsInstantiator.Instantiate(opts.Mech)
+
 	var logger log.Logger
 	if opts.Logger == nil {
 		logger = log.NewNopLogger()
@@ -60,44 +56,14 @@ func NewDiscovery(opts Options) *Discovery {
 		logger:   logger,
 		interval: opts.Interval,
 		refreshf: opts.RefreshF,
-		failures: prometheus.NewCounter(
-			prometheus.CounterOpts{
-				Name: "prometheus_sd_refresh_failures_total",
-				Help: "Number of refresh failures for the given SD mechanism.",
-				ConstLabels: prometheus.Labels{
-					"mechanism": opts.Mech,
-				},
-			}),
-		duration: prometheus.NewSummary(
-			prometheus.SummaryOpts{
-				Name:       "prometheus_sd_refresh_duration_seconds",
-				Help:       "The duration of a refresh in seconds for the given SD mechanism.",
-				Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
-				ConstLabels: prometheus.Labels{
-					"mechanism": opts.Mech,
-				},
-			}),
+		metrics:  m,
 	}
 
-	metrics := []prometheus.Collector{d.failures, d.duration}
-	if opts.Metrics != nil {
-		metrics = append(metrics, opts.Metrics...)
-	}
-
-	d.metricRegisterer = discovery.NewMetricRegisterer(opts.Registry, metrics)
-
 	return &d
 }
 
 // Run implements the Discoverer interface.
 func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
-	err := d.metricRegisterer.RegisterMetrics()
-	if err != nil {
-		level.Error(d.logger).Log("msg", "Unable to register metrics", "err", err.Error())
-		return
-	}
-	defer d.metricRegisterer.UnregisterMetrics()
-
 	// Get an initial set right away.
 	tgs, err := d.refresh(ctx)
 	if err != nil {
@@ -140,12 +106,12 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 	now := time.Now()
 	defer func() {
-		d.duration.Observe(time.Since(now).Seconds())
+		d.metrics.Duration.Observe(time.Since(now).Seconds())
 	}()
 
 	tgs, err := d.refreshf(ctx)
 	if err != nil {
-		d.failures.Inc()
+		d.metrics.Failures.Inc()
 	}
 	return tgs, err
 }
diff --git a/discovery/refresh/refresh_test.go b/discovery/refresh/refresh_test.go
index 12e7ab3be..407f0a7fa 100644
--- a/discovery/refresh/refresh_test.go
+++ b/discovery/refresh/refresh_test.go
@@ -24,6 +24,7 @@ import (
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -66,13 +67,18 @@ func TestRefresh(t *testing.T) {
 		return nil, fmt.Errorf("some error")
 	}
 	interval := time.Millisecond
+
+	metrics := discovery.NewRefreshMetrics(prometheus.NewRegistry())
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+
 	d := NewDiscovery(
 		Options{
-			Logger:   nil,
-			Mech:     "test",
-			Interval: interval,
-			RefreshF: refresh,
-			Registry: prometheus.NewRegistry(),
+			Logger:              nil,
+			Mech:                "test",
+			Interval:            interval,
+			RefreshF:            refresh,
+			MetricsInstantiator: metrics,
 		},
 	)
 
diff --git a/discovery/registry.go b/discovery/registry.go
index 13168a07a..1f491d4ca 100644
--- a/discovery/registry.go
+++ b/discovery/registry.go
@@ -24,6 +24,8 @@ import (
 
 	"gopkg.in/yaml.v2"
 
+	"github.com/prometheus/client_golang/prometheus"
+
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -258,3 +260,24 @@ func replaceYAMLTypeError(err error, oldTyp, newTyp reflect.Type) error {
 	}
 	return err
 }
+
+// RegisterSDMetrics registers the metrics used by service discovery mechanisms.
+// RegisterSDMetrics should be called only once during the lifetime of the Prometheus process.
+// There is no need for the Prometheus process to unregister the metrics.
+func RegisterSDMetrics(registerer prometheus.Registerer, rmm RefreshMetricsManager) (map[string]DiscovererMetrics, error) {
+	err := rmm.Register()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create service discovery refresh metrics")
+	}
+
+	metrics := make(map[string]DiscovererMetrics)
+	for _, conf := range configNames {
+		currentSdMetrics := conf.NewDiscovererMetrics(registerer, rmm)
+		err = currentSdMetrics.Register()
+		if err != nil {
+			return nil, fmt.Errorf("failed to create service discovery metrics")
+		}
+		metrics[conf.Name()] = currentSdMetrics
+	}
+	return metrics, nil
+}
diff --git a/discovery/scaleway/metrics.go b/discovery/scaleway/metrics.go
new file mode 100644
index 000000000..a8277d0fb
--- /dev/null
+++ b/discovery/scaleway/metrics.go
@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scaleway
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*scalewayMetrics)(nil)
+
+type scalewayMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *scalewayMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *scalewayMetrics) Unregister() {}
diff --git a/discovery/scaleway/scaleway.go b/discovery/scaleway/scaleway.go
index 86527b34e..f8e1a83f5 100644
--- a/discovery/scaleway/scaleway.go
+++ b/discovery/scaleway/scaleway.go
@@ -104,6 +104,13 @@ type SDConfig struct {
 	Role role `yaml:"role"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &scalewayMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 func (c SDConfig) Name() string {
 	return "scaleway"
 }
@@ -161,7 +168,7 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 }
 
 func (c SDConfig) NewDiscoverer(options discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(&c, options.Logger, options.Registerer)
+	return NewDiscovery(&c, options.Logger, options.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -178,7 +185,12 @@ func init() {
 // the Discoverer interface.
 type Discovery struct{}
 
-func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*refresh.Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*refresh.Discovery, error) {
+	m, ok := metrics.(*scalewayMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	r, err := newRefresher(conf)
 	if err != nil {
 		return nil, err
@@ -186,11 +198,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer)
 
 	return refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
-			Mech:     "scaleway",
-			Interval: time.Duration(conf.RefreshInterval),
-			RefreshF: r.refresh,
-			Registry: reg,
+			Logger:              logger,
+			Mech:                "scaleway",
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            r.refresh,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	), nil
 }
diff --git a/discovery/triton/metrics.go b/discovery/triton/metrics.go
new file mode 100644
index 000000000..3e34a840a
--- /dev/null
+++ b/discovery/triton/metrics.go
@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package triton
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*tritonMetrics)(nil)
+
+type tritonMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *tritonMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *tritonMetrics) Unregister() {}
diff --git a/discovery/triton/triton.go b/discovery/triton/triton.go
index 4839827ad..e56b7951b 100644
--- a/discovery/triton/triton.go
+++ b/discovery/triton/triton.go
@@ -70,12 +70,19 @@ type SDConfig struct {
 	Version         int              `yaml:"version"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &tritonMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "triton" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return New(opts.Logger, c, opts.Registerer)
+	return New(opts.Logger, c, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -139,7 +146,12 @@ type Discovery struct {
 }
 
 // New returns a new Discovery which periodically refreshes its targets.
-func New(logger log.Logger, conf *SDConfig, reg prometheus.Registerer) (*Discovery, error) {
+func New(logger log.Logger, conf *SDConfig, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*tritonMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	tls, err := config.NewTLSConfig(&conf.TLSConfig)
 	if err != nil {
 		return nil, err
@@ -161,11 +173,11 @@ func New(logger log.Logger, conf *SDConfig, reg prometheus.Registerer) (*Discove
 	}
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
-			Mech:     "triton",
-			Interval: time.Duration(conf.RefreshInterval),
-			RefreshF: d.refresh,
-			Registry: reg,
+			Logger:              logger,
+			Mech:                "triton",
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
 	return d, nil
diff --git a/discovery/triton/triton_test.go b/discovery/triton/triton_test.go
index fa51a2e47..e37693e6b 100644
--- a/discovery/triton/triton_test.go
+++ b/discovery/triton/triton_test.go
@@ -28,6 +28,8 @@ import (
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/prometheus/discovery"
 )
 
 var (
@@ -79,12 +81,26 @@ var (
 	}
 )
 
-func newTritonDiscovery(c SDConfig) (*Discovery, error) {
-	return New(nil, &c, prometheus.NewRegistry())
+func newTritonDiscovery(c SDConfig) (*Discovery, discovery.DiscovererMetrics, error) {
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	// TODO(ptodev): Add the ability to unregister refresh metrics.
+	metrics := c.NewDiscovererMetrics(reg, refreshMetrics)
+	err := metrics.Register()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	d, err := New(nil, &c, metrics)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return d, metrics, nil
 }
 
 func TestTritonSDNew(t *testing.T) {
-	td, err := newTritonDiscovery(conf)
+	td, m, err := newTritonDiscovery(conf)
 	require.NoError(t, err)
 	require.NotNil(t, td)
 	require.NotNil(t, td.client)
@@ -94,16 +110,17 @@ func TestTritonSDNew(t *testing.T) {
 	require.Equal(t, conf.DNSSuffix, td.sdConfig.DNSSuffix)
 	require.Equal(t, conf.Endpoint, td.sdConfig.Endpoint)
 	require.Equal(t, conf.Port, td.sdConfig.Port)
+	m.Unregister()
 }
 
 func TestTritonSDNewBadConfig(t *testing.T) {
-	td, err := newTritonDiscovery(badconf)
+	td, _, err := newTritonDiscovery(badconf)
 	require.Error(t, err)
 	require.Nil(t, td)
 }
 
 func TestTritonSDNewGroupsConfig(t *testing.T) {
-	td, err := newTritonDiscovery(groupsconf)
+	td, m, err := newTritonDiscovery(groupsconf)
 	require.NoError(t, err)
 	require.NotNil(t, td)
 	require.NotNil(t, td.client)
@@ -114,10 +131,11 @@ func TestTritonSDNewGroupsConfig(t *testing.T) {
 	require.Equal(t, groupsconf.Endpoint, td.sdConfig.Endpoint)
 	require.Equal(t, groupsconf.Groups, td.sdConfig.Groups)
 	require.Equal(t, groupsconf.Port, td.sdConfig.Port)
+	m.Unregister()
 }
 
 func TestTritonSDNewCNConfig(t *testing.T) {
-	td, err := newTritonDiscovery(cnconf)
+	td, m, err := newTritonDiscovery(cnconf)
 	require.NoError(t, err)
 	require.NotNil(t, td)
 	require.NotNil(t, td.client)
@@ -128,6 +146,7 @@ func TestTritonSDNewCNConfig(t *testing.T) {
 	require.Equal(t, cnconf.DNSSuffix, td.sdConfig.DNSSuffix)
 	require.Equal(t, cnconf.Endpoint, td.sdConfig.Endpoint)
 	require.Equal(t, cnconf.Port, td.sdConfig.Port)
+	m.Unregister()
 }
 
 func TestTritonSDRefreshNoTargets(t *testing.T) {
@@ -160,21 +179,23 @@ func TestTritonSDRefreshMultipleTargets(t *testing.T) {
 }
 
 func TestTritonSDRefreshNoServer(t *testing.T) {
-	td, _ := newTritonDiscovery(conf)
+	td, m, _ := newTritonDiscovery(conf)
 
 	_, err := td.refresh(context.Background())
 	require.Error(t, err)
 	require.True(t, strings.Contains(err.Error(), "an error occurred when requesting targets from the discovery endpoint"))
+	m.Unregister()
 }
 
 func TestTritonSDRefreshCancelled(t *testing.T) {
-	td, _ := newTritonDiscovery(conf)
+	td, m, _ := newTritonDiscovery(conf)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel()
 	_, err := td.refresh(ctx)
 	require.Error(t, err)
 	require.True(t, strings.Contains(err.Error(), context.Canceled.Error()))
+	m.Unregister()
 }
 
 func TestTritonSDRefreshCNsUUIDOnly(t *testing.T) {
@@ -211,8 +232,8 @@ func TestTritonSDRefreshCNsWithHostname(t *testing.T) {
 
 func testTritonSDRefresh(t *testing.T, c SDConfig, dstr string) []model.LabelSet {
 	var (
-		td, _ = newTritonDiscovery(c)
-		s     = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		td, m, _ = newTritonDiscovery(c)
+		s        = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			fmt.Fprintln(w, dstr)
 		}))
 	)
@@ -240,5 +261,7 @@ func testTritonSDRefresh(t *testing.T, c SDConfig, dstr string) []model.LabelSet
 	tg := tgs[0]
 	require.NotNil(t, tg)
 
+	m.Unregister()
+
 	return tg.Targets
 }
diff --git a/discovery/uyuni/metrics.go b/discovery/uyuni/metrics.go
new file mode 100644
index 000000000..6793b5920
--- /dev/null
+++ b/discovery/uyuni/metrics.go
@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package uyuni
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*uyuniMetrics)(nil)
+
+type uyuniMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *uyuniMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *uyuniMetrics) Unregister() {}
diff --git a/discovery/uyuni/uyuni.go b/discovery/uyuni/uyuni.go
index 744f3f96c..e885ef2e8 100644
--- a/discovery/uyuni/uyuni.go
+++ b/discovery/uyuni/uyuni.go
@@ -111,12 +111,19 @@ type Discovery struct {
 	logger          log.Logger
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &uyuniMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "uyuni" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.Registerer)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -204,7 +211,12 @@ func getEndpointInfoForSystems(
 }
 
 // NewDiscovery returns a uyuni discovery for the given configuration.
-func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*uyuniMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	apiURL, err := url.Parse(conf.Server)
 	if err != nil {
 		return nil, err
@@ -229,11 +241,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer)
 
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
-			Mech:     "uyuni",
-			Interval: time.Duration(conf.RefreshInterval),
-			RefreshF: d.refresh,
-			Registry: reg,
+			Logger:              logger,
+			Mech:                "uyuni",
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
 	return d, nil
diff --git a/discovery/uyuni/uyuni_test.go b/discovery/uyuni/uyuni_test.go
index fd03c88f1..09be23e2b 100644
--- a/discovery/uyuni/uyuni_test.go
+++ b/discovery/uyuni/uyuni_test.go
@@ -25,6 +25,7 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -37,7 +38,17 @@ func testUpdateServices(respHandler http.HandlerFunc) ([]*targetgroup.Group, err
 		Server: ts.URL,
 	}
 
-	md, err := NewDiscovery(&conf, nil, prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := conf.NewDiscovererMetrics(reg, refreshMetrics)
+	err := metrics.Register()
+	if err != nil {
+		return nil, err
+	}
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	md, err := NewDiscovery(&conf, nil, metrics)
 	if err != nil {
 		return nil, err
 	}
@@ -110,7 +121,14 @@ func TestUyuniSDSkipLogin(t *testing.T) {
 		Server: ts.URL,
 	}
 
-	md, err := NewDiscovery(&conf, nil, prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := conf.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	md, err := NewDiscovery(&conf, nil, metrics)
 	if err != nil {
 		t.Error(err)
 	}
diff --git a/discovery/vultr/metrics.go b/discovery/vultr/metrics.go
new file mode 100644
index 000000000..193436aad
--- /dev/null
+++ b/discovery/vultr/metrics.go
@@ -0,0 +1,32 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vultr
+
+import (
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*vultrMetrics)(nil)
+
+type vultrMetrics struct {
+	refreshMetrics discovery.RefreshMetricsInstantiator
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *vultrMetrics) Register() error {
+	return nil
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *vultrMetrics) Unregister() {}
diff --git a/discovery/vultr/vultr.go b/discovery/vultr/vultr.go
index 129800048..aaa9c64e4 100644
--- a/discovery/vultr/vultr.go
+++ b/discovery/vultr/vultr.go
@@ -74,12 +74,19 @@ type SDConfig struct {
 	Port            int            `yaml:"port"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &vultrMetrics{
+		refreshMetrics: rmi,
+	}
+}
+
 // Name returns the name of the Config.
 func (*SDConfig) Name() string { return "vultr" }
 
 // NewDiscoverer returns a Discoverer for the Config.
 func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
-	return NewDiscovery(c, opts.Logger, opts.Registerer)
+	return NewDiscovery(c, opts.Logger, opts.Metrics)
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -107,7 +114,12 @@ type Discovery struct {
 }
 
 // NewDiscovery returns a new Discovery which periodically refreshes its targets.
-func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
+func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) {
+	m, ok := metrics.(*vultrMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	d := &Discovery{
 		port: conf.Port,
 	}
@@ -130,11 +142,11 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, reg prometheus.Registerer)
 
 	d.Discovery = refresh.NewDiscovery(
 		refresh.Options{
-			Logger:   logger,
-			Mech:     "vultr",
-			Interval: time.Duration(conf.RefreshInterval),
-			RefreshF: d.refresh,
-			Registry: reg,
+			Logger:              logger,
+			Mech:                "vultr",
+			Interval:            time.Duration(conf.RefreshInterval),
+			RefreshF:            d.refresh,
+			MetricsInstantiator: m.refreshMetrics,
 		},
 	)
 	return d, nil
diff --git a/discovery/vultr/vultr_test.go b/discovery/vultr/vultr_test.go
index c50b11d2d..2f12a3552 100644
--- a/discovery/vultr/vultr_test.go
+++ b/discovery/vultr/vultr_test.go
@@ -23,6 +23,8 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/prometheus/discovery"
 )
 
 type VultrSDTestSuite struct {
@@ -47,7 +49,15 @@ func TestVultrSDRefresh(t *testing.T) {
 
 	cfg := DefaultSDConfig
 	cfg.HTTPClientConfig.BearerToken = APIKey
-	d, err := NewDiscovery(&cfg, log.NewNopLogger(), prometheus.NewRegistry())
+
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	defer metrics.Unregister()
+	defer refreshMetrics.Unregister()
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics)
 	require.NoError(t, err)
 	endpoint, err := url.Parse(sdMock.Mock.Endpoint())
 	require.NoError(t, err)
diff --git a/discovery/xds/kuma.go b/discovery/xds/kuma.go
index c74bc552c..d1d540aaf 100644
--- a/discovery/xds/kuma.go
+++ b/discovery/xds/kuma.go
@@ -58,6 +58,11 @@ const (
 
 type KumaSDConfig = SDConfig
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*KumaSDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return newDiscovererMetrics(reg, rmi)
+}
+
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
 func (c *KumaSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	*c = DefaultKumaSDConfig
@@ -97,7 +102,7 @@ func (c *KumaSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discover
 		logger = log.NewNopLogger()
 	}
 
-	return NewKumaHTTPDiscovery(c, logger, opts.Registerer)
+	return NewKumaHTTPDiscovery(c, logger, opts.Metrics)
 }
 
 func convertKumaV1MonitoringAssignment(assignment *MonitoringAssignment) []model.LabelSet {
@@ -153,7 +158,12 @@ func kumaMadsV1ResourceParser(resources []*anypb.Any, typeURL string) ([]model.L
 	return targets, nil
 }
 
-func NewKumaHTTPDiscovery(conf *KumaSDConfig, logger log.Logger, reg prometheus.Registerer) (discovery.Discoverer, error) {
+func NewKumaHTTPDiscovery(conf *KumaSDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (discovery.Discoverer, error) {
+	m, ok := metrics.(*xdsMetrics)
+	if !ok {
+		return nil, fmt.Errorf("invalid discovery metrics type")
+	}
+
 	// Default to "prometheus" if hostname is unavailable.
 	clientID := conf.ClientID
 	if clientID == "" {
@@ -189,36 +199,8 @@ func NewKumaHTTPDiscovery(conf *KumaSDConfig, logger log.Logger, reg prometheus.
 		refreshInterval: time.Duration(conf.RefreshInterval),
 		source:          "kuma",
 		parseResources:  kumaMadsV1ResourceParser,
-		fetchFailuresCount: prometheus.NewCounter(
-			prometheus.CounterOpts{
-				Namespace: namespace,
-				Name:      "sd_kuma_fetch_failures_total",
-				Help:      "The number of Kuma MADS fetch call failures.",
-			}),
-		fetchSkipUpdateCount: prometheus.NewCounter(
-			prometheus.CounterOpts{
-				Namespace: namespace,
-				Name:      "sd_kuma_fetch_skipped_updates_total",
-				Help:      "The number of Kuma MADS fetch calls that result in no updates to the targets.",
-			}),
-		fetchDuration: prometheus.NewSummary(
-			prometheus.SummaryOpts{
-				Namespace:  namespace,
-				Name:       "sd_kuma_fetch_duration_seconds",
-				Help:       "The duration of a Kuma MADS fetch call.",
-				Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
-			},
-		),
+		metrics:         m,
 	}
 
-	d.metricRegisterer = discovery.NewMetricRegisterer(
-		reg,
-		[]prometheus.Collector{
-			d.fetchFailuresCount,
-			d.fetchSkipUpdateCount,
-			d.fetchDuration,
-		},
-	)
-
 	return d, nil
 }
diff --git a/discovery/xds/kuma_test.go b/discovery/xds/kuma_test.go
index 5e2417c96..cfb9cbac5 100644
--- a/discovery/xds/kuma_test.go
+++ b/discovery/xds/kuma_test.go
@@ -28,6 +28,7 @@ import (
 	"google.golang.org/protobuf/types/known/anypb"
 	"gopkg.in/yaml.v2"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -108,7 +109,16 @@ func getKumaMadsV1DiscoveryResponse(resources ...*MonitoringAssignment) (*v3.Dis
 }
 
 func newKumaTestHTTPDiscovery(c KumaSDConfig) (*fetchDiscovery, error) {
-	kd, err := NewKumaHTTPDiscovery(&c, nopLogger, prometheus.NewRegistry())
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	// TODO(ptodev): Add the ability to unregister refresh metrics.
+	metrics := c.NewDiscovererMetrics(reg, refreshMetrics)
+	err := metrics.Register()
+	if err != nil {
+		return nil, err
+	}
+
+	kd, err := NewKumaHTTPDiscovery(&c, nopLogger, metrics)
 	if err != nil {
 		return nil, err
 	}
@@ -207,6 +217,8 @@ func TestNewKumaHTTPDiscovery(t *testing.T) {
 	require.Equal(t, KumaMadsV1ResourceTypeURL, resClient.ResourceTypeURL())
 	require.Equal(t, kumaConf.ClientID, resClient.ID())
 	require.Equal(t, KumaMadsV1ResourceType, resClient.config.ResourceType)
+
+	kd.metrics.Unregister()
 }
 
 func TestKumaHTTPDiscoveryRefresh(t *testing.T) {
@@ -301,4 +313,6 @@ tls_config:
 	case <-ch:
 		require.Fail(t, "no update expected")
 	}
+
+	kd.metrics.Unregister()
 }
diff --git a/discovery/xds/metrics.go b/discovery/xds/metrics.go
new file mode 100644
index 000000000..597d51656
--- /dev/null
+++ b/discovery/xds/metrics.go
@@ -0,0 +1,73 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package xds
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/prometheus/prometheus/discovery"
+)
+
+var _ discovery.DiscovererMetrics = (*xdsMetrics)(nil)
+
+type xdsMetrics struct {
+	fetchDuration        prometheus.Summary
+	fetchSkipUpdateCount prometheus.Counter
+	fetchFailuresCount   prometheus.Counter
+
+	metricRegisterer discovery.MetricRegisterer
+}
+
+func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	m := &xdsMetrics{
+		fetchFailuresCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      "sd_kuma_fetch_failures_total",
+				Help:      "The number of Kuma MADS fetch call failures.",
+			}),
+		fetchSkipUpdateCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      "sd_kuma_fetch_skipped_updates_total",
+				Help:      "The number of Kuma MADS fetch calls that result in no updates to the targets.",
+			}),
+		fetchDuration: prometheus.NewSummary(
+			prometheus.SummaryOpts{
+				Namespace:  namespace,
+				Name:       "sd_kuma_fetch_duration_seconds",
+				Help:       "The duration of a Kuma MADS fetch call.",
+				Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+			},
+		),
+	}
+
+	m.metricRegisterer = discovery.NewMetricRegisterer(reg, []prometheus.Collector{
+		m.fetchFailuresCount,
+		m.fetchSkipUpdateCount,
+		m.fetchDuration,
+	})
+
+	return m
+}
+
+// Register implements discovery.DiscovererMetrics.
+func (m *xdsMetrics) Register() error {
+	return m.metricRegisterer.RegisterMetrics()
+}
+
+// Unregister implements discovery.DiscovererMetrics.
+func (m *xdsMetrics) Unregister() {
+	m.metricRegisterer.UnregisterMetrics()
+}
diff --git a/discovery/xds/xds.go b/discovery/xds/xds.go
index 8b6cb7e65..8191d6be1 100644
--- a/discovery/xds/xds.go
+++ b/discovery/xds/xds.go
@@ -20,7 +20,6 @@ import (
 	v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
-	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"google.golang.org/protobuf/encoding/protojson"
@@ -107,20 +106,10 @@ type fetchDiscovery struct {
 	parseResources resourceParser
 	logger         log.Logger
 
-	fetchDuration        prometheus.Summary
-	fetchSkipUpdateCount prometheus.Counter
-	fetchFailuresCount   prometheus.Counter
-
-	metricRegisterer discovery.MetricRegisterer
+	metrics *xdsMetrics
 }
 
 func (d *fetchDiscovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
-	err := d.metricRegisterer.RegisterMetrics()
-	if err != nil {
-		level.Error(d.logger).Log("msg", "Unable to register metrics", "err", err.Error())
-		return
-	}
-	defer d.metricRegisterer.UnregisterMetrics()
 	defer d.client.Close()
 
 	ticker := time.NewTicker(d.refreshInterval)
@@ -141,7 +130,7 @@ func (d *fetchDiscovery) poll(ctx context.Context, ch chan<- []*targetgroup.Grou
 	t0 := time.Now()
 	response, err := d.client.Fetch(ctx)
 	elapsed := time.Since(t0)
-	d.fetchDuration.Observe(elapsed.Seconds())
+	d.metrics.fetchDuration.Observe(elapsed.Seconds())
 
 	// Check the context before in order to exit early.
 	select {
@@ -152,20 +141,20 @@ func (d *fetchDiscovery) poll(ctx context.Context, ch chan<- []*targetgroup.Grou
 
 	if err != nil {
 		level.Error(d.logger).Log("msg", "error parsing resources", "err", err)
-		d.fetchFailuresCount.Inc()
+		d.metrics.fetchFailuresCount.Inc()
 		return
 	}
 
 	if response == nil {
 		// No update needed.
-		d.fetchSkipUpdateCount.Inc()
+		d.metrics.fetchSkipUpdateCount.Inc()
 		return
 	}
 
 	parsedTargets, err := d.parseResources(response.Resources, response.TypeUrl)
 	if err != nil {
 		level.Error(d.logger).Log("msg", "error parsing resources", "err", err)
-		d.fetchFailuresCount.Inc()
+		d.metrics.fetchFailuresCount.Inc()
 		return
 	}
 
diff --git a/discovery/xds/xds_test.go b/discovery/xds/xds_test.go
index f57fff996..7cce021c5 100644
--- a/discovery/xds/xds_test.go
+++ b/discovery/xds/xds_test.go
@@ -29,24 +29,15 @@ import (
 	"go.uber.org/goleak"
 	"google.golang.org/protobuf/types/known/anypb"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
-var (
-	sdConf = SDConfig{
-		Server:          "http://127.0.0.1",
-		RefreshInterval: model.Duration(10 * time.Second),
-		ClientID:        "test-id",
-	}
-
-	testFetchFailuresCount = prometheus.NewCounter(
-		prometheus.CounterOpts{})
-	testFetchSkipUpdateCount = prometheus.NewCounter(
-		prometheus.CounterOpts{})
-	testFetchDuration = prometheus.NewSummary(
-		prometheus.SummaryOpts{},
-	)
-)
+var sdConf = SDConfig{
+	Server:          "http://127.0.0.1",
+	RefreshInterval: model.Duration(10 * time.Second),
+	ClientID:        "test-id",
+}
 
 func TestMain(m *testing.M) {
 	goleak.VerifyTestMain(m)
@@ -133,12 +124,22 @@ func TestPollingRefreshSkipUpdate(t *testing.T) {
 			return nil, nil
 		},
 	}
+
+	cfg := &SDConfig{}
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+
+	xdsMetrics, ok := metrics.(*xdsMetrics)
+	if !ok {
+		require.Fail(t, "invalid discovery metrics type")
+	}
+
 	pd := &fetchDiscovery{
-		client:               rc,
-		logger:               nopLogger,
-		fetchDuration:        testFetchDuration,
-		fetchFailuresCount:   testFetchFailuresCount,
-		fetchSkipUpdateCount: testFetchSkipUpdateCount,
+		client:  rc,
+		logger:  nopLogger,
+		metrics: xdsMetrics,
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -155,6 +156,9 @@ func TestPollingRefreshSkipUpdate(t *testing.T) {
 	case <-ch:
 		require.Fail(t, "no update expected")
 	}
+
+	metrics.Unregister()
+	refreshMetrics.Unregister()
 }
 
 func TestPollingRefreshAttachesGroupMetadata(t *testing.T) {
@@ -167,13 +171,18 @@ func TestPollingRefreshAttachesGroupMetadata(t *testing.T) {
 			return &v3.DiscoveryResponse{}, nil
 		},
 	}
+
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := newDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+	xdsMetrics, ok := metrics.(*xdsMetrics)
+	require.True(t, ok)
+
 	pd := &fetchDiscovery{
-		source:               source,
-		client:               rc,
-		logger:               nopLogger,
-		fetchDuration:        testFetchDuration,
-		fetchFailuresCount:   testFetchFailuresCount,
-		fetchSkipUpdateCount: testFetchSkipUpdateCount,
+		source: source,
+		client: rc,
+		logger: nopLogger,
 		parseResources: constantResourceParser([]model.LabelSet{
 			{
 				"__meta_custom_xds_label": "a-value",
@@ -186,6 +195,7 @@ func TestPollingRefreshAttachesGroupMetadata(t *testing.T) {
 				"instance":                "prometheus-02",
 			},
 		}, nil),
+		metrics: xdsMetrics,
 	}
 	ch := make(chan []*targetgroup.Group, 1)
 	pd.poll(context.Background(), ch)
@@ -202,6 +212,9 @@ func TestPollingRefreshAttachesGroupMetadata(t *testing.T) {
 	target2 := group.Targets[1]
 	require.Contains(t, target2, model.LabelName("__meta_custom_xds_label"))
 	require.Equal(t, model.LabelValue("a-value"), target2["__meta_custom_xds_label"])
+
+	metrics.Unregister()
+	refreshMetrics.Unregister()
 }
 
 func TestPollingDisappearingTargets(t *testing.T) {
@@ -243,14 +256,23 @@ func TestPollingDisappearingTargets(t *testing.T) {
 		}, nil
 	}
 
+	cfg := &SDConfig{}
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
+	require.NoError(t, metrics.Register())
+
+	xdsMetrics, ok := metrics.(*xdsMetrics)
+	if !ok {
+		require.Fail(t, "invalid discovery metrics type")
+	}
+
 	pd := &fetchDiscovery{
-		source:               source,
-		client:               rc,
-		logger:               nopLogger,
-		fetchDuration:        testFetchDuration,
-		fetchFailuresCount:   testFetchFailuresCount,
-		fetchSkipUpdateCount: testFetchSkipUpdateCount,
-		parseResources:       parser,
+		source:         source,
+		client:         rc,
+		logger:         nopLogger,
+		parseResources: parser,
+		metrics:        xdsMetrics,
 	}
 
 	ch := make(chan []*targetgroup.Group, 1)
@@ -271,4 +293,7 @@ func TestPollingDisappearingTargets(t *testing.T) {
 
 	require.Equal(t, source, groups[0].Source)
 	require.Len(t, groups[0].Targets, 1)
+
+	metrics.Unregister()
+	refreshMetrics.Unregister()
 }
diff --git a/discovery/zookeeper/zookeeper.go b/discovery/zookeeper/zookeeper.go
index 308d63a5f..7d8761527 100644
--- a/discovery/zookeeper/zookeeper.go
+++ b/discovery/zookeeper/zookeeper.go
@@ -25,6 +25,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-zookeeper/zk"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 
 	"github.com/prometheus/prometheus/discovery"
@@ -56,6 +57,11 @@ type ServersetSDConfig struct {
 	Timeout model.Duration `yaml:"timeout,omitempty"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*ServersetSDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &discovery.NoopDiscovererMetrics{}
+}
+
 // Name returns the name of the Config.
 func (*ServersetSDConfig) Name() string { return "serverset" }
 
@@ -93,6 +99,11 @@ type NerveSDConfig struct {
 	Timeout model.Duration `yaml:"timeout,omitempty"`
 }
 
+// NewDiscovererMetrics implements discovery.Config.
+func (*NerveSDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+	return &discovery.NoopDiscovererMetrics{}
+}
+
 // Name returns the name of the Config.
 func (*NerveSDConfig) Name() string { return "nerve" }
 
diff --git a/documentation/examples/custom-sd/adapter-usage/main.go b/documentation/examples/custom-sd/adapter-usage/main.go
index f4bba7394..bfbca7b70 100644
--- a/documentation/examples/custom-sd/adapter-usage/main.go
+++ b/documentation/examples/custom-sd/adapter-usage/main.go
@@ -28,8 +28,10 @@ import (
 	"github.com/alecthomas/kingpin/v2"
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 
+	prom_discovery "github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 	"github.com/prometheus/prometheus/documentation/examples/custom-sd/adapter"
 	"github.com/prometheus/prometheus/util/strutil"
@@ -273,7 +275,16 @@ func main() {
 		level.Error(logger).Log("msg", "failed to create discovery metrics", "err", err)
 		os.Exit(1)
 	}
-	sdAdapter := adapter.NewAdapter(ctx, *outputFile, "exampleSD", disc, logger)
+
+	reg := prometheus.NewRegistry()
+	refreshMetrics := prom_discovery.NewRefreshMetrics(reg)
+	metrics, err := prom_discovery.RegisterSDMetrics(reg, refreshMetrics)
+	if err != nil {
+		level.Error(logger).Log("msg", "failed to register service discovery metrics", "err", err)
+		os.Exit(1)
+	}
+
+	sdAdapter := adapter.NewAdapter(ctx, *outputFile, "exampleSD", disc, logger, metrics, reg)
 	sdAdapter.Run()
 
 	<-ctx.Done()
diff --git a/documentation/examples/custom-sd/adapter/adapter.go b/documentation/examples/custom-sd/adapter/adapter.go
index 7fbf94aa9..dcf5a2b78 100644
--- a/documentation/examples/custom-sd/adapter/adapter.go
+++ b/documentation/examples/custom-sd/adapter/adapter.go
@@ -163,12 +163,12 @@ func (a *Adapter) Run() {
 }
 
 // NewAdapter creates a new instance of Adapter.
-func NewAdapter(ctx context.Context, file, name string, d discovery.Discoverer, logger log.Logger) *Adapter {
+func NewAdapter(ctx context.Context, file, name string, d discovery.Discoverer, logger log.Logger, sdMetrics map[string]discovery.DiscovererMetrics, registerer prometheus.Registerer) *Adapter {
 	return &Adapter{
 		ctx:     ctx,
 		disc:    d,
 		groups:  make(map[string]*customSD),
-		manager: discovery.NewManager(ctx, logger, prometheus.NewRegistry()),
+		manager: discovery.NewManager(ctx, logger, registerer, sdMetrics),
 		output:  file,
 		name:    name,
 		logger:  logger,
diff --git a/documentation/examples/custom-sd/adapter/adapter_test.go b/documentation/examples/custom-sd/adapter/adapter_test.go
index 8e5920eb4..329ca8c29 100644
--- a/documentation/examples/custom-sd/adapter/adapter_test.go
+++ b/documentation/examples/custom-sd/adapter/adapter_test.go
@@ -18,9 +18,11 @@ import (
 	"os"
 	"testing"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 
+	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
@@ -227,6 +229,12 @@ func TestWriteOutput(t *testing.T) {
 	defer os.Remove(tmpfile.Name())
 	tmpfile.Close()
 	require.NoError(t, err)
-	adapter := NewAdapter(ctx, tmpfile.Name(), "test_sd", nil, nil)
+
+	reg := prometheus.NewRegistry()
+	refreshMetrics := discovery.NewRefreshMetrics(reg)
+	sdMetrics, err := discovery.RegisterSDMetrics(reg, refreshMetrics)
+	require.NoError(t, err)
+
+	adapter := NewAdapter(ctx, tmpfile.Name(), "test_sd", nil, nil, sdMetrics, reg)
 	require.NoError(t, adapter.writeOutput())
 }

From 583f3e587cf7b8f94b325ae72906815d5be480f4 Mon Sep 17 00:00:00 2001
From: Filip Petkovski <filip.petkovsky@gmail.com>
Date: Tue, 23 Jan 2024 17:02:14 +0100
Subject: [PATCH 106/115] Optimize histogram iterators (#13340)

Optimize histogram iterators

Histogram iterators allocate new objects in the AtHistogram and
AtFloatHistogram methods, which makes calculating rates over long
ranges expensive.

In #13215 we allowed an existing object to be reused
when converting an integer histogram to a float histogram. This commit follows
the same idea and allows injecting an existing object in the AtHistogram and
AtFloatHistogram methods. When the injected value is nil, iterators allocate
new histograms, otherwise they populate and return the injected object.

The commit also adds a CopyTo method to Histogram and FloatHistogram which
is used in the BufferedIterator to overwrite items in the ring instead of making
new copies.

Note that a specialized HPoint pool is needed for all of this to work
(`matrixSelectorHPool`).

---------

Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
---
 cmd/promtool/tsdb.go                    |   8 +-
 model/histogram/float_histogram.go      |  40 +++++++-
 model/histogram/float_histogram_test.go | 112 ++++++++++++++++++++++
 model/histogram/histogram.go            |  32 ++++++-
 model/histogram/histogram_test.go       | 122 ++++++++++++++++++++++++
 promql/engine.go                        |  67 +++++++++----
 promql/value.go                         |   4 +-
 rules/manager_test.go                   |   2 +-
 storage/buffer.go                       |  35 +++++--
 storage/buffer_test.go                  |   8 +-
 storage/memoized_iterator.go            |   4 +-
 storage/merge.go                        |   8 +-
 storage/merge_test.go                   |  12 +--
 storage/remote/codec.go                 |   8 +-
 storage/remote/codec_test.go            |  18 ++--
 storage/series.go                       |  12 +--
 tsdb/block_test.go                      |   4 +-
 tsdb/chunkenc/chunk.go                  |  39 +++++---
 tsdb/chunkenc/float_histogram.go        |  53 +++++++---
 tsdb/chunkenc/float_histogram_test.go   |  12 +--
 tsdb/chunkenc/histogram.go              | 115 ++++++++++++++++------
 tsdb/chunkenc/histogram_test.go         |  26 ++---
 tsdb/chunkenc/xor.go                    |   4 +-
 tsdb/chunks/chunks.go                   |   4 +-
 tsdb/compact_test.go                    |   4 +-
 tsdb/db_test.go                         |   8 +-
 tsdb/head_test.go                       |   8 +-
 tsdb/querier.go                         |  24 ++---
 tsdb/querier_test.go                    |   8 +-
 tsdb/tsdbblockutil.go                   |   4 +-
 web/federate.go                         |   2 +-
 31 files changed, 621 insertions(+), 186 deletions(-)

diff --git a/cmd/promtool/tsdb.go b/cmd/promtool/tsdb.go
index a9239d937..4bba8421c 100644
--- a/cmd/promtool/tsdb.go
+++ b/cmd/promtool/tsdb.go
@@ -667,7 +667,7 @@ func analyzeCompaction(ctx context.Context, block tsdb.BlockReader, indexr tsdb.
 				it := fhchk.Iterator(nil)
 				bucketCount := 0
 				for it.Next() == chunkenc.ValFloatHistogram {
-					_, f := it.AtFloatHistogram()
+					_, f := it.AtFloatHistogram(nil)
 					bucketCount += len(f.PositiveBuckets)
 					bucketCount += len(f.NegativeBuckets)
 				}
@@ -682,7 +682,7 @@ func analyzeCompaction(ctx context.Context, block tsdb.BlockReader, indexr tsdb.
 				it := hchk.Iterator(nil)
 				bucketCount := 0
 				for it.Next() == chunkenc.ValHistogram {
-					_, f := it.AtHistogram()
+					_, f := it.AtHistogram(nil)
 					bucketCount += len(f.PositiveBuckets)
 					bucketCount += len(f.NegativeBuckets)
 				}
@@ -745,11 +745,11 @@ func dumpSamples(ctx context.Context, path string, mint, maxt int64, match []str
 			fmt.Printf("%s %g %d\n", lbs, val, ts)
 		}
 		for it.Next() == chunkenc.ValFloatHistogram {
-			ts, fh := it.AtFloatHistogram()
+			ts, fh := it.AtFloatHistogram(nil)
 			fmt.Printf("%s %s %d\n", lbs, fh.String(), ts)
 		}
 		for it.Next() == chunkenc.ValHistogram {
-			ts, h := it.AtHistogram()
+			ts, h := it.AtHistogram(nil)
 			fmt.Printf("%s %s %d\n", lbs, h.String(), ts)
 		}
 		if it.Err() != nil {
diff --git a/model/histogram/float_histogram.go b/model/histogram/float_histogram.go
index bf89f2a47..19a92b3d5 100644
--- a/model/histogram/float_histogram.go
+++ b/model/histogram/float_histogram.go
@@ -53,21 +53,28 @@ type FloatHistogram struct {
 
 // Copy returns a deep copy of the Histogram.
 func (h *FloatHistogram) Copy() *FloatHistogram {
-	c := *h
+	c := FloatHistogram{
+		CounterResetHint: h.CounterResetHint,
+		Schema:           h.Schema,
+		ZeroThreshold:    h.ZeroThreshold,
+		ZeroCount:        h.ZeroCount,
+		Count:            h.Count,
+		Sum:              h.Sum,
+	}
 
-	if h.PositiveSpans != nil {
+	if len(h.PositiveSpans) != 0 {
 		c.PositiveSpans = make([]Span, len(h.PositiveSpans))
 		copy(c.PositiveSpans, h.PositiveSpans)
 	}
-	if h.NegativeSpans != nil {
+	if len(h.NegativeSpans) != 0 {
 		c.NegativeSpans = make([]Span, len(h.NegativeSpans))
 		copy(c.NegativeSpans, h.NegativeSpans)
 	}
-	if h.PositiveBuckets != nil {
+	if len(h.PositiveBuckets) != 0 {
 		c.PositiveBuckets = make([]float64, len(h.PositiveBuckets))
 		copy(c.PositiveBuckets, h.PositiveBuckets)
 	}
-	if h.NegativeBuckets != nil {
+	if len(h.NegativeBuckets) != 0 {
 		c.NegativeBuckets = make([]float64, len(h.NegativeBuckets))
 		copy(c.NegativeBuckets, h.NegativeBuckets)
 	}
@@ -75,6 +82,29 @@ func (h *FloatHistogram) Copy() *FloatHistogram {
 	return &c
 }
 
+// CopyTo makes a deep copy into the given FloatHistogram.
+// The destination object has to be a non-nil pointer.
+func (h *FloatHistogram) CopyTo(to *FloatHistogram) {
+	to.CounterResetHint = h.CounterResetHint
+	to.Schema = h.Schema
+	to.ZeroThreshold = h.ZeroThreshold
+	to.ZeroCount = h.ZeroCount
+	to.Count = h.Count
+	to.Sum = h.Sum
+
+	to.PositiveSpans = resize(to.PositiveSpans, len(h.PositiveSpans))
+	copy(to.PositiveSpans, h.PositiveSpans)
+
+	to.NegativeSpans = resize(to.NegativeSpans, len(h.NegativeSpans))
+	copy(to.NegativeSpans, h.NegativeSpans)
+
+	to.PositiveBuckets = resize(to.PositiveBuckets, len(h.PositiveBuckets))
+	copy(to.PositiveBuckets, h.PositiveBuckets)
+
+	to.NegativeBuckets = resize(to.NegativeBuckets, len(h.NegativeBuckets))
+	copy(to.NegativeBuckets, h.NegativeBuckets)
+}
+
 // CopyToSchema works like Copy, but the returned deep copy has the provided
 // target schema, which must be ≤ the original schema (i.e. it must have a lower
 // resolution).
diff --git a/model/histogram/float_histogram_test.go b/model/histogram/float_histogram_test.go
index 3d20960f6..49fb77ab0 100644
--- a/model/histogram/float_histogram_test.go
+++ b/model/histogram/float_histogram_test.go
@@ -142,6 +142,118 @@ func TestFloatHistogramMul(t *testing.T) {
 	}
 }
 
+func TestFloatHistogramCopy(t *testing.T) {
+	cases := []struct {
+		name     string
+		orig     *FloatHistogram
+		expected *FloatHistogram
+	}{
+		{
+			name:     "without buckets",
+			orig:     &FloatHistogram{},
+			expected: &FloatHistogram{},
+		},
+		{
+			name: "with buckets",
+			orig: &FloatHistogram{
+				PositiveSpans:   []Span{{-2, 1}},
+				PositiveBuckets: []float64{1, 3, -3, 42},
+				NegativeSpans:   []Span{{3, 2}},
+				NegativeBuckets: []float64{5, 3, 1.234e5, 1000},
+			},
+			expected: &FloatHistogram{
+				PositiveSpans:   []Span{{-2, 1}},
+				PositiveBuckets: []float64{1, 3, -3, 42},
+				NegativeSpans:   []Span{{3, 2}},
+				NegativeBuckets: []float64{5, 3, 1.234e5, 1000},
+			},
+		},
+		{
+			name: "with empty buckets and non empty capacity",
+			orig: &FloatHistogram{
+				PositiveSpans:   make([]Span, 0, 1),
+				PositiveBuckets: make([]float64, 0, 1),
+				NegativeSpans:   make([]Span, 0, 1),
+				NegativeBuckets: make([]float64, 0, 1),
+			},
+			expected: &FloatHistogram{},
+		},
+	}
+
+	for _, tcase := range cases {
+		t.Run(tcase.name, func(t *testing.T) {
+			hCopy := tcase.orig.Copy()
+
+			// Modify a primitive value in the original histogram.
+			tcase.orig.Sum++
+			require.Equal(t, tcase.expected, hCopy)
+			assertDeepCopyFHSpans(t, tcase.orig, hCopy, tcase.expected)
+		})
+	}
+}
+
+func TestFloatHistogramCopyTo(t *testing.T) {
+	cases := []struct {
+		name     string
+		orig     *FloatHistogram
+		expected *FloatHistogram
+	}{
+		{
+			name:     "without buckets",
+			orig:     &FloatHistogram{},
+			expected: &FloatHistogram{},
+		},
+		{
+			name: "with buckets",
+			orig: &FloatHistogram{
+				PositiveSpans:   []Span{{-2, 1}},
+				PositiveBuckets: []float64{1, 3, -3, 42},
+				NegativeSpans:   []Span{{3, 2}},
+				NegativeBuckets: []float64{5, 3, 1.234e5, 1000},
+			},
+			expected: &FloatHistogram{
+				PositiveSpans:   []Span{{-2, 1}},
+				PositiveBuckets: []float64{1, 3, -3, 42},
+				NegativeSpans:   []Span{{3, 2}},
+				NegativeBuckets: []float64{5, 3, 1.234e5, 1000},
+			},
+		},
+		{
+			name: "with empty buckets and non empty capacity",
+			orig: &FloatHistogram{
+				PositiveSpans:   make([]Span, 0, 1),
+				PositiveBuckets: make([]float64, 0, 1),
+				NegativeSpans:   make([]Span, 0, 1),
+				NegativeBuckets: make([]float64, 0, 1),
+			},
+			expected: &FloatHistogram{},
+		},
+	}
+
+	for _, tcase := range cases {
+		t.Run(tcase.name, func(t *testing.T) {
+			hCopy := &FloatHistogram{}
+			tcase.orig.CopyTo(hCopy)
+
+			// Modify a primitive value in the original histogram.
+			tcase.orig.Sum++
+			require.Equal(t, tcase.expected, hCopy)
+			assertDeepCopyFHSpans(t, tcase.orig, hCopy, tcase.expected)
+		})
+	}
+}
+
+func assertDeepCopyFHSpans(t *testing.T, orig, hCopy, expected *FloatHistogram) {
+	// Do an in-place expansion of an original spans slice.
+	orig.PositiveSpans = expandSpans(orig.PositiveSpans)
+	orig.PositiveSpans[len(orig.PositiveSpans)-1] = Span{1, 2}
+
+	hCopy.PositiveSpans = expandSpans(hCopy.PositiveSpans)
+	expected.PositiveSpans = expandSpans(expected.PositiveSpans)
+	// Expand the copy spans and assert that modifying the original has not affected the copy.
+	require.Equal(t, expected, hCopy)
+}
+
 func TestFloatHistogramDiv(t *testing.T) {
 	cases := []struct {
 		name     string
diff --git a/model/histogram/histogram.go b/model/histogram/histogram.go
index f4d292b34..d40adeb62 100644
--- a/model/histogram/histogram.go
+++ b/model/histogram/histogram.go
@@ -83,7 +83,14 @@ type Span struct {
 
 // Copy returns a deep copy of the Histogram.
 func (h *Histogram) Copy() *Histogram {
-	c := *h
+	c := Histogram{
+		CounterResetHint: h.CounterResetHint,
+		Schema:           h.Schema,
+		ZeroThreshold:    h.ZeroThreshold,
+		ZeroCount:        h.ZeroCount,
+		Count:            h.Count,
+		Sum:              h.Sum,
+	}
 
 	if len(h.PositiveSpans) != 0 {
 		c.PositiveSpans = make([]Span, len(h.PositiveSpans))
@@ -105,6 +112,29 @@ func (h *Histogram) Copy() *Histogram {
 	return &c
 }
 
+// CopyTo makes a deep copy into the given Histogram object.
+// The destination object has to be a non-nil pointer.
+func (h *Histogram) CopyTo(to *Histogram) {
+	to.CounterResetHint = h.CounterResetHint
+	to.Schema = h.Schema
+	to.ZeroThreshold = h.ZeroThreshold
+	to.ZeroCount = h.ZeroCount
+	to.Count = h.Count
+	to.Sum = h.Sum
+
+	to.PositiveSpans = resize(to.PositiveSpans, len(h.PositiveSpans))
+	copy(to.PositiveSpans, h.PositiveSpans)
+
+	to.NegativeSpans = resize(to.NegativeSpans, len(h.NegativeSpans))
+	copy(to.NegativeSpans, h.NegativeSpans)
+
+	to.PositiveBuckets = resize(to.PositiveBuckets, len(h.PositiveBuckets))
+	copy(to.PositiveBuckets, h.PositiveBuckets)
+
+	to.NegativeBuckets = resize(to.NegativeBuckets, len(h.NegativeBuckets))
+	copy(to.NegativeBuckets, h.NegativeBuckets)
+}
+
 // String returns a string representation of the Histogram.
 func (h *Histogram) String() string {
 	var sb strings.Builder
diff --git a/model/histogram/histogram_test.go b/model/histogram/histogram_test.go
index 9a64faaaa..14a948e64 100644
--- a/model/histogram/histogram_test.go
+++ b/model/histogram/histogram_test.go
@@ -604,6 +604,128 @@ func TestHistogramEquals(t *testing.T) {
 	notEquals(*hStale, *hNaN)
 }
 
+func TestHistogramCopy(t *testing.T) {
+	cases := []struct {
+		name     string
+		orig     *Histogram
+		expected *Histogram
+	}{
+		{
+			name:     "without buckets",
+			orig:     &Histogram{},
+			expected: &Histogram{},
+		},
+		{
+			name: "with buckets",
+			orig: &Histogram{
+				PositiveSpans:   []Span{{-2, 1}},
+				PositiveBuckets: []int64{1, 3, -3, 42},
+				NegativeSpans:   []Span{{3, 2}},
+				NegativeBuckets: []int64{5, 3, 1.234e5, 1000},
+			},
+			expected: &Histogram{
+				PositiveSpans:   []Span{{-2, 1}},
+				PositiveBuckets: []int64{1, 3, -3, 42},
+				NegativeSpans:   []Span{{3, 2}},
+				NegativeBuckets: []int64{5, 3, 1.234e5, 1000},
+			},
+		},
+		{
+			name: "with empty buckets and non empty capacity",
+			orig: &Histogram{
+				PositiveSpans:   make([]Span, 0, 1),
+				PositiveBuckets: make([]int64, 0, 1),
+				NegativeSpans:   make([]Span, 0, 1),
+				NegativeBuckets: make([]int64, 0, 1),
+			},
+			expected: &Histogram{},
+		},
+	}
+
+	for _, tcase := range cases {
+		t.Run(tcase.name, func(t *testing.T) {
+			hCopy := tcase.orig.Copy()
+
+			// Modify a primitive value in the original histogram.
+			tcase.orig.Sum++
+			require.Equal(t, tcase.expected, hCopy)
+			assertDeepCopyHSpans(t, tcase.orig, hCopy, tcase.expected)
+		})
+	}
+}
+
+func TestHistogramCopyTo(t *testing.T) {
+	cases := []struct {
+		name     string
+		orig     *Histogram
+		expected *Histogram
+	}{
+		{
+			name:     "without buckets",
+			orig:     &Histogram{},
+			expected: &Histogram{},
+		},
+		{
+			name: "with buckets",
+			orig: &Histogram{
+				PositiveSpans:   []Span{{-2, 1}},
+				PositiveBuckets: []int64{1, 3, -3, 42},
+				NegativeSpans:   []Span{{3, 2}},
+				NegativeBuckets: []int64{5, 3, 1.234e5, 1000},
+			},
+			expected: &Histogram{
+				PositiveSpans:   []Span{{-2, 1}},
+				PositiveBuckets: []int64{1, 3, -3, 42},
+				NegativeSpans:   []Span{{3, 2}},
+				NegativeBuckets: []int64{5, 3, 1.234e5, 1000},
+			},
+		},
+		{
+			name: "with empty buckets and non empty capacity",
+			orig: &Histogram{
+				PositiveSpans:   make([]Span, 0, 1),
+				PositiveBuckets: make([]int64, 0, 1),
+				NegativeSpans:   make([]Span, 0, 1),
+				NegativeBuckets: make([]int64, 0, 1),
+			},
+			expected: &Histogram{},
+		},
+	}
+
+	for _, tcase := range cases {
+		t.Run(tcase.name, func(t *testing.T) {
+			hCopy := &Histogram{}
+			tcase.orig.CopyTo(hCopy)
+
+			// Modify a primitive value in the original histogram.
+			tcase.orig.Sum++
+			require.Equal(t, tcase.expected, hCopy)
+			assertDeepCopyHSpans(t, tcase.orig, hCopy, tcase.expected)
+		})
+	}
+}
+
+func assertDeepCopyHSpans(t *testing.T, orig, hCopy, expected *Histogram) {
+	// Do an in-place expansion of an original spans slice.
+	orig.PositiveSpans = expandSpans(orig.PositiveSpans)
+	orig.PositiveSpans[len(orig.PositiveSpans)-1] = Span{1, 2}
+
+	hCopy.PositiveSpans = expandSpans(hCopy.PositiveSpans)
+	expected.PositiveSpans = expandSpans(expected.PositiveSpans)
+	// Expand the copy spans and assert that modifying the original has not affected the copy.
+	require.Equal(t, expected, hCopy)
+}
+
+func expandSpans(spans []Span) []Span {
+	n := len(spans)
+	if cap(spans) > n {
+		spans = spans[:n+1]
+	} else {
+		spans = append(spans, Span{})
+	}
+	return spans
+}
+
 func TestHistogramCompact(t *testing.T) {
 	cases := []struct {
 		name            string
diff --git a/promql/engine.go b/promql/engine.go
index 7165631e0..8c8afd181 100644
--- a/promql/engine.go
+++ b/promql/engine.go
@@ -66,6 +66,9 @@ const (
 	// The getHPointSlice and getFPointSlice functions are called with an estimated size which often can be
 	// over-estimated.
 	maxPointsSliceSize = 5000
+
+	// The default buffer size for points used by the matrix selector.
+	matrixSelectorSliceSize = 16
 )
 
 type engineMetrics struct {
@@ -1564,7 +1567,7 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, annotations.Annotatio
 
 		ev.currentSamples -= len(floats) + totalHPointSize(histograms)
 		putFPointSlice(floats)
-		putHPointSlice(histograms)
+		putMatrixSelectorHPointSlice(histograms)
 
 		// The absent_over_time function returns 0 or 1 series. So far, the matrix
 		// contains multiple series. The following code will create a new series
@@ -1940,6 +1943,13 @@ func (ev *evaluator) vectorSelectorSingle(it *storage.MemoizedSeriesIterator, no
 var (
 	fPointPool zeropool.Pool[[]FPoint]
 	hPointPool zeropool.Pool[[]HPoint]
+
+	// matrixSelectorHPool holds reusable histogram slices used by the matrix
+	// selector. The key difference between this pool and the hPointPool is that
+	// slices returned by this pool should never hold multiple copies of the same
+	// histogram pointer since histogram objects are reused across query evaluation
+	// steps.
+	matrixSelectorHPool zeropool.Pool[[]HPoint]
 )
 
 func getFPointSlice(sz int) []FPoint {
@@ -1982,6 +1992,20 @@ func putHPointSlice(p []HPoint) {
 	}
 }
 
+func getMatrixSelectorHPoints() []HPoint {
+	if p := matrixSelectorHPool.Get(); p != nil {
+		return p
+	}
+
+	return make([]HPoint, 0, matrixSelectorSliceSize)
+}
+
+func putMatrixSelectorHPointSlice(p []HPoint) {
+	if p != nil {
+		matrixSelectorHPool.Put(p[:0])
+	}
+}
+
 // matrixSelector evaluates a *parser.MatrixSelector expression.
 func (ev *evaluator) matrixSelector(node *parser.MatrixSelector) (Matrix, annotations.Annotations) {
 	var (
@@ -2106,13 +2130,13 @@ loop:
 			// Values in the buffer are guaranteed to be smaller than maxt.
 			if t >= mintHistograms {
 				if histograms == nil {
-					histograms = getHPointSlice(16)
+					histograms = getMatrixSelectorHPoints()
 				}
 				n := len(histograms)
 				if n < cap(histograms) {
 					histograms = histograms[:n+1]
 				} else {
-					histograms = append(histograms, HPoint{})
+					histograms = append(histograms, HPoint{H: &histogram.FloatHistogram{}})
 				}
 				histograms[n].T, histograms[n].H = buf.AtFloatHistogram(histograms[n].H)
 				if value.IsStaleNaN(histograms[n].H.Sum) {
@@ -2145,23 +2169,28 @@ loop:
 	// The sought sample might also be in the range.
 	switch soughtValueType {
 	case chunkenc.ValFloatHistogram, chunkenc.ValHistogram:
-		t := it.AtT()
-		if t == maxt {
-			_, h := it.AtFloatHistogram()
-			if !value.IsStaleNaN(h.Sum) {
-				if ev.currentSamples >= ev.maxSamples {
-					ev.error(ErrTooManySamples(env))
-				}
-				if histograms == nil {
-					histograms = getHPointSlice(16)
-				}
-				// The last sample comes directly from the iterator, so we need to copy it to
-				// avoid having the same reference twice in the buffer.
-				point := HPoint{T: t, H: h.Copy()}
-				histograms = append(histograms, point)
-				ev.currentSamples += point.size()
-			}
+		if it.AtT() != maxt {
+			break
 		}
+		if histograms == nil {
+			histograms = getMatrixSelectorHPoints()
+		}
+		n := len(histograms)
+		if n < cap(histograms) {
+			histograms = histograms[:n+1]
+		} else {
+			histograms = append(histograms, HPoint{H: &histogram.FloatHistogram{}})
+		}
+		histograms[n].T, histograms[n].H = it.AtFloatHistogram(histograms[n].H)
+		if value.IsStaleNaN(histograms[n].H.Sum) {
+			histograms = histograms[:n]
+			break
+		}
+		if ev.currentSamples >= ev.maxSamples {
+			ev.error(ErrTooManySamples(env))
+		}
+		ev.currentSamples += histograms[n].size()
+
 	case chunkenc.ValFloat:
 		t, f := it.At()
 		if t == maxt && !value.IsStaleNaN(f) {
diff --git a/promql/value.go b/promql/value.go
index 28cf3fe31..f129137d8 100644
--- a/promql/value.go
+++ b/promql/value.go
@@ -464,11 +464,11 @@ func (ssi *storageSeriesIterator) At() (t int64, v float64) {
 	return ssi.currT, ssi.currF
 }
 
-func (ssi *storageSeriesIterator) AtHistogram() (int64, *histogram.Histogram) {
+func (ssi *storageSeriesIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) {
 	panic(errors.New("storageSeriesIterator: AtHistogram not supported"))
 }
 
-func (ssi *storageSeriesIterator) AtFloatHistogram() (int64, *histogram.FloatHistogram) {
+func (ssi *storageSeriesIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
 	return ssi.currT, ssi.currH
 }
 
diff --git a/rules/manager_test.go b/rules/manager_test.go
index 6418c5a37..3feae51de 100644
--- a/rules/manager_test.go
+++ b/rules/manager_test.go
@@ -1397,7 +1397,7 @@ func TestNativeHistogramsInRecordingRules(t *testing.T) {
 
 	it := s.Iterator(nil)
 	require.Equal(t, chunkenc.ValFloatHistogram, it.Next())
-	tsp, fh := it.AtFloatHistogram()
+	tsp, fh := it.AtFloatHistogram(nil)
 	require.Equal(t, ts.Add(10*time.Second).UnixMilli(), tsp)
 	require.Equal(t, expHist, fh)
 	require.Equal(t, chunkenc.ValNone, it.Next())
diff --git a/storage/buffer.go b/storage/buffer.go
index d19f841d4..b3c789e97 100644
--- a/storage/buffer.go
+++ b/storage/buffer.go
@@ -24,6 +24,9 @@ import (
 
 // BufferedSeriesIterator wraps an iterator with a look-back buffer.
 type BufferedSeriesIterator struct {
+	hReader  histogram.Histogram
+	fhReader histogram.FloatHistogram
+
 	it    chunkenc.Iterator
 	buf   *sampleRing
 	delta int64
@@ -118,10 +121,10 @@ func (b *BufferedSeriesIterator) Next() chunkenc.ValueType {
 		t, f := b.it.At()
 		b.buf.addF(fSample{t: t, f: f})
 	case chunkenc.ValHistogram:
-		t, h := b.it.AtHistogram()
+		t, h := b.it.AtHistogram(&b.hReader)
 		b.buf.addH(hSample{t: t, h: h})
 	case chunkenc.ValFloatHistogram:
-		t, fh := b.it.AtFloatHistogram()
+		t, fh := b.it.AtFloatHistogram(&b.fhReader)
 		b.buf.addFH(fhSample{t: t, fh: fh})
 	default:
 		panic(fmt.Errorf("BufferedSeriesIterator: unknown value type %v", b.valueType))
@@ -140,13 +143,13 @@ func (b *BufferedSeriesIterator) At() (int64, float64) {
 }
 
 // AtHistogram returns the current histogram element of the iterator.
-func (b *BufferedSeriesIterator) AtHistogram() (int64, *histogram.Histogram) {
-	return b.it.AtHistogram()
+func (b *BufferedSeriesIterator) AtHistogram(fh *histogram.Histogram) (int64, *histogram.Histogram) {
+	return b.it.AtHistogram(fh)
 }
 
 // AtFloatHistogram returns the current float-histogram element of the iterator.
-func (b *BufferedSeriesIterator) AtFloatHistogram() (int64, *histogram.FloatHistogram) {
-	return b.it.AtFloatHistogram()
+func (b *BufferedSeriesIterator) AtFloatHistogram(fh *histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
+	return b.it.AtFloatHistogram(fh)
 }
 
 // AtT returns the current timestamp of the iterator.
@@ -378,7 +381,11 @@ func (it *SampleRingIterator) AtFloatHistogram(fh *histogram.FloatHistogram) (in
 	if it.fh == nil {
 		return it.t, it.h.ToFloat(fh)
 	}
-	return it.t, it.fh
+	if fh != nil {
+		it.fh.CopyTo(fh)
+		return it.t, fh
+	}
+	return it.t, it.fh.Copy()
 }
 
 func (it *SampleRingIterator) AtT() int64 {
@@ -672,7 +679,12 @@ func addH(s hSample, buf []hSample, r *sampleRing) []hSample {
 		}
 	}
 
-	buf[r.i] = s
+	buf[r.i].t = s.t
+	if buf[r.i].h == nil {
+		buf[r.i].h = s.h.Copy()
+	} else {
+		s.h.CopyTo(buf[r.i].h)
+	}
 	r.l++
 
 	// Free head of the buffer of samples that just fell out of the range.
@@ -711,7 +723,12 @@ func addFH(s fhSample, buf []fhSample, r *sampleRing) []fhSample {
 		}
 	}
 
-	buf[r.i] = s
+	buf[r.i].t = s.t
+	if buf[r.i].fh == nil {
+		buf[r.i].fh = s.fh.Copy()
+	} else {
+		s.fh.CopyTo(buf[r.i].fh)
+	}
 	r.l++
 
 	// Free head of the buffer of samples that just fell out of the range.
diff --git a/storage/buffer_test.go b/storage/buffer_test.go
index 12e6ff0f0..61074c212 100644
--- a/storage/buffer_test.go
+++ b/storage/buffer_test.go
@@ -277,11 +277,11 @@ func (m *mockSeriesIterator) At() (int64, float64)            { return m.at() }
 func (m *mockSeriesIterator) Next() chunkenc.ValueType        { return m.next() }
 func (m *mockSeriesIterator) Err() error                      { return m.err() }
 
-func (m *mockSeriesIterator) AtHistogram() (int64, *histogram.Histogram) {
+func (m *mockSeriesIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) {
 	return 0, nil // Not really mocked.
 }
 
-func (m *mockSeriesIterator) AtFloatHistogram() (int64, *histogram.FloatHistogram) {
+func (m *mockSeriesIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
 	return 0, nil // Not really mocked.
 }
 
@@ -303,11 +303,11 @@ func (it *fakeSeriesIterator) At() (int64, float64) {
 	return it.idx * it.step, 123 // Value doesn't matter.
 }
 
-func (it *fakeSeriesIterator) AtHistogram() (int64, *histogram.Histogram) {
+func (it *fakeSeriesIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) {
 	return it.idx * it.step, &histogram.Histogram{} // Value doesn't matter.
 }
 
-func (it *fakeSeriesIterator) AtFloatHistogram() (int64, *histogram.FloatHistogram) {
+func (it *fakeSeriesIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
 	return it.idx * it.step, &histogram.FloatHistogram{} // Value doesn't matter.
 }
 
diff --git a/storage/memoized_iterator.go b/storage/memoized_iterator.go
index cb9fdeef4..4ab2aa5d7 100644
--- a/storage/memoized_iterator.go
+++ b/storage/memoized_iterator.go
@@ -113,7 +113,7 @@ func (b *MemoizedSeriesIterator) Next() chunkenc.ValueType {
 		b.prevFloatHistogram = nil
 	case chunkenc.ValHistogram, chunkenc.ValFloatHistogram:
 		b.prevValue = 0
-		b.prevTime, b.prevFloatHistogram = b.it.AtFloatHistogram()
+		b.prevTime, b.prevFloatHistogram = b.it.AtFloatHistogram(nil)
 	}
 
 	b.valueType = b.it.Next()
@@ -133,7 +133,7 @@ func (b *MemoizedSeriesIterator) At() (int64, float64) {
 
 // AtFloatHistogram returns the current float-histogram element of the iterator.
 func (b *MemoizedSeriesIterator) AtFloatHistogram() (int64, *histogram.FloatHistogram) {
-	return b.it.AtFloatHistogram()
+	return b.it.AtFloatHistogram(nil)
 }
 
 // Err returns the last encountered error.
diff --git a/storage/merge.go b/storage/merge.go
index bcb0f66fb..38897449b 100644
--- a/storage/merge.go
+++ b/storage/merge.go
@@ -525,11 +525,11 @@ func (c *chainSampleIterator) At() (t int64, v float64) {
 	return c.curr.At()
 }
 
-func (c *chainSampleIterator) AtHistogram() (int64, *histogram.Histogram) {
+func (c *chainSampleIterator) AtHistogram(h *histogram.Histogram) (int64, *histogram.Histogram) {
 	if c.curr == nil {
 		panic("chainSampleIterator.AtHistogram called before first .Next or after .Next returned false.")
 	}
-	t, h := c.curr.AtHistogram()
+	t, h := c.curr.AtHistogram(h)
 	// If the current sample is not consecutive with the previous one, we
 	// cannot be sure anymore about counter resets for counter histograms.
 	// TODO(beorn7): If a `NotCounterReset` sample is followed by a
@@ -542,11 +542,11 @@ func (c *chainSampleIterator) AtHistogram() (int64, *histogram.Histogram) {
 	return t, h
 }
 
-func (c *chainSampleIterator) AtFloatHistogram() (int64, *histogram.FloatHistogram) {
+func (c *chainSampleIterator) AtFloatHistogram(fh *histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
 	if c.curr == nil {
 		panic("chainSampleIterator.AtFloatHistogram called before first .Next or after .Next returned false.")
 	}
-	t, fh := c.curr.AtFloatHistogram()
+	t, fh := c.curr.AtFloatHistogram(fh)
 	// If the current sample is not consecutive with the previous one, we
 	// cannot be sure anymore about counter resets for counter histograms.
 	// TODO(beorn7): If a `NotCounterReset` sample is followed by a
diff --git a/storage/merge_test.go b/storage/merge_test.go
index 02c2a3409..05e1c7527 100644
--- a/storage/merge_test.go
+++ b/storage/merge_test.go
@@ -1173,10 +1173,10 @@ func TestChainSampleIteratorSeek(t *testing.T) {
 					t, f := merged.At()
 					actual = append(actual, fSample{t, f})
 				case chunkenc.ValHistogram:
-					t, h := merged.AtHistogram()
+					t, h := merged.AtHistogram(nil)
 					actual = append(actual, hSample{t, h})
 				case chunkenc.ValFloatHistogram:
-					t, fh := merged.AtFloatHistogram()
+					t, fh := merged.AtFloatHistogram(nil)
 					actual = append(actual, fhSample{t, fh})
 				}
 				s, err := ExpandSamples(merged, nil)
@@ -1259,10 +1259,10 @@ func TestChainSampleIteratorSeekHistogramCounterResetHint(t *testing.T) {
 					t, f := merged.At()
 					actual = append(actual, fSample{t, f})
 				case chunkenc.ValHistogram:
-					t, h := merged.AtHistogram()
+					t, h := merged.AtHistogram(nil)
 					actual = append(actual, hSample{t, h})
 				case chunkenc.ValFloatHistogram:
-					t, fh := merged.AtFloatHistogram()
+					t, fh := merged.AtFloatHistogram(nil)
 					actual = append(actual, fhSample{t, fh})
 				}
 				s, err := ExpandSamples(merged, nil)
@@ -1629,11 +1629,11 @@ func (e errIterator) At() (int64, float64) {
 	return 0, 0
 }
 
-func (e errIterator) AtHistogram() (int64, *histogram.Histogram) {
+func (e errIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) {
 	return 0, nil
 }
 
-func (e errIterator) AtFloatHistogram() (int64, *histogram.FloatHistogram) {
+func (e errIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
 	return 0, nil
 }
 
diff --git a/storage/remote/codec.go b/storage/remote/codec.go
index ffab821a5..9cf1ed871 100644
--- a/storage/remote/codec.go
+++ b/storage/remote/codec.go
@@ -152,10 +152,10 @@ func ToQueryResult(ss storage.SeriesSet, sampleLimit int) (*prompb.QueryResult,
 					Value:     val,
 				})
 			case chunkenc.ValHistogram:
-				ts, h := iter.AtHistogram()
+				ts, h := iter.AtHistogram(nil)
 				histograms = append(histograms, HistogramToHistogramProto(ts, h))
 			case chunkenc.ValFloatHistogram:
-				ts, fh := iter.AtFloatHistogram()
+				ts, fh := iter.AtFloatHistogram(nil)
 				histograms = append(histograms, FloatHistogramToHistogramProto(ts, fh))
 			default:
 				return nil, ss.Warnings(), fmt.Errorf("unrecognized value type: %s", valType)
@@ -475,7 +475,7 @@ func (c *concreteSeriesIterator) At() (t int64, v float64) {
 }
 
 // AtHistogram implements chunkenc.Iterator.
-func (c *concreteSeriesIterator) AtHistogram() (int64, *histogram.Histogram) {
+func (c *concreteSeriesIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) {
 	if c.curValType != chunkenc.ValHistogram {
 		panic("iterator is not on an integer histogram sample")
 	}
@@ -484,7 +484,7 @@ func (c *concreteSeriesIterator) AtHistogram() (int64, *histogram.Histogram) {
 }
 
 // AtFloatHistogram implements chunkenc.Iterator.
-func (c *concreteSeriesIterator) AtFloatHistogram() (int64, *histogram.FloatHistogram) {
+func (c *concreteSeriesIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
 	switch c.curValType {
 	case chunkenc.ValHistogram:
 		fh := c.series.histograms[c.histogramsCur]
diff --git a/storage/remote/codec_test.go b/storage/remote/codec_test.go
index dbc22a377..0451953cb 100644
--- a/storage/remote/codec_test.go
+++ b/storage/remote/codec_test.go
@@ -278,31 +278,31 @@ func TestConcreteSeriesIterator_HistogramSamples(t *testing.T) {
 
 	// Seek to the first sample with ts=1.
 	require.Equal(t, chunkenc.ValHistogram, it.Seek(1))
-	ts, v := it.AtHistogram()
+	ts, v := it.AtHistogram(nil)
 	require.Equal(t, int64(1), ts)
 	require.Equal(t, histograms[0], v)
 
 	// Seek one further, next sample still has ts=1.
 	require.Equal(t, chunkenc.ValHistogram, it.Next())
-	ts, v = it.AtHistogram()
+	ts, v = it.AtHistogram(nil)
 	require.Equal(t, int64(1), ts)
 	require.Equal(t, histograms[1], v)
 
 	// Seek again to 1 and make sure we stay where we are.
 	require.Equal(t, chunkenc.ValHistogram, it.Seek(1))
-	ts, v = it.AtHistogram()
+	ts, v = it.AtHistogram(nil)
 	require.Equal(t, int64(1), ts)
 	require.Equal(t, histograms[1], v)
 
 	// Another seek.
 	require.Equal(t, chunkenc.ValHistogram, it.Seek(3))
-	ts, v = it.AtHistogram()
+	ts, v = it.AtHistogram(nil)
 	require.Equal(t, int64(3), ts)
 	require.Equal(t, histograms[3], v)
 
 	// And we don't go back.
 	require.Equal(t, chunkenc.ValHistogram, it.Seek(2))
-	ts, v = it.AtHistogram()
+	ts, v = it.AtHistogram(nil)
 	require.Equal(t, int64(3), ts)
 	require.Equal(t, histograms[3], v)
 
@@ -347,12 +347,12 @@ func TestConcreteSeriesIterator_FloatAndHistogramSamples(t *testing.T) {
 		fh *histogram.FloatHistogram
 	)
 	require.Equal(t, chunkenc.ValHistogram, it.Next())
-	ts, h = it.AtHistogram()
+	ts, h = it.AtHistogram(nil)
 	require.Equal(t, int64(1), ts)
 	require.Equal(t, histograms[0], h)
 
 	require.Equal(t, chunkenc.ValHistogram, it.Next())
-	ts, h = it.AtHistogram()
+	ts, h = it.AtHistogram(nil)
 	require.Equal(t, int64(2), ts)
 	require.Equal(t, histograms[1], h)
 
@@ -393,13 +393,13 @@ func TestConcreteSeriesIterator_FloatAndHistogramSamples(t *testing.T) {
 	require.Equal(t, 8., v)
 
 	require.Equal(t, chunkenc.ValHistogram, it.Next())
-	ts, h = it.AtHistogram()
+	ts, h = it.AtHistogram(nil)
 	require.Equal(t, int64(16), ts)
 	require.Equal(t, histograms[10], h)
 
 	// Getting a float histogram from an int histogram works.
 	require.Equal(t, chunkenc.ValHistogram, it.Next())
-	ts, fh = it.AtFloatHistogram()
+	ts, fh = it.AtFloatHistogram(nil)
 	require.Equal(t, int64(17), ts)
 	expected := HistogramProtoToFloatHistogram(HistogramToHistogramProto(int64(17), histograms[11]))
 	require.Equal(t, expected, fh)
diff --git a/storage/series.go b/storage/series.go
index b111505aa..eba11b4d9 100644
--- a/storage/series.go
+++ b/storage/series.go
@@ -123,12 +123,12 @@ func (it *listSeriesIterator) At() (int64, float64) {
 	return s.T(), s.F()
 }
 
-func (it *listSeriesIterator) AtHistogram() (int64, *histogram.Histogram) {
+func (it *listSeriesIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) {
 	s := it.samples.Get(it.idx)
 	return s.T(), s.H()
 }
 
-func (it *listSeriesIterator) AtFloatHistogram() (int64, *histogram.FloatHistogram) {
+func (it *listSeriesIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
 	s := it.samples.Get(it.idx)
 	return s.T(), s.FH()
 }
@@ -337,7 +337,7 @@ func (s *seriesToChunkEncoder) Iterator(it chunks.Iterator) chunks.Iterator {
 			t, v = seriesIter.At()
 			app.Append(t, v)
 		case chunkenc.ValHistogram:
-			t, h = seriesIter.AtHistogram()
+			t, h = seriesIter.AtHistogram(nil)
 			newChk, recoded, app, err = app.AppendHistogram(nil, t, h, false)
 			if err != nil {
 				return errChunksIterator{err: err}
@@ -352,7 +352,7 @@ func (s *seriesToChunkEncoder) Iterator(it chunks.Iterator) chunks.Iterator {
 				chk = newChk
 			}
 		case chunkenc.ValFloatHistogram:
-			t, fh = seriesIter.AtFloatHistogram()
+			t, fh = seriesIter.AtFloatHistogram(nil)
 			newChk, recoded, app, err = app.AppendFloatHistogram(nil, t, fh, false)
 			if err != nil {
 				return errChunksIterator{err: err}
@@ -438,10 +438,10 @@ func ExpandSamples(iter chunkenc.Iterator, newSampleFn func(t int64, f float64,
 			}
 			result = append(result, newSampleFn(t, f, nil, nil))
 		case chunkenc.ValHistogram:
-			t, h := iter.AtHistogram()
+			t, h := iter.AtHistogram(nil)
 			result = append(result, newSampleFn(t, 0, h, nil))
 		case chunkenc.ValFloatHistogram:
-			t, fh := iter.AtFloatHistogram()
+			t, fh := iter.AtFloatHistogram(nil)
 			result = append(result, newSampleFn(t, 0, nil, fh))
 		}
 	}
diff --git a/tsdb/block_test.go b/tsdb/block_test.go
index c155b4451..d48418b57 100644
--- a/tsdb/block_test.go
+++ b/tsdb/block_test.go
@@ -555,10 +555,10 @@ func createHead(tb testing.TB, w *wlog.WL, series []storage.Series, chunkDir str
 				t, v := it.At()
 				ref, err = app.Append(ref, lset, t, v)
 			case chunkenc.ValHistogram:
-				t, h := it.AtHistogram()
+				t, h := it.AtHistogram(nil)
 				ref, err = app.AppendHistogram(ref, lset, t, h, nil)
 			case chunkenc.ValFloatHistogram:
-				t, fh := it.AtFloatHistogram()
+				t, fh := it.AtFloatHistogram(nil)
 				ref, err = app.AppendHistogram(ref, lset, t, nil, fh)
 			default:
 				err = fmt.Errorf("unknown sample type %s", typ.String())
diff --git a/tsdb/chunkenc/chunk.go b/tsdb/chunkenc/chunk.go
index 0126f1fbd..21c41257b 100644
--- a/tsdb/chunkenc/chunk.go
+++ b/tsdb/chunkenc/chunk.go
@@ -131,16 +131,20 @@ type Iterator interface {
 	// At returns the current timestamp/value pair if the value is a float.
 	// Before the iterator has advanced, the behaviour is unspecified.
 	At() (int64, float64)
-	// AtHistogram returns the current timestamp/value pair if the value is
-	// a histogram with integer counts. Before the iterator has advanced,
-	// the behaviour is unspecified.
-	AtHistogram() (int64, *histogram.Histogram)
+	// AtHistogram returns the current timestamp/value pair if the value is a
+	// histogram with integer counts. Before the iterator has advanced, the behaviour
+	// is unspecified.
+	// The method accepts an optional Histogram object which will be
+	// reused when not nil. Otherwise, a new Histogram object will be allocated.
+	AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram)
 	// AtFloatHistogram returns the current timestamp/value pair if the
 	// value is a histogram with floating-point counts. It also works if the
 	// value is a histogram with integer counts, in which case a
 	// FloatHistogram copy of the histogram is returned. Before the iterator
 	// has advanced, the behaviour is unspecified.
-	AtFloatHistogram() (int64, *histogram.FloatHistogram)
+	// The method accepts an optional FloatHistogram object which will be
+	// reused when not nil. Otherwise, a new FloatHistogram object will be allocated.
+	AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram)
 	// AtT returns the current timestamp.
 	// Before the iterator has advanced, the behaviour is unspecified.
 	AtT() int64
@@ -222,9 +226,11 @@ func (it *mockSeriesIterator) At() (int64, float64) {
 	return it.timeStamps[it.currIndex], it.values[it.currIndex]
 }
 
-func (it *mockSeriesIterator) AtHistogram() (int64, *histogram.Histogram) { return math.MinInt64, nil }
+func (it *mockSeriesIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) {
+	return math.MinInt64, nil
+}
 
-func (it *mockSeriesIterator) AtFloatHistogram() (int64, *histogram.FloatHistogram) {
+func (it *mockSeriesIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
 	return math.MinInt64, nil
 }
 
@@ -249,13 +255,18 @@ func NewNopIterator() Iterator {
 
 type nopIterator struct{}
 
-func (nopIterator) Next() ValueType                                      { return ValNone }
-func (nopIterator) Seek(int64) ValueType                                 { return ValNone }
-func (nopIterator) At() (int64, float64)                                 { return math.MinInt64, 0 }
-func (nopIterator) AtHistogram() (int64, *histogram.Histogram)           { return math.MinInt64, nil }
-func (nopIterator) AtFloatHistogram() (int64, *histogram.FloatHistogram) { return math.MinInt64, nil }
-func (nopIterator) AtT() int64                                           { return math.MinInt64 }
-func (nopIterator) Err() error                                           { return nil }
+func (nopIterator) Next() ValueType      { return ValNone }
+func (nopIterator) Seek(int64) ValueType { return ValNone }
+func (nopIterator) At() (int64, float64) { return math.MinInt64, 0 }
+func (nopIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) {
+	return math.MinInt64, nil
+}
+
+func (nopIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
+	return math.MinInt64, nil
+}
+func (nopIterator) AtT() int64 { return math.MinInt64 }
+func (nopIterator) Err() error { return nil }
 
 // Pool is used to create and reuse chunk references to avoid allocations.
 type Pool interface {
diff --git a/tsdb/chunkenc/float_histogram.go b/tsdb/chunkenc/float_histogram.go
index 3d76cdf65..88d189254 100644
--- a/tsdb/chunkenc/float_histogram.go
+++ b/tsdb/chunkenc/float_histogram.go
@@ -527,7 +527,7 @@ func (a *FloatHistogramAppender) recode(
 	numPositiveBuckets, numNegativeBuckets := countSpans(positiveSpans), countSpans(negativeSpans)
 
 	for it.Next() == ValFloatHistogram {
-		tOld, hOld := it.AtFloatHistogram()
+		tOld, hOld := it.AtFloatHistogram(nil)
 
 		// We have to newly allocate slices for the modified buckets
 		// here because they are kept by the appender until the next
@@ -728,27 +728,50 @@ func (it *floatHistogramIterator) At() (int64, float64) {
 	panic("cannot call floatHistogramIterator.At")
 }
 
-func (it *floatHistogramIterator) AtHistogram() (int64, *histogram.Histogram) {
+func (it *floatHistogramIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) {
 	panic("cannot call floatHistogramIterator.AtHistogram")
 }
 
-func (it *floatHistogramIterator) AtFloatHistogram() (int64, *histogram.FloatHistogram) {
+func (it *floatHistogramIterator) AtFloatHistogram(fh *histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
 	if value.IsStaleNaN(it.sum.value) {
 		return it.t, &histogram.FloatHistogram{Sum: it.sum.value}
 	}
-	it.atFloatHistogramCalled = true
-	return it.t, &histogram.FloatHistogram{
-		CounterResetHint: counterResetHint(it.counterResetHeader, it.numRead),
-		Count:            it.cnt.value,
-		ZeroCount:        it.zCnt.value,
-		Sum:              it.sum.value,
-		ZeroThreshold:    it.zThreshold,
-		Schema:           it.schema,
-		PositiveSpans:    it.pSpans,
-		NegativeSpans:    it.nSpans,
-		PositiveBuckets:  it.pBuckets,
-		NegativeBuckets:  it.nBuckets,
+	if fh == nil {
+		it.atFloatHistogramCalled = true
+		return it.t, &histogram.FloatHistogram{
+			CounterResetHint: counterResetHint(it.counterResetHeader, it.numRead),
+			Count:            it.cnt.value,
+			ZeroCount:        it.zCnt.value,
+			Sum:              it.sum.value,
+			ZeroThreshold:    it.zThreshold,
+			Schema:           it.schema,
+			PositiveSpans:    it.pSpans,
+			NegativeSpans:    it.nSpans,
+			PositiveBuckets:  it.pBuckets,
+			NegativeBuckets:  it.nBuckets,
+		}
 	}
+
+	fh.CounterResetHint = counterResetHint(it.counterResetHeader, it.numRead)
+	fh.Schema = it.schema
+	fh.ZeroThreshold = it.zThreshold
+	fh.ZeroCount = it.zCnt.value
+	fh.Count = it.cnt.value
+	fh.Sum = it.sum.value
+
+	fh.PositiveSpans = resize(fh.PositiveSpans, len(it.pSpans))
+	copy(fh.PositiveSpans, it.pSpans)
+
+	fh.NegativeSpans = resize(fh.NegativeSpans, len(it.nSpans))
+	copy(fh.NegativeSpans, it.nSpans)
+
+	fh.PositiveBuckets = resize(fh.PositiveBuckets, len(it.pBuckets))
+	copy(fh.PositiveBuckets, it.pBuckets)
+
+	fh.NegativeBuckets = resize(fh.NegativeBuckets, len(it.nBuckets))
+	copy(fh.NegativeBuckets, it.nBuckets)
+
+	return it.t, fh
 }
 
 func (it *floatHistogramIterator) AtT() int64 {
diff --git a/tsdb/chunkenc/float_histogram_test.go b/tsdb/chunkenc/float_histogram_test.go
index 6f5a95fb1..054c17f7d 100644
--- a/tsdb/chunkenc/float_histogram_test.go
+++ b/tsdb/chunkenc/float_histogram_test.go
@@ -140,7 +140,7 @@ func TestFloatHistogramChunkSameBuckets(t *testing.T) {
 	require.NoError(t, it.Err())
 	var act []floatResult
 	for it.Next() == ValFloatHistogram {
-		fts, fh := it.AtFloatHistogram()
+		fts, fh := it.AtFloatHistogram(nil)
 		act = append(act, floatResult{t: fts, h: fh})
 	}
 	require.NoError(t, it.Err())
@@ -150,7 +150,7 @@ func TestFloatHistogramChunkSameBuckets(t *testing.T) {
 	it2 := c.Iterator(it)
 	var act2 []floatResult
 	for it2.Next() == ValFloatHistogram {
-		fts, fh := it2.AtFloatHistogram()
+		fts, fh := it2.AtFloatHistogram(nil)
 		act2 = append(act2, floatResult{t: fts, h: fh})
 	}
 	require.NoError(t, it2.Err())
@@ -164,7 +164,7 @@ func TestFloatHistogramChunkSameBuckets(t *testing.T) {
 	it3 := c.iterator(itX)
 	var act3 []floatResult
 	for it3.Next() == ValFloatHistogram {
-		fts, fh := it3.AtFloatHistogram()
+		fts, fh := it3.AtFloatHistogram(nil)
 		act3 = append(act3, floatResult{t: fts, h: fh})
 	}
 	require.NoError(t, it3.Err())
@@ -178,10 +178,10 @@ func TestFloatHistogramChunkSameBuckets(t *testing.T) {
 	// Below ones should not matter.
 	require.Equal(t, ValFloatHistogram, it4.Seek(exp[mid].t))
 	require.Equal(t, ValFloatHistogram, it4.Seek(exp[mid].t))
-	fts, fh := it4.AtFloatHistogram()
+	fts, fh := it4.AtFloatHistogram(nil)
 	act4 = append(act4, floatResult{t: fts, h: fh})
 	for it4.Next() == ValFloatHistogram {
-		fts, fh := it4.AtFloatHistogram()
+		fts, fh := it4.AtFloatHistogram(nil)
 		act4 = append(act4, floatResult{t: fts, h: fh})
 	}
 	require.NoError(t, it4.Err())
@@ -272,7 +272,7 @@ func TestFloatHistogramChunkBucketChanges(t *testing.T) {
 	it := c.Iterator(nil)
 	var act []floatResult
 	for it.Next() == ValFloatHistogram {
-		fts, fh := it.AtFloatHistogram()
+		fts, fh := it.AtFloatHistogram(nil)
 		act = append(act, floatResult{t: fts, h: fh})
 	}
 	require.NoError(t, it.Err())
diff --git a/tsdb/chunkenc/histogram.go b/tsdb/chunkenc/histogram.go
index ac84e7a1e..cb09eda26 100644
--- a/tsdb/chunkenc/histogram.go
+++ b/tsdb/chunkenc/histogram.go
@@ -558,7 +558,7 @@ func (a *HistogramAppender) recode(
 	numPositiveBuckets, numNegativeBuckets := countSpans(positiveSpans), countSpans(negativeSpans)
 
 	for it.Next() == ValHistogram {
-		tOld, hOld := it.AtHistogram()
+		tOld, hOld := it.AtHistogram(nil)
 
 		// We have to newly allocate slices for the modified buckets
 		// here because they are kept by the appender until the next
@@ -776,42 +776,96 @@ func (it *histogramIterator) At() (int64, float64) {
 	panic("cannot call histogramIterator.At")
 }
 
-func (it *histogramIterator) AtHistogram() (int64, *histogram.Histogram) {
+func (it *histogramIterator) AtHistogram(h *histogram.Histogram) (int64, *histogram.Histogram) {
 	if value.IsStaleNaN(it.sum) {
 		return it.t, &histogram.Histogram{Sum: it.sum}
 	}
-	it.atHistogramCalled = true
-	return it.t, &histogram.Histogram{
-		CounterResetHint: counterResetHint(it.counterResetHeader, it.numRead),
-		Count:            it.cnt,
-		ZeroCount:        it.zCnt,
-		Sum:              it.sum,
-		ZeroThreshold:    it.zThreshold,
-		Schema:           it.schema,
-		PositiveSpans:    it.pSpans,
-		NegativeSpans:    it.nSpans,
-		PositiveBuckets:  it.pBuckets,
-		NegativeBuckets:  it.nBuckets,
+	if h == nil {
+		it.atHistogramCalled = true
+		return it.t, &histogram.Histogram{
+			CounterResetHint: counterResetHint(it.counterResetHeader, it.numRead),
+			Count:            it.cnt,
+			ZeroCount:        it.zCnt,
+			Sum:              it.sum,
+			ZeroThreshold:    it.zThreshold,
+			Schema:           it.schema,
+			PositiveSpans:    it.pSpans,
+			NegativeSpans:    it.nSpans,
+			PositiveBuckets:  it.pBuckets,
+			NegativeBuckets:  it.nBuckets,
+		}
 	}
+
+	h.CounterResetHint = counterResetHint(it.counterResetHeader, it.numRead)
+	h.Schema = it.schema
+	h.ZeroThreshold = it.zThreshold
+	h.ZeroCount = it.zCnt
+	h.Count = it.cnt
+	h.Sum = it.sum
+
+	h.PositiveSpans = resize(h.PositiveSpans, len(it.pSpans))
+	copy(h.PositiveSpans, it.pSpans)
+
+	h.NegativeSpans = resize(h.NegativeSpans, len(it.nSpans))
+	copy(h.NegativeSpans, it.nSpans)
+
+	h.PositiveBuckets = resize(h.PositiveBuckets, len(it.pBuckets))
+	copy(h.PositiveBuckets, it.pBuckets)
+
+	h.NegativeBuckets = resize(h.NegativeBuckets, len(it.nBuckets))
+	copy(h.NegativeBuckets, it.nBuckets)
+
+	return it.t, h
 }
 
-func (it *histogramIterator) AtFloatHistogram() (int64, *histogram.FloatHistogram) {
+func (it *histogramIterator) AtFloatHistogram(fh *histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
 	if value.IsStaleNaN(it.sum) {
 		return it.t, &histogram.FloatHistogram{Sum: it.sum}
 	}
-	it.atFloatHistogramCalled = true
-	return it.t, &histogram.FloatHistogram{
-		CounterResetHint: counterResetHint(it.counterResetHeader, it.numRead),
-		Count:            float64(it.cnt),
-		ZeroCount:        float64(it.zCnt),
-		Sum:              it.sum,
-		ZeroThreshold:    it.zThreshold,
-		Schema:           it.schema,
-		PositiveSpans:    it.pSpans,
-		NegativeSpans:    it.nSpans,
-		PositiveBuckets:  it.pFloatBuckets,
-		NegativeBuckets:  it.nFloatBuckets,
+	if fh == nil {
+		it.atFloatHistogramCalled = true
+		return it.t, &histogram.FloatHistogram{
+			CounterResetHint: counterResetHint(it.counterResetHeader, it.numRead),
+			Count:            float64(it.cnt),
+			ZeroCount:        float64(it.zCnt),
+			Sum:              it.sum,
+			ZeroThreshold:    it.zThreshold,
+			Schema:           it.schema,
+			PositiveSpans:    it.pSpans,
+			NegativeSpans:    it.nSpans,
+			PositiveBuckets:  it.pFloatBuckets,
+			NegativeBuckets:  it.nFloatBuckets,
+		}
 	}
+
+	fh.CounterResetHint = counterResetHint(it.counterResetHeader, it.numRead)
+	fh.Schema = it.schema
+	fh.ZeroThreshold = it.zThreshold
+	fh.ZeroCount = float64(it.zCnt)
+	fh.Count = float64(it.cnt)
+	fh.Sum = it.sum
+
+	fh.PositiveSpans = resize(fh.PositiveSpans, len(it.pSpans))
+	copy(fh.PositiveSpans, it.pSpans)
+
+	fh.NegativeSpans = resize(fh.NegativeSpans, len(it.nSpans))
+	copy(fh.NegativeSpans, it.nSpans)
+
+	fh.PositiveBuckets = resize(fh.PositiveBuckets, len(it.pBuckets))
+	var currentPositive float64
+	for i, b := range it.pBuckets {
+		currentPositive += float64(b)
+		fh.PositiveBuckets[i] = currentPositive
+	}
+
+	fh.NegativeBuckets = resize(fh.NegativeBuckets, len(it.nBuckets))
+	var currentNegative float64
+	for i, b := range it.nBuckets {
+		currentNegative += float64(b)
+		fh.NegativeBuckets[i] = currentNegative
+	}
+
+	return it.t, fh
 }
 
 func (it *histogramIterator) AtT() int64 {
@@ -1056,3 +1110,10 @@ func (it *histogramIterator) readSum() bool {
 	}
 	return true
 }
+
+func resize[T any](items []T, n int) []T {
+	if cap(items) < n {
+		return make([]T, n)
+	}
+	return items[:n]
+}
diff --git a/tsdb/chunkenc/histogram_test.go b/tsdb/chunkenc/histogram_test.go
index 53aee89db..f7609c193 100644
--- a/tsdb/chunkenc/histogram_test.go
+++ b/tsdb/chunkenc/histogram_test.go
@@ -141,8 +141,8 @@ func TestHistogramChunkSameBuckets(t *testing.T) {
 	require.NoError(t, it.Err())
 	var act []result
 	for it.Next() == ValHistogram {
-		ts, h := it.AtHistogram()
-		fts, fh := it.AtFloatHistogram()
+		ts, h := it.AtHistogram(nil)
+		fts, fh := it.AtFloatHistogram(nil)
 		require.Equal(t, ts, fts)
 		act = append(act, result{t: ts, h: h, fh: fh})
 	}
@@ -153,8 +153,8 @@ func TestHistogramChunkSameBuckets(t *testing.T) {
 	it2 := c.Iterator(it)
 	var act2 []result
 	for it2.Next() == ValHistogram {
-		ts, h := it2.AtHistogram()
-		fts, fh := it2.AtFloatHistogram()
+		ts, h := it2.AtHistogram(nil)
+		fts, fh := it2.AtFloatHistogram(nil)
 		require.Equal(t, ts, fts)
 		act2 = append(act2, result{t: ts, h: h, fh: fh})
 	}
@@ -169,8 +169,8 @@ func TestHistogramChunkSameBuckets(t *testing.T) {
 	it3 := c.iterator(itX)
 	var act3 []result
 	for it3.Next() == ValHistogram {
-		ts, h := it3.AtHistogram()
-		fts, fh := it3.AtFloatHistogram()
+		ts, h := it3.AtHistogram(nil)
+		fts, fh := it3.AtFloatHistogram(nil)
 		require.Equal(t, ts, fts)
 		act3 = append(act3, result{t: ts, h: h, fh: fh})
 	}
@@ -185,13 +185,13 @@ func TestHistogramChunkSameBuckets(t *testing.T) {
 	// Below ones should not matter.
 	require.Equal(t, ValHistogram, it4.Seek(exp[mid].t))
 	require.Equal(t, ValHistogram, it4.Seek(exp[mid].t))
-	ts, h = it4.AtHistogram()
-	fts, fh := it4.AtFloatHistogram()
+	ts, h = it4.AtHistogram(nil)
+	fts, fh := it4.AtFloatHistogram(nil)
 	require.Equal(t, ts, fts)
 	act4 = append(act4, result{t: ts, h: h, fh: fh})
 	for it4.Next() == ValHistogram {
-		ts, h := it4.AtHistogram()
-		fts, fh := it4.AtFloatHistogram()
+		ts, h := it4.AtHistogram(nil)
+		fts, fh := it4.AtFloatHistogram(nil)
 		require.Equal(t, ts, fts)
 		act4 = append(act4, result{t: ts, h: h, fh: fh})
 	}
@@ -284,8 +284,8 @@ func TestHistogramChunkBucketChanges(t *testing.T) {
 	it := c.Iterator(nil)
 	var act []result
 	for it.Next() == ValHistogram {
-		ts, h := it.AtHistogram()
-		fts, fh := it.AtFloatHistogram()
+		ts, h := it.AtHistogram(nil)
+		fts, fh := it.AtFloatHistogram(nil)
 		require.Equal(t, ts, fts)
 		act = append(act, result{t: ts, h: h, fh: fh})
 	}
@@ -897,7 +897,7 @@ func TestAtFloatHistogram(t *testing.T) {
 	it := chk.Iterator(nil)
 	i := int64(0)
 	for it.Next() != ValNone {
-		ts, h := it.AtFloatHistogram()
+		ts, h := it.AtFloatHistogram(nil)
 		require.Equal(t, i, ts)
 		require.Equal(t, expOutput[i], h, "histogram %d unequal", i)
 		i++
diff --git a/tsdb/chunkenc/xor.go b/tsdb/chunkenc/xor.go
index d54e5dbab..07b923831 100644
--- a/tsdb/chunkenc/xor.go
+++ b/tsdb/chunkenc/xor.go
@@ -260,11 +260,11 @@ func (it *xorIterator) At() (int64, float64) {
 	return it.t, it.val
 }
 
-func (it *xorIterator) AtHistogram() (int64, *histogram.Histogram) {
+func (it *xorIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) {
 	panic("cannot call xorIterator.AtHistogram")
 }
 
-func (it *xorIterator) AtFloatHistogram() (int64, *histogram.FloatHistogram) {
+func (it *xorIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
 	panic("cannot call xorIterator.AtFloatHistogram")
 }
 
diff --git a/tsdb/chunks/chunks.go b/tsdb/chunks/chunks.go
index 2c6db3637..543b98c28 100644
--- a/tsdb/chunks/chunks.go
+++ b/tsdb/chunks/chunks.go
@@ -226,10 +226,10 @@ func ChunkMetasToSamples(chunks []Meta) (result []Sample) {
 				t, v := it.At()
 				result = append(result, sample{t: t, f: v})
 			case chunkenc.ValHistogram:
-				t, h := it.AtHistogram()
+				t, h := it.AtHistogram(nil)
 				result = append(result, sample{t: t, h: h})
 			case chunkenc.ValFloatHistogram:
-				t, fh := it.AtFloatHistogram()
+				t, fh := it.AtFloatHistogram(nil)
 				result = append(result, sample{t: t, fh: fh})
 			default:
 				panic("unexpected value type")
diff --git a/tsdb/compact_test.go b/tsdb/compact_test.go
index f33bb73c1..b2d2ea6e7 100644
--- a/tsdb/compact_test.go
+++ b/tsdb/compact_test.go
@@ -1007,10 +1007,10 @@ func TestCompaction_populateBlock(t *testing.T) {
 							s.t, s.f = iter.At()
 							samples = append(samples, s)
 						case chunkenc.ValHistogram:
-							s.t, s.h = iter.AtHistogram()
+							s.t, s.h = iter.AtHistogram(nil)
 							samples = append(samples, s)
 						case chunkenc.ValFloatHistogram:
-							s.t, s.fh = iter.AtFloatHistogram()
+							s.t, s.fh = iter.AtFloatHistogram(nil)
 							samples = append(samples, s)
 						default:
 							require.Fail(t, "unexpected value type")
diff --git a/tsdb/db_test.go b/tsdb/db_test.go
index 3bc094a3d..9e543ed50 100644
--- a/tsdb/db_test.go
+++ b/tsdb/db_test.go
@@ -107,10 +107,10 @@ func query(t testing.TB, q storage.Querier, matchers ...*labels.Matcher) map[str
 				ts, v := it.At()
 				samples = append(samples, sample{t: ts, f: v})
 			case chunkenc.ValHistogram:
-				ts, h := it.AtHistogram()
+				ts, h := it.AtHistogram(nil)
 				samples = append(samples, sample{t: ts, h: h})
 			case chunkenc.ValFloatHistogram:
-				ts, fh := it.AtFloatHistogram()
+				ts, fh := it.AtFloatHistogram(nil)
 				samples = append(samples, sample{t: ts, fh: fh})
 			default:
 				t.Fatalf("unknown sample type in query %s", typ.String())
@@ -6664,10 +6664,10 @@ func TestQueryHistogramFromBlocksWithCompaction(t *testing.T) {
 						ts, v := it.At()
 						slice = append(slice, sample{t: ts, f: v})
 					case chunkenc.ValHistogram:
-						ts, h := it.AtHistogram()
+						ts, h := it.AtHistogram(nil)
 						slice = append(slice, sample{t: ts, h: h})
 					case chunkenc.ValFloatHistogram:
-						ts, h := it.AtFloatHistogram()
+						ts, h := it.AtFloatHistogram(nil)
 						slice = append(slice, sample{t: ts, fh: h})
 					default:
 						t.Fatalf("unexpected sample value type %d", typ)
diff --git a/tsdb/head_test.go b/tsdb/head_test.go
index 5c2749bed..653c53a74 100644
--- a/tsdb/head_test.go
+++ b/tsdb/head_test.go
@@ -3377,10 +3377,10 @@ func TestAppendHistogram(t *testing.T) {
 			for typ := it.Next(); typ != chunkenc.ValNone; typ = it.Next() {
 				switch typ {
 				case chunkenc.ValHistogram:
-					ts, h := it.AtHistogram()
+					ts, h := it.AtHistogram(nil)
 					actHistograms = append(actHistograms, sample{t: ts, h: h})
 				case chunkenc.ValFloatHistogram:
-					ts, fh := it.AtFloatHistogram()
+					ts, fh := it.AtFloatHistogram(nil)
 					actFloatHistograms = append(actFloatHistograms, sample{t: ts, fh: fh})
 				}
 			}
@@ -4025,10 +4025,10 @@ func testHistogramStaleSampleHelper(t *testing.T, floatHistogram bool) {
 		for typ := it.Next(); typ != chunkenc.ValNone; typ = it.Next() {
 			switch typ {
 			case chunkenc.ValHistogram:
-				t, h := it.AtHistogram()
+				t, h := it.AtHistogram(nil)
 				actHistograms = append(actHistograms, timedHistogram{t: t, h: h})
 			case chunkenc.ValFloatHistogram:
-				t, h := it.AtFloatHistogram()
+				t, h := it.AtFloatHistogram(nil)
 				actHistograms = append(actHistograms, timedHistogram{t: t, fh: h})
 			}
 		}
diff --git a/tsdb/querier.go b/tsdb/querier.go
index 68aa95746..a692c98f1 100644
--- a/tsdb/querier.go
+++ b/tsdb/querier.go
@@ -820,12 +820,12 @@ func (p *populateWithDelSeriesIterator) At() (int64, float64) {
 	return p.curr.At()
 }
 
-func (p *populateWithDelSeriesIterator) AtHistogram() (int64, *histogram.Histogram) {
-	return p.curr.AtHistogram()
+func (p *populateWithDelSeriesIterator) AtHistogram(h *histogram.Histogram) (int64, *histogram.Histogram) {
+	return p.curr.AtHistogram(h)
 }
 
-func (p *populateWithDelSeriesIterator) AtFloatHistogram() (int64, *histogram.FloatHistogram) {
-	return p.curr.AtFloatHistogram()
+func (p *populateWithDelSeriesIterator) AtFloatHistogram(fh *histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
+	return p.curr.AtFloatHistogram(fh)
 }
 
 func (p *populateWithDelSeriesIterator) AtT() int64 {
@@ -937,7 +937,7 @@ func (p *populateWithDelChunkSeriesIterator) populateCurrForSingleChunk() bool {
 				break
 			}
 			var h *histogram.Histogram
-			t, h = p.currDelIter.AtHistogram()
+			t, h = p.currDelIter.AtHistogram(nil)
 			_, _, app, err = app.AppendHistogram(nil, t, h, true)
 			if err != nil {
 				break
@@ -968,7 +968,7 @@ func (p *populateWithDelChunkSeriesIterator) populateCurrForSingleChunk() bool {
 				break
 			}
 			var h *histogram.FloatHistogram
-			t, h = p.currDelIter.AtFloatHistogram()
+			t, h = p.currDelIter.AtFloatHistogram(nil)
 			_, _, app, err = app.AppendFloatHistogram(nil, t, h, true)
 			if err != nil {
 				break
@@ -1054,7 +1054,7 @@ func (p *populateWithDelChunkSeriesIterator) populateChunksFromIterable() bool {
 		case chunkenc.ValHistogram:
 			{
 				var v *histogram.Histogram
-				t, v = p.currDelIter.AtHistogram()
+				t, v = p.currDelIter.AtHistogram(nil)
 				// No need to set prevApp as AppendHistogram will set the
 				// counter reset header for the appender that's returned.
 				newChunk, recoded, app, err = app.AppendHistogram(nil, t, v, false)
@@ -1062,7 +1062,7 @@ func (p *populateWithDelChunkSeriesIterator) populateChunksFromIterable() bool {
 		case chunkenc.ValFloatHistogram:
 			{
 				var v *histogram.FloatHistogram
-				t, v = p.currDelIter.AtFloatHistogram()
+				t, v = p.currDelIter.AtFloatHistogram(nil)
 				// No need to set prevApp as AppendHistogram will set the
 				// counter reset header for the appender that's returned.
 				newChunk, recoded, app, err = app.AppendFloatHistogram(nil, t, v, false)
@@ -1233,13 +1233,13 @@ func (it *DeletedIterator) At() (int64, float64) {
 	return it.Iter.At()
 }
 
-func (it *DeletedIterator) AtHistogram() (int64, *histogram.Histogram) {
-	t, h := it.Iter.AtHistogram()
+func (it *DeletedIterator) AtHistogram(h *histogram.Histogram) (int64, *histogram.Histogram) {
+	t, h := it.Iter.AtHistogram(h)
 	return t, h
 }
 
-func (it *DeletedIterator) AtFloatHistogram() (int64, *histogram.FloatHistogram) {
-	t, h := it.Iter.AtFloatHistogram()
+func (it *DeletedIterator) AtFloatHistogram(fh *histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
+	t, h := it.Iter.AtFloatHistogram(fh)
 	return t, h
 }
 
diff --git a/tsdb/querier_test.go b/tsdb/querier_test.go
index 18d81b85b..1ed59ef1a 100644
--- a/tsdb/querier_test.go
+++ b/tsdb/querier_test.go
@@ -775,11 +775,11 @@ func (it *mockSampleIterator) At() (int64, float64) {
 	return it.s[it.idx].T(), it.s[it.idx].F()
 }
 
-func (it *mockSampleIterator) AtHistogram() (int64, *histogram.Histogram) {
+func (it *mockSampleIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) {
 	return it.s[it.idx].T(), it.s[it.idx].H()
 }
 
-func (it *mockSampleIterator) AtFloatHistogram() (int64, *histogram.FloatHistogram) {
+func (it *mockSampleIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
 	return it.s[it.idx].T(), it.s[it.idx].FH()
 }
 
@@ -1822,12 +1822,12 @@ func checkCurrVal(t *testing.T, valType chunkenc.ValueType, it *populateWithDelS
 		require.Equal(t, int64(expectedTs), ts)
 		require.Equal(t, float64(expectedValue), v)
 	case chunkenc.ValHistogram:
-		ts, h := it.AtHistogram()
+		ts, h := it.AtHistogram(nil)
 		require.Equal(t, int64(expectedTs), ts)
 		h.CounterResetHint = histogram.UnknownCounterReset
 		require.Equal(t, tsdbutil.GenerateTestHistogram(expectedValue), h)
 	case chunkenc.ValFloatHistogram:
-		ts, h := it.AtFloatHistogram()
+		ts, h := it.AtFloatHistogram(nil)
 		require.Equal(t, int64(expectedTs), ts)
 		h.CounterResetHint = histogram.UnknownCounterReset
 		require.Equal(t, tsdbutil.GenerateTestFloatHistogram(expectedValue), h)
diff --git a/tsdb/tsdbblockutil.go b/tsdb/tsdbblockutil.go
index d4e43b73c..f7b27c2e0 100644
--- a/tsdb/tsdbblockutil.go
+++ b/tsdb/tsdbblockutil.go
@@ -73,10 +73,10 @@ func CreateBlock(series []storage.Series, dir string, chunkRange int64, logger l
 				t, v := it.At()
 				ref, err = app.Append(ref, lset, t, v)
 			case chunkenc.ValHistogram:
-				t, h := it.AtHistogram()
+				t, h := it.AtHistogram(nil)
 				ref, err = app.AppendHistogram(ref, lset, t, h, nil)
 			case chunkenc.ValFloatHistogram:
-				t, fh := it.AtFloatHistogram()
+				t, fh := it.AtFloatHistogram(nil)
 				ref, err = app.AppendHistogram(ref, lset, t, nil, fh)
 			default:
 				return "", fmt.Errorf("unknown sample type %s", typ.String())
diff --git a/web/federate.go b/web/federate.go
index 22384a696..62e8c97b7 100644
--- a/web/federate.go
+++ b/web/federate.go
@@ -123,7 +123,7 @@ Loop:
 		case chunkenc.ValFloat:
 			t, f = it.At()
 		case chunkenc.ValFloatHistogram, chunkenc.ValHistogram:
-			t, fh = it.AtFloatHistogram()
+			t, fh = it.AtFloatHistogram(nil)
 		default:
 			sample, ok := it.PeekBack(1)
 			if !ok {

From 89523cf9b300fd0178c49dd6a7d36e56d3f654d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Rabenstein?= <beorn@grafana.com>
Date: Tue, 23 Jan 2024 17:05:34 +0100
Subject: [PATCH 107/115] doc: Mark `mad_over_time` as experimental (#13440)

We forgot to do that in
https://github.com/prometheus/prometheus/pull/13059

Signed-off-by: beorn7 <beorn@grafana.com>
---
 docs/querying/functions.md | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/docs/querying/functions.md b/docs/querying/functions.md
index 607e16e82..375104a0d 100644
--- a/docs/querying/functions.md
+++ b/docs/querying/functions.md
@@ -644,10 +644,15 @@ over time and return an instant vector with per-series aggregation results:
 * `quantile_over_time(scalar, range-vector)`: the φ-quantile (0 ≤ φ ≤ 1) of the values in the specified interval.
 * `stddev_over_time(range-vector)`: the population standard deviation of the values in the specified interval.
 * `stdvar_over_time(range-vector)`: the population standard variance of the values in the specified interval.
-* `mad_over_time(range-vector)`: the median absolute deviation of all points in the specified interval.
 * `last_over_time(range-vector)`: the most recent point value in the specified interval.
 * `present_over_time(range-vector)`: the value 1 for any series in the specified interval.
 
+If the [feature flag](../feature_flags/)
+`--enable-feature=promql-experimental-functions` is set, the following
+additional functions are available:
+
+* `mad_over_time(range-vector)`: the median absolute deviation of all points in the specified interval.
+
 Note that all values in the specified interval have the same weight in the
 aggregation even if the values are not equally spaced throughout the interval.
 

From ba7012ec6a7c104beb8866b79f6d07972c9e27c4 Mon Sep 17 00:00:00 2001
From: Arve Knudsen <arve.knudsen@gmail.com>
Date: Wed, 24 Jan 2024 10:47:56 +0100
Subject: [PATCH 108/115] TestHeadLabelValuesWithMatchers: Add test case
 (#13414)

Add test case to TestHeadLabelValuesWithMatchers, while fixing a couple
of typos in other test cases. Also enclosing some implicit sub-tests in a
`t.Run` call to make them explicitly sub-tests.

Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
---
 tsdb/block_test.go   |  2 +-
 tsdb/head_test.go    | 17 ++++++++++++++++-
 tsdb/querier_test.go | 16 +++++++++-------
 3 files changed, 26 insertions(+), 9 deletions(-)

diff --git a/tsdb/block_test.go b/tsdb/block_test.go
index d48418b57..ad7eb5557 100644
--- a/tsdb/block_test.go
+++ b/tsdb/block_test.go
@@ -264,7 +264,7 @@ func TestLabelValuesWithMatchers(t *testing.T) {
 			matchers:       []*labels.Matcher{labels.MustNewMatcher(labels.MatchRegexp, "unique", "value[5-7]5")},
 			expectedValues: []string{"value5", "value6", "value7"},
 		}, {
-			name:           "get tens by matching for absence of unique label",
+			name:           "get tens by matching for presence of unique label",
 			labelName:      "tens",
 			matchers:       []*labels.Matcher{labels.MustNewMatcher(labels.MatchNotEqual, "unique", "")},
 			expectedValues: []string{"value0", "value1", "value2", "value3", "value4", "value5", "value6", "value7", "value8", "value9"},
diff --git a/tsdb/head_test.go b/tsdb/head_test.go
index 653c53a74..e30f16d46 100644
--- a/tsdb/head_test.go
+++ b/tsdb/head_test.go
@@ -2772,6 +2772,13 @@ func TestHeadLabelValuesWithMatchers(t *testing.T) {
 	}
 	require.NoError(t, app.Commit())
 
+	var uniqueWithout30s []string
+	for i := 0; i < 100; i++ {
+		if i/10 != 3 {
+			uniqueWithout30s = append(uniqueWithout30s, fmt.Sprintf("value%d", i))
+		}
+	}
+	sort.Strings(uniqueWithout30s)
 	testCases := []struct {
 		name           string
 		labelName      string
@@ -2794,10 +2801,18 @@ func TestHeadLabelValuesWithMatchers(t *testing.T) {
 			matchers:       []*labels.Matcher{labels.MustNewMatcher(labels.MatchRegexp, "unique", "value[5-7]5")},
 			expectedValues: []string{"value5", "value6", "value7"},
 		}, {
-			name:           "get tens by matching for absence of unique label",
+			name:           "get tens by matching for presence of unique label",
 			labelName:      "tens",
 			matchers:       []*labels.Matcher{labels.MustNewMatcher(labels.MatchNotEqual, "unique", "")},
 			expectedValues: []string{"value0", "value1", "value2", "value3", "value4", "value5", "value6", "value7", "value8", "value9"},
+		}, {
+			name:      "get unique IDs based on tens not being equal to a certain value, while not empty",
+			labelName: "unique",
+			matchers: []*labels.Matcher{
+				labels.MustNewMatcher(labels.MatchNotEqual, "tens", "value3"),
+				labels.MustNewMatcher(labels.MatchNotEqual, "tens", ""),
+			},
+			expectedValues: uniqueWithout30s,
 		},
 	}
 
diff --git a/tsdb/querier_test.go b/tsdb/querier_test.go
index 1ed59ef1a..fcedc5462 100644
--- a/tsdb/querier_test.go
+++ b/tsdb/querier_test.go
@@ -3309,13 +3309,15 @@ func TestPostingsForMatcher(t *testing.T) {
 	}
 
 	for _, tc := range cases {
-		ir := &mockMatcherIndex{}
-		_, err := postingsForMatcher(ctx, ir, tc.matcher)
-		if tc.hasError {
-			require.Error(t, err)
-		} else {
-			require.NoError(t, err)
-		}
+		t.Run(tc.matcher.String(), func(t *testing.T) {
+			ir := &mockMatcherIndex{}
+			_, err := postingsForMatcher(ctx, ir, tc.matcher)
+			if tc.hasError {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+		})
 	}
 }
 

From 9a9e73b34a7d6a34083f486a66ce4243b89f45c6 Mon Sep 17 00:00:00 2001
From: Augustin Husson <husson.augustin@gmail.com>
Date: Wed, 24 Jan 2024 16:53:32 +0100
Subject: [PATCH 109/115] update all go dependencies (#13438)

Signed-off-by: Augustin Husson <husson.augustin@gmail.com>
---
 discovery/moby/docker.go                     |   6 +-
 documentation/examples/remote_storage/go.mod |  17 +-
 documentation/examples/remote_storage/go.sum | 114 +++++++--
 go.mod                                       | 110 ++++-----
 go.sum                                       | 236 ++++++++++---------
 tracing/tracing.go                           |   2 +-
 6 files changed, 292 insertions(+), 193 deletions(-)

diff --git a/discovery/moby/docker.go b/discovery/moby/docker.go
index 57fa62bb5..6a2b2d930 100644
--- a/discovery/moby/docker.go
+++ b/discovery/moby/docker.go
@@ -22,7 +22,7 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
 	"github.com/go-kit/log"
@@ -77,7 +77,7 @@ type DockerSDConfig struct {
 }
 
 // NewDiscovererMetrics implements discovery.Config.
-func (*DockerSDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
+func (*DockerSDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
 	return &dockerMetrics{
 		refreshMetrics: rmi,
 	}
@@ -192,7 +192,7 @@ func (d *DockerDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, er
 		Source: "Docker",
 	}
 
-	containers, err := d.client.ContainerList(ctx, types.ContainerListOptions{Filters: d.filters})
+	containers, err := d.client.ContainerList(ctx, container.ListOptions{Filters: d.filters})
 	if err != nil {
 		return nil, fmt.Errorf("error while listing containers: %w", err)
 	}
diff --git a/documentation/examples/remote_storage/go.mod b/documentation/examples/remote_storage/go.mod
index fa54daacc..e7b5bb8be 100644
--- a/documentation/examples/remote_storage/go.mod
+++ b/documentation/examples/remote_storage/go.mod
@@ -1,6 +1,6 @@
 module github.com/prometheus/prometheus/documentation/examples/remote_storage
 
-go 1.20
+go 1.21
 
 require (
 	github.com/alecthomas/kingpin/v2 v2.4.0
@@ -9,7 +9,7 @@ require (
 	github.com/golang/snappy v0.0.4
 	github.com/influxdata/influxdb v1.11.4
 	github.com/prometheus/client_golang v1.18.0
-	github.com/prometheus/common v0.45.0
+	github.com/prometheus/common v0.46.0
 	github.com/prometheus/prometheus v0.48.1
 	github.com/stretchr/testify v1.8.4
 )
@@ -39,7 +39,6 @@ require (
 	github.com/klauspost/compress v1.17.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
@@ -58,17 +57,17 @@ require (
 	go.opentelemetry.io/otel/trace v1.19.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/crypto v0.18.0 // indirect
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
-	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/oauth2 v0.13.0 // indirect
-	golang.org/x/sys v0.15.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/net v0.20.0 // indirect
+	golang.org/x/oauth2 v0.16.0 // indirect
+	golang.org/x/sys v0.16.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c // indirect
 	google.golang.org/grpc v1.58.3 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	google.golang.org/protobuf v1.32.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
diff --git a/documentation/examples/remote_storage/go.sum b/documentation/examples/remote_storage/go.sum
index 2e3af173b..9b8208baa 100644
--- a/documentation/examples/remote_storage/go.sum
+++ b/documentation/examples/remote_storage/go.sum
@@ -5,11 +5,14 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4 v4.2.1 h1:UPeCRD+XY7QlaGQte2EVI2iOcWvUYA2XY8w5T/8v0NQ=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4 v4.2.1/go.mod h1:oGV6NlB0cvi1ZbYRR2UN44QHxWFyGk+iylgD0qaMXjA=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0 h1:QM6sE5k2ZT/vI5BEe0r7mqjsUSnhVBFbOsVkEuaEfiA=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1 h1:bWh0Z2rOEDfB/ywv/l0iHN1JgyazE6kW/aIA89+CEK0=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1/go.mod h1:Bzf34hhAE9NSxailk8xVeLEZbUjOXcC+GnU1mMKdhLw=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 h1:WpB/QDNLpMw72xHJc34BNNykqSOeEJDAWkhf0u12/Jk=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
 github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
+github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
 github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjHpqDjYY=
 github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -20,6 +23,7 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
 github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
+github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
 github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/aws/aws-sdk-go v1.45.25 h1:c4fLlh5sLdK2DCRTY1z0hyuJZU4ygxX8m1FswL6/nF4=
 github.com/aws/aws-sdk-go v1.45.25/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
@@ -31,6 +35,7 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k=
+github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -39,19 +44,31 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc
 github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE=
 github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA=
 github.com/digitalocean/godo v1.104.1 h1:SZNxjAsskM/su0YW9P8Wx3gU0W1Z13b6tZlYNpl5BnA=
+github.com/digitalocean/godo v1.104.1/go.mod h1:VAI/L5YDzMuPRU01lEEUSQ/sp5Z//1HnnFv/RBTEdbg=
 github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
+github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
 github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
+github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v24.0.6+incompatible h1:hceabKCtUgDqPu+qm0NgsaXf28Ljf4/pWFL7xjWWDgE=
+github.com/docker/docker v24.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
+github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
+github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ=
+github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q=
 github.com/emicklei/go-restful/v3 v3.10.2 h1:hIovbnmBTLjHXkqEBUz3HGpXZdM7ZrE9fJIZIqlJLqE=
+github.com/emicklei/go-restful/v3 v3.10.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/envoyproxy/go-control-plane v0.11.1 h1:wSUXTlLfiAQRWs2F+p+EKOY9rUyis1MyGqJ2DIk5HpM=
+github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQLaTEw+YhGluxZkrTmD0g=
 github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA=
+github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE=
 github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
+github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
 github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
 github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
+github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
@@ -68,11 +85,16 @@ github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ=
+github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA=
 github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
+github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
 github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU=
+github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
 github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY=
+github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-zookeeper/zk v1.0.3 h1:7M2kwOsc//9VeeFiPtf+uSJlVpU66x9Ba5+8XK7/TDg=
+github.com/go-zookeeper/zk v1.0.3/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
@@ -94,43 +116,65 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
+github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
+github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
+github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
 github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gophercloud/gophercloud v1.7.0 h1:fyJGKh0LBvIZKLvBWvQdIgkaV5yTM3Jh9EYUh+UNCAs=
+github.com/gophercloud/gophercloud v1.7.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM=
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
+github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd h1:PpuIBO5P3e9hpqBD0O/HjhShYuM6XE0i/lbE6J94kww=
 github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A=
 github.com/hashicorp/consul/api v1.25.1 h1:CqrdhYzc8XZuPnhIYZWH45toM0LB9ZeYr/gvpLVI3PE=
+github.com/hashicorp/consul/api v1.25.1/go.mod h1:iiLVwR/htV7mas/sy0O+XSuEnrdBUUydemjxcUrAt4g=
 github.com/hashicorp/cronexpr v1.1.2 h1:wG/ZYIKT+RT3QkOdgYc+xsKWVRgnxJ1OJtjjy84fJ9A=
+github.com/hashicorp/cronexpr v1.1.2/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1Efb3g1PWA4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
+github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
 github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c=
+github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
 github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc=
+github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA=
+github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
 github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
+github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
 github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4=
+github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/nomad/api v0.0.0-20230721134942-515895c7690c h1:Nc3Mt2BAnq0/VoLEntF/nipX+K1S7pG+RgwiitSv6v0=
+github.com/hashicorp/nomad/api v0.0.0-20230721134942-515895c7690c/go.mod h1:O23qLAZuCx4htdY9zBaO4cJPXgleSFEdq6D/sezGgYE=
 github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY=
+github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4=
 github.com/hetznercloud/hcloud-go/v2 v2.4.0 h1:MqlAE+w125PLvJRCpAJmEwrIxoVdUdOyuFUhE/Ukbok=
+github.com/hetznercloud/hcloud-go/v2 v2.4.0/go.mod h1:l7fA5xsncFBzQTyw29/dw5Yr88yEGKKdc6BHf24ONS0=
 github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
+github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
 github.com/influxdata/influxdb v1.11.4 h1:H3pVW+/tWQ4lkHhZxVQ13Ov1hmhHYaAzz8L5aq3ZNtw=
 github.com/influxdata/influxdb v1.11.4/go.mod h1:VO6X2zlamfmEf+Esc9dR+7UQhdE/krspWNEZPwxCrp0=
 github.com/ionos-cloud/sdk-go/v6 v6.1.9 h1:Iq3VIXzeEbc8EbButuACgfLMiY5TPVWUPNrF+Vsddo4=
+github.com/ionos-cloud/sdk-go/v6 v6.1.9/go.mod h1:EzEgRIDxBELvfoa/uBN0kOQaqovLjUWEB7iW4/Q+t4k=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -145,11 +189,13 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2gPS5g=
 github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b h1:udzkj9S/zlT5X367kqJis0QP7YMxobob6zhzq6Yre00=
+github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -157,15 +203,20 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/linode/linodego v1.23.0 h1:s0ReCZtuN9Z1IoUN9w1RLeYO1dMZUGPwOQ/IBFsBHtU=
+github.com/linode/linodego v1.23.0/go.mod h1:0U7wj/UQOqBNbKv1FYTXiBUXueR8DY4HvIotwE0ENgg=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=
-github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
 github.com/miekg/dns v1.1.56 h1:5imZaSeoRNvpM9SzWNhEcP9QliKiz20/dA2QabIGVnE=
+github.com/miekg/dns v1.1.56/go.mod h1:cRm6Oo2C8TY9ZS/TqsSrseAcncm74lfK5G+ikN2SWWY=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
+github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -174,13 +225,18 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=
+github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/ovh/go-ovh v1.4.3 h1:Gs3V823zwTFpzgGLZNI6ILS4rmxZgJwJCz54Er9LwD0=
+github.com/ovh/go-ovh v1.4.3/go.mod h1:AkPXVtgwB6xlKblMjRKJJmjRp+ogrE7fz2lVgcQY8SY=
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -205,8 +261,8 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
 github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
-github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM=
-github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY=
+github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y=
+github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ=
 github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4=
 github.com/prometheus/common/sigv4 v0.1.0/go.mod h1:2Jkxxk9yYvCkE5G1sQT7GuEXm57JrvHu9k5YwTjsNtI=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
@@ -218,14 +274,18 @@ github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3c
 github.com/prometheus/prometheus v0.48.1 h1:CTszphSNTXkuCG6O0IfpKdHcJkvvnAAE1GbELKS+NFk=
 github.com/prometheus/prometheus v0.48.1/go.mod h1:SRw624aMAxTfryAcP8rOjg4S/sHHaetx2lyJJ2nM83g=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21 h1:yWfiTPwYxB0l5fGMhl/G+liULugVIHD9AU77iNLrURQ=
+github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -233,6 +293,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/vultr/govultr/v2 v2.17.2 h1:gej/rwr91Puc/tgh+j33p/BLR16UrIPnSr+AIwYWZQs=
+github.com/vultr/govultr/v2 v2.17.2/go.mod h1:ZFOKGWmgjytfyjeyAdhQlSWwTjh2ig+X49cAp50dzXI=
 github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc=
 github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -253,6 +314,7 @@ go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmY
 go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
 go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
+go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -260,14 +322,15 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
-golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
+golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
+golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -284,12 +347,12 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
-golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
-golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
+golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY=
-golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0=
+golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ=
+golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -298,6 +361,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
+golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -317,20 +381,21 @@ golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
+golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE=
+golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
 golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -339,6 +404,7 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=
+golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -348,7 +414,9 @@ google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID
 google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 h1:SeZZZx0cP0fqUyA+oRzP9k7cSwJlvDFiROO72uwD6i0=
+google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97/go.mod h1:t1VqOqqvce95G3hIDCT5FeO3YUc6Q4Oe24L/+rNMxRk=
 google.golang.org/genproto/googleapis/api v0.0.0-20231012201019-e917dd12ba7a h1:myvhA4is3vrit1a6NZCWBIwN0kNEnX21DJOJX/NvIfI=
+google.golang.org/genproto/googleapis/api v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:SUBoKXbI1Efip18FClrQVGjWcyd0QZd8KkvdP34t7ww=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c h1:jHkCUWkseRf+W+edG5hMzr/Uh1xkDREY4caybAq4dpY=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c/go.mod h1:4cYg8o5yUbm77w8ZX00LhMVNl/YVBFJRYWDc0uYWMs0=
 google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ=
@@ -361,14 +429,17 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
-google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
+google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
+gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -381,12 +452,21 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 k8s.io/api v0.28.2 h1:9mpl5mOb6vXZvqbQmankOfPIGiudghwCoLl1EYfUZbw=
+k8s.io/api v0.28.2/go.mod h1:RVnJBsjU8tcMq7C3iaRSGMeaKt2TWEUXcpIt/90fjEg=
 k8s.io/apimachinery v0.28.2 h1:KCOJLrc6gu+wV1BYgwik4AF4vXOlVJPdiqn0yAWWwXQ=
+k8s.io/apimachinery v0.28.2/go.mod h1:RdzF87y/ngqk9H4z3EL2Rppv5jj95vGS/HaFXrLDApU=
 k8s.io/client-go v0.28.2 h1:DNoYI1vGq0slMBN/SWKMZMw0Rq+0EQW6/AK4v9+3VeY=
+k8s.io/client-go v0.28.2/go.mod h1:sMkApowspLuc7omj1FOSUxSoqjr+d5Q0Yc0LOFnYFJY=
 k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
 k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
+k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ=
+k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM=
 k8s.io/utils v0.0.0-20230711102312-30195339c3c7 h1:ZgnF1KZsYxWIifwSNZFZgNtWE89WI5yiP5WwlfDoIyc=
+k8s.io/utils v0.0.0-20230711102312-30195339c3c7/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
+sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk=
+sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
 sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
diff --git a/go.mod b/go.mod
index fba6a7739..5147eaa9e 100644
--- a/go.mod
+++ b/go.mod
@@ -1,47 +1,47 @@
 module github.com/prometheus/prometheus
 
-go 1.20
+go 1.21
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0
-	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.3.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.4.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0
 	github.com/Code-Hex/go-generics-cache v1.3.1
 	github.com/KimMachineGun/automemlimit v0.5.0
 	github.com/alecthomas/kingpin/v2 v2.4.0
 	github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9
-	github.com/aws/aws-sdk-go v1.48.14
+	github.com/aws/aws-sdk-go v1.50.0
 	github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3
 	github.com/cespare/xxhash/v2 v2.2.0
 	github.com/dennwc/varint v1.0.0
-	github.com/digitalocean/godo v1.106.0
-	github.com/docker/docker v24.0.7+incompatible
+	github.com/digitalocean/godo v1.108.0
+	github.com/docker/docker v25.0.0+incompatible
 	github.com/edsrzf/mmap-go v1.1.0
-	github.com/envoyproxy/go-control-plane v0.11.1
-	github.com/envoyproxy/protoc-gen-validate v1.0.2
+	github.com/envoyproxy/go-control-plane v0.12.0
+	github.com/envoyproxy/protoc-gen-validate v1.0.4
 	github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb
 	github.com/fsnotify/fsnotify v1.7.0
 	github.com/go-kit/log v0.2.1
 	github.com/go-logfmt/logfmt v0.6.0
-	github.com/go-openapi/strfmt v0.21.9
+	github.com/go-openapi/strfmt v0.22.0
 	github.com/go-zookeeper/zk v1.0.3
 	github.com/gogo/protobuf v1.3.2
 	github.com/golang/snappy v0.0.4
-	github.com/google/pprof v0.0.0-20231205033806-a5a03c77bf08
-	github.com/google/uuid v1.4.0
+	github.com/google/pprof v0.0.0-20240117000934-35fc243c5815
+	github.com/google/uuid v1.5.0
 	github.com/gophercloud/gophercloud v1.8.0
 	github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0
-	github.com/hashicorp/consul/api v1.26.1
+	github.com/hashicorp/consul/api v1.27.0
 	github.com/hashicorp/nomad/api v0.0.0-20230721134942-515895c7690c // Not upgrading this for now due to https://github.com/prometheus/prometheus/pull/13255#issuecomment-1845237409
 	github.com/hetznercloud/hcloud-go/v2 v2.6.0
-	github.com/ionos-cloud/sdk-go/v6 v6.1.10
+	github.com/ionos-cloud/sdk-go/v6 v6.1.11
 	github.com/json-iterator/go v1.1.12
 	github.com/klauspost/compress v1.17.4
 	github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b
 	github.com/linode/linodego v1.27.1
-	github.com/miekg/dns v1.1.57
+	github.com/miekg/dns v1.1.58
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f
 	github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1
@@ -51,76 +51,77 @@ require (
 	github.com/prometheus/alertmanager v0.26.0
 	github.com/prometheus/client_golang v1.18.0
 	github.com/prometheus/client_model v0.5.0
-	github.com/prometheus/common v0.45.1-0.20231122191551-832cd6e99f99
+	github.com/prometheus/common v0.46.0
 	github.com/prometheus/common/assets v0.2.0
 	github.com/prometheus/common/sigv4 v0.1.0
-	github.com/prometheus/exporter-toolkit v0.10.0
-	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21
+	github.com/prometheus/exporter-toolkit v0.11.0
+	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.22
 	github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c
 	github.com/stretchr/testify v1.8.4
 	github.com/vultr/govultr/v2 v2.17.2
-	go.opentelemetry.io/collector/featuregate v1.0.0
-	go.opentelemetry.io/collector/pdata v1.0.0
-	go.opentelemetry.io/collector/semconv v0.90.1
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1
-	go.opentelemetry.io/otel v1.21.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0
-	go.opentelemetry.io/otel/sdk v1.21.0
-	go.opentelemetry.io/otel/trace v1.21.0
+	go.opentelemetry.io/collector/featuregate v1.0.1
+	go.opentelemetry.io/collector/pdata v1.0.1
+	go.opentelemetry.io/collector/semconv v0.92.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0
+	go.opentelemetry.io/otel v1.22.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0
+	go.opentelemetry.io/otel/sdk v1.22.0
+	go.opentelemetry.io/otel/trace v1.22.0
 	go.uber.org/atomic v1.11.0
 	go.uber.org/automaxprocs v1.5.3
 	go.uber.org/goleak v1.3.0
 	go.uber.org/multierr v1.11.0
-	golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb
+	golang.org/x/exp v0.0.0-20240119083558-1b970713d09a
 	golang.org/x/net v0.20.0
-	golang.org/x/oauth2 v0.15.0
-	golang.org/x/sync v0.5.0
+	golang.org/x/oauth2 v0.16.0
+	golang.org/x/sync v0.6.0
 	golang.org/x/sys v0.16.0
 	golang.org/x/time v0.5.0
-	golang.org/x/tools v0.16.0
-	google.golang.org/api v0.153.0
-	google.golang.org/genproto/googleapis/api v0.0.0-20231127180814-3a041ad873d4
-	google.golang.org/grpc v1.59.0
-	google.golang.org/protobuf v1.31.0
+	golang.org/x/tools v0.17.0
+	google.golang.org/api v0.157.0
+	google.golang.org/genproto/googleapis/api v0.0.0-20240116215550-a9fa1716bcac
+	google.golang.org/grpc v1.61.0
+	google.golang.org/protobuf v1.32.0
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.28.4
 	k8s.io/apimachinery v0.28.4
 	k8s.io/client-go v0.28.4
 	k8s.io/klog v1.0.0
-	k8s.io/klog/v2 v2.110.1
+	k8s.io/klog/v2 v2.120.1
 )
 
 require (
 	cloud.google.com/go/compute v1.23.3 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cilium/ebpf v0.11.0 // indirect
-	github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 // indirect
+	github.com/cncf/xds/go v0.0.0-20231109132714-523115ebc101 // indirect
 	github.com/containerd/cgroups/v3 v3.0.3 // indirect
+	github.com/containerd/log v0.1.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/docker/distribution v2.8.2+incompatible // indirect
+	github.com/distribution/reference v0.5.0 // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
-	github.com/emicklei/go-restful/v3 v3.10.2 // indirect
+	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/fatih/color v1.15.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-kit/kit v0.12.0 // indirect
-	github.com/go-logr/logr v1.3.0 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/analysis v0.21.4 // indirect
-	github.com/go-openapi/errors v0.20.4 // indirect
+	github.com/go-openapi/errors v0.21.0 // indirect
 	github.com/go-openapi/jsonpointer v0.20.0 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/loads v0.21.2 // indirect
@@ -129,7 +130,7 @@ require (
 	github.com/go-openapi/validate v0.22.1 // indirect
 	github.com/go-resty/resty/v2 v2.11.0 // indirect
 	github.com/godbus/dbus/v5 v5.0.4 // indirect
-	github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
+	github.com/golang-jwt/jwt/v5 v5.2.0 // indirect
 	github.com/golang/glog v1.1.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
@@ -162,7 +163,6 @@ require (
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
-	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
@@ -173,32 +173,32 @@ require (
 	github.com/opencontainers/image-spec v1.0.2 // indirect
 	github.com/opencontainers/runtime-spec v1.0.2 // indirect
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
-	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
+	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
-	github.com/sirupsen/logrus v1.9.0 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/objx v0.5.0 // indirect
 	github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
 	go.mongodb.org/mongo-driver v1.13.1 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.opentelemetry.io/otel/metric v1.21.0 // indirect
+	go.opentelemetry.io/otel/metric v1.22.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
 	golang.org/x/crypto v0.18.0 // indirect
 	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/term v0.16.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect
+	google.golang.org/appengine v1.6.8 // indirect
+	google.golang.org/genproto v0.0.0-20240102182953-50ed04b92917 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gotest.tools/v3 v3.0.3 // indirect
-	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
-	k8s.io/utils v0.0.0-20230711102312-30195339c3c7 // indirect
+	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
+	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
diff --git a/go.sum b/go.sum
index e5c269f47..209f58cbb 100644
--- a/go.sum
+++ b/go.sum
@@ -34,22 +34,24 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0 h1:fb8kj/Dh4CSwgsOzHeZY4Xh68cFVbzXx+ONXGMY//4w=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0/go.mod h1:uReU2sSxZExRPBAg3qKzmAucSi51+SP1OhohieR821Q=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZMmXGkOcvfFtD0oHVZ1TIPRI=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG+S3q8UoJcmyU6nUeunJcMDHcRYHhs=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 h1:d81/ng9rET2YqdVkVwkb6EXeRrLJIwyGnJcAlAWKwhs=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.3.0 h1:qgs/VAMSR+9qFhwTw4OwF2NbVuw+2m83pVZJjqkKQMw=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.3.0/go.mod h1:uYt4CfhkJA9o0FN7jfE5minm/i4nUE4MjGUJkzB6Zs8=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 h1:lGlwhPtrX6EVml1hO0ivjkUxsSyl4dsiw9qcA1k/3IQ=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1/go.mod h1:RKUqNu35KJYcVG/fqTRqmuXJZYNhYkBrnC/hX7yGbTA=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 h1:6oNBlSdi1QqM1PNW7FPA6xOGA5UNsXnkaYZz9vdPGhA=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.4.0 h1:QfV5XZt6iNa2aWMAt96CZEbfJ7kgG/qYIpq465Shr5E=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.4.0/go.mod h1:uYt4CfhkJA9o0FN7jfE5minm/i4nUE4MjGUJkzB6Zs8=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0 h1:bXwSugBiSbgtz7rOtbfGf+woewp4f06orW9OP5BjHLA=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0/go.mod h1:Y/HgrePTmGy9HjdSGTqZNa+apUpTVIEVKXJyARP2lrk=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 h1:7CBQ+Ei8SP2c6ydQTGCCrS35bDxgTMfoP2miAwK++OU=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1/go.mod h1:c/wcGeGx5FUPbM/JltUYHZcKmigwyVLJlDq+4HdtXaw=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 h1:WpB/QDNLpMw72xHJc34BNNykqSOeEJDAWkhf0u12/Jk=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Code-Hex/go-generics-cache v1.3.1 h1:i8rLwyhoyhaerr7JpjtYjJZUcCbWOdiYO3fZXLiEC4g=
@@ -91,8 +93,8 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:W
 github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
 github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
-github.com/aws/aws-sdk-go v1.48.14 h1:nVLrp+F84SG+xGiFMfe1TE6ZV6smF+42tuuNgYGV30s=
-github.com/aws/aws-sdk-go v1.48.14/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
+github.com/aws/aws-sdk-go v1.50.0 h1:HBtrLeO+QyDKnc3t1+5DR1RxodOHCGr8ZcrHudpv7jI=
+github.com/aws/aws-sdk-go v1.50.0/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
 github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 h1:6df1vn4bBlDDo4tARvBm7l6KA9iVMnE3NWizDeWSrps=
 github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3/go.mod h1:CIWtjkly68+yqLPbvwwR/fjNJA/idrtULjZWh2v1ys0=
@@ -119,12 +121,14 @@ github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp
 github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k=
-github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20231109132714-523115ebc101 h1:7To3pQ+pZo0i3dsWEbinPNFs5gPSBOsJtx3wTT94VBY=
+github.com/cncf/xds/go v0.0.0-20231109132714-523115ebc101/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0=
 github.com/containerd/cgroups/v3 v3.0.3/go.mod h1:8HBe7V3aWGLFPd/k03swSIsGjZhHI2WzJmticMgVuz0=
+github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
+github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
@@ -141,13 +145,14 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc
 github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE=
 github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/digitalocean/godo v1.106.0 h1:m5iErwl3xHovGFlawd50n54ntgXHt1BLsvU6BXsVxEU=
-github.com/digitalocean/godo v1.106.0/go.mod h1:R6EmmWI8CT1+fCtjWY9UCB+L5uufuZH13wk3YhxycCs=
+github.com/digitalocean/godo v1.108.0 h1:fWyMENvtxpCpva1UbKzOFnyAS04N1FNuBWWfPeTGquQ=
+github.com/digitalocean/godo v1.108.0/go.mod h1:R6EmmWI8CT1+fCtjWY9UCB+L5uufuZH13wk3YhxycCs=
+github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0=
+github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
 github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
-github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
-github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
-github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
+github.com/docker/docker v25.0.0+incompatible h1:g9b6wZTblhMgzOT2tspESstfw6ySZ9kdm94BLDKaZac=
+github.com/docker/docker v25.0.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
@@ -159,17 +164,17 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP
 github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
 github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ=
 github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q=
-github.com/emicklei/go-restful/v3 v3.10.2 h1:hIovbnmBTLjHXkqEBUz3HGpXZdM7ZrE9fJIZIqlJLqE=
-github.com/emicklei/go-restful/v3 v3.10.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
+github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/go-control-plane v0.11.1 h1:wSUXTlLfiAQRWs2F+p+EKOY9rUyis1MyGqJ2DIk5HpM=
-github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQLaTEw+YhGluxZkrTmD0g=
+github.com/envoyproxy/go-control-plane v0.12.0 h1:4X+VP1GHd1Mhj6IB5mMeGbLCleqxjletLK6K0rbxyZI=
+github.com/envoyproxy/go-control-plane v0.12.0/go.mod h1:ZBTaoJ23lqITozF0M6G4/IragXCQKCnYbmlmtHvwRG0=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA=
-github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE=
+github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A=
+github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew=
 github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=
 github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb h1:IT4JYU7k4ikYg1SCxNI1/Tieq/NFvh6dzLdgi7eu0tM=
@@ -184,6 +189,7 @@ github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw
 github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
 github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
 github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA=
+github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
@@ -206,8 +212,8 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG
 github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4=
 github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
-github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
+github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-openapi/analysis v0.21.2/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY=
@@ -216,8 +222,8 @@ github.com/go-openapi/analysis v0.21.4/go.mod h1:4zQ35W4neeZTqh3ol0rv/O8JBbka9Qy
 github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
 github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
 github.com/go-openapi/errors v0.20.2/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
-github.com/go-openapi/errors v0.20.4 h1:unTcVm6PispJsMECE3zWgvG4xTiKda1LIR5rCRWLG6M=
-github.com/go-openapi/errors v0.20.4/go.mod h1:Z3FlZ4I8jEGxjUK+bugx3on2mIAk4txuAOhlsB1FSgk=
+github.com/go-openapi/errors v0.21.0 h1:FhChC/duCnfoLj1gZ0BgaBmzhJC2SL/sJr8a2vAobSY=
+github.com/go-openapi/errors v0.21.0/go.mod h1:jxNTMUxRCKj65yb/okJGEtahVd7uvWnuWfj53bse4ho=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
@@ -237,8 +243,8 @@ github.com/go-openapi/spec v0.20.9/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6
 github.com/go-openapi/strfmt v0.21.0/go.mod h1:ZRQ409bWMj+SOgXofQAGTIo2Ebu72Gs+WaRADcS5iNg=
 github.com/go-openapi/strfmt v0.21.1/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k=
 github.com/go-openapi/strfmt v0.21.3/go.mod h1:k+RzNO0Da+k3FrrynSNN8F7n/peCmQQqbbXjtDfvmGg=
-github.com/go-openapi/strfmt v0.21.9 h1:LnEGOO9qyEC1v22Bzr323M98G13paIUGPU7yeJtG9Xs=
-github.com/go-openapi/strfmt v0.21.9/go.mod h1:0k3v301mglEaZRJdDDGSlN6Npq4VMVU69DE0LUyf7uA=
+github.com/go-openapi/strfmt v0.22.0 h1:Ew9PnEYc246TwrEspvBdDHS4BVKXy/AOVsfqGDgAcaI=
+github.com/go-openapi/strfmt v0.22.0/go.mod h1:HzJ9kokGIju3/K6ap8jL+OlGAbjpSv27135Yr9OivU4=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
 github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
@@ -252,6 +258,7 @@ github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
 github.com/go-zookeeper/zk v1.0.3 h1:7M2kwOsc//9VeeFiPtf+uSJlVpU66x9Ba5+8XK7/TDg=
 github.com/go-zookeeper/zk v1.0.3/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
 github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
@@ -286,8 +293,8 @@ github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE=
-github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
+github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw=
+github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo=
 github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ=
@@ -319,6 +326,7 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
@@ -328,6 +336,7 @@ github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4=
+github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
 github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
 github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -358,16 +367,16 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20231205033806-a5a03c77bf08 h1:PxlBVtIFHR/mtWk2i0gTEdCz+jBnqiuHNSki0epDbVs=
-github.com/google/pprof v0.0.0-20231205033806-a5a03c77bf08/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
+github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 h1:WzfWbQz/Ze8v6l++GGbGNFZnUShVpP/0xffCPLL+ax8=
+github.com/google/pprof v0.0.0-20240117000934-35fc243c5815/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o=
 github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
-github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
+github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs=
 github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
@@ -393,10 +402,11 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFb
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg=
 github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE=
-github.com/hashicorp/consul/api v1.26.1 h1:5oSXOO5fboPZeW5SN+TdGFP/BILDgBm19OrPZ/pICIM=
-github.com/hashicorp/consul/api v1.26.1/go.mod h1:B4sQTeaSO16NtynqrAdwOlahJ7IUDZM9cj2420xYL8A=
+github.com/hashicorp/consul/api v1.27.0 h1:gmJ6DPKQog1426xsdmgk5iqDyoRiNc+ipBdJOqKQFjc=
+github.com/hashicorp/consul/api v1.27.0/go.mod h1:JkekNRSou9lANFdt+4IKx3Za7XY0JzzpQjEb4Ivo1c8=
 github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
-github.com/hashicorp/consul/sdk v0.15.0 h1:2qK9nDrr4tiJKRoxPGhm6B7xJjLVIQqkjiab2M4aKjU=
+github.com/hashicorp/consul/sdk v0.15.1 h1:kKIGxc7CZtflcF5DLfHeq7rOQmRq3vk7kwISN9bif8Q=
+github.com/hashicorp/consul/sdk v0.15.1/go.mod h1:7pxqqhqoaPqnBnzXD1StKed62LqJeClzVsUEy85Zr0A=
 github.com/hashicorp/cronexpr v1.1.2 h1:wG/ZYIKT+RT3QkOdgYc+xsKWVRgnxJ1OJtjjy84fJ9A=
 github.com/hashicorp/cronexpr v1.1.2/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1Efb3g1PWA4=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -414,6 +424,7 @@ github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJ
 github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI=
+github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
@@ -426,10 +437,12 @@ github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5O
 github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc=
+github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
+github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
 github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
@@ -458,9 +471,10 @@ github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
 github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
-github.com/ionos-cloud/sdk-go/v6 v6.1.10 h1:3815Q2Hw/wc4cJ8wD7bwfsmDsdfIEp80B7BQMj0YP2w=
-github.com/ionos-cloud/sdk-go/v6 v6.1.10/go.mod h1:EzEgRIDxBELvfoa/uBN0kOQaqovLjUWEB7iW4/Q+t4k=
+github.com/ionos-cloud/sdk-go/v6 v6.1.11 h1:J/uRN4UWO3wCyGOeDdMKv8LWRzKu6UIkLEaes38Kzh8=
+github.com/ionos-cloud/sdk-go/v6 v6.1.11/go.mod h1:EzEgRIDxBELvfoa/uBN0kOQaqovLjUWEB7iW4/Q+t4k=
 github.com/jarcoal/httpmock v1.3.0 h1:2RJ8GP0IIaWwcC9Fp2BmVi8Kog3v2Hn7VXM3fTd+nuc=
+github.com/jarcoal/httpmock v1.3.0/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
@@ -503,6 +517,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -539,14 +554,13 @@ github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APP
 github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=
-github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
 github.com/maxatome/go-testdeep v1.12.0 h1:Ql7Go8Tg0C1D/uMMX59LAoYK7LffeJQ6X2T04nTH68g=
+github.com/maxatome/go-testdeep v1.12.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
 github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
-github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM=
-github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk=
+github.com/miekg/dns v1.1.58 h1:ca2Hdkz+cDg/7eNF6V56jjzuZ4aCAE+DbVkILdQWG/4=
+github.com/miekg/dns v1.1.58/go.mod h1:Ypv+3b/KadlvW9vJfXOTf300O4UqaHFzFCuHz+rPkBY=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@@ -554,6 +568,7 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU=
+github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8=
 github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
@@ -600,8 +615,10 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo/v2 v2.9.4 h1:xR7vG4IXt5RWx6FfIjyAtsoMAtnc3C/rFXBBd2AjZwE=
+github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLywzIhbKM=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
+github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
@@ -630,8 +647,8 @@ github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAv
 github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
 github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
-github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
-github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
+github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
+github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -643,6 +660,7 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=
 github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
+github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
 github.com/prometheus/alertmanager v0.26.0 h1:uOMJWfIwJguc3NaM3appWNbbrh6G/OjvaHMk22aBBYc=
 github.com/prometheus/alertmanager v0.26.0/go.mod h1:rVcnARltVjavgVaNnmevxK7kOn7IZavyf0KNgHkbEpU=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
@@ -669,14 +687,14 @@ github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8b
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
 github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
-github.com/prometheus/common v0.45.1-0.20231122191551-832cd6e99f99 h1:V5ajRiLiCQGO+ggTr+07gMUcTqlIMMkDBfrJe5zKLmc=
-github.com/prometheus/common v0.45.1-0.20231122191551-832cd6e99f99/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY=
+github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y=
+github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ=
 github.com/prometheus/common/assets v0.2.0 h1:0P5OrzoHrYBOSM1OigWL3mY8ZvV2N4zIE/5AahrSrfM=
 github.com/prometheus/common/assets v0.2.0/go.mod h1:D17UVUE12bHbim7HzwUvtqm6gwBEaDQ0F+hIGbFbccI=
 github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4=
 github.com/prometheus/common/sigv4 v0.1.0/go.mod h1:2Jkxxk9yYvCkE5G1sQT7GuEXm57JrvHu9k5YwTjsNtI=
-github.com/prometheus/exporter-toolkit v0.10.0 h1:yOAzZTi4M22ZzVxD+fhy1URTuNRj/36uQJJ5S8IPza8=
-github.com/prometheus/exporter-toolkit v0.10.0/go.mod h1:+sVFzuvV5JDyw+Ih6p3zFxZNVnKQa3x5qPmDSiPu4ZY=
+github.com/prometheus/exporter-toolkit v0.11.0 h1:yNTsuZ0aNCNFQ3aFTD2uhPOvr4iD7fdBvKPAEGkNf+g=
+github.com/prometheus/exporter-toolkit v0.11.0/go.mod h1:BVnENhnNecpwoTLiABx7mrPB/OLRIgN74qlQbV+FK1Q=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
@@ -692,14 +710,16 @@ github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
+github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
-github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21 h1:yWfiTPwYxB0l5fGMhl/G+liULugVIHD9AU77iNLrURQ=
-github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg=
+github.com/scaleway/scaleway-sdk-go v1.0.0-beta.22 h1:wJrcTdddKOI8TFxs8cemnhKP2EmKy3yfUKHj3ZdfzYo=
+github.com/scaleway/scaleway-sdk-go v1.0.0-beta.22/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/shoenig/test v0.6.6 h1:Oe8TPH9wAbv++YPNDKJWUnI8Q4PPWCx3UbOfH+FxiMU=
+github.com/shoenig/test v0.6.6/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
 github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c h1:aqg5Vm5dwtvL+YgDpBcK1ITf3o96N/K7/wsRXQnUTEs=
 github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c/go.mod h1:owqhoLW1qZoYLZzLnBw+QkPP9WZnjlSWihhxAJC1+/M=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
@@ -712,8 +732,8 @@ github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
-github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
-github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
@@ -781,28 +801,28 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
-go.opentelemetry.io/collector/featuregate v1.0.0 h1:5MGqe2v5zxaoo73BUOvUTunftX5J8RGrbFsC2Ha7N3g=
-go.opentelemetry.io/collector/featuregate v1.0.0/go.mod h1:xGbRuw+GbutRtVVSEy3YR2yuOlEyiUMhN2M9DJljgqY=
-go.opentelemetry.io/collector/pdata v1.0.0 h1:ECP2jnLztewsHmL1opL8BeMtWVc7/oSlKNhfY9jP8ec=
-go.opentelemetry.io/collector/pdata v1.0.0/go.mod h1:TsDFgs4JLNG7t6x9D8kGswXUz4mme+MyNChHx8zSF6k=
-go.opentelemetry.io/collector/semconv v0.90.1 h1:2fkQZbefQBbIcNb9Rk1mRcWlFZgQOk7CpST1e1BK8eg=
-go.opentelemetry.io/collector/semconv v0.90.1/go.mod h1:j/8THcqVxFna1FpvA2zYIsUperEtOaRaqoLYIN4doWw=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo=
-go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc=
-go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruOgJP5QiA1pw4fYYdv6nc6CBWw=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0/go.mod h1:zgBdWWAu7oEEMC06MMKc5NLbA/1YDXV1sMpSqEeLQLg=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 h1:tIqheXEFWAZ7O8A7m+J0aPTmpJN3YQ7qetUAdkkkKpk=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0/go.mod h1:nUeKExfxAQVbiVFn32YXpXZZHZ61Cc3s3Rn1pDBGAb0=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 h1:digkEZCJWobwBqMwC0cwCq8/wkkRy/OowZg5OArWZrM=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0/go.mod h1:/OpE/y70qVkndM0TrxT4KBoN3RsFZP0QaofcfYrj76I=
-go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4=
-go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM=
-go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8=
-go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E=
-go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc=
-go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ=
+go.opentelemetry.io/collector/featuregate v1.0.1 h1:ok//hLSXttBbyu4sSV1pTx1nKdr5udSmrWy5sFMIIbM=
+go.opentelemetry.io/collector/featuregate v1.0.1/go.mod h1:QQXjP4etmJQhkQ20j4P/rapWuItYxoFozg/iIwuKnYg=
+go.opentelemetry.io/collector/pdata v1.0.1 h1:dGX2h7maA6zHbl5D3AsMnF1c3Nn+3EUftbVCLzeyNvA=
+go.opentelemetry.io/collector/pdata v1.0.1/go.mod h1:jutXeu0QOXYY8wcZ/hege+YAnSBP3+jpTqYU1+JTI5Y=
+go.opentelemetry.io/collector/semconv v0.92.0 h1:3+OGPPuVu4rtrz8qGbpbiw7eKKULj4iJaSDTV52HM40=
+go.opentelemetry.io/collector/semconv v0.92.0/go.mod h1:gZ0uzkXsN+J5NpiRcdp9xOhNGQDDui8Y62p15sKrlzo=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 h1:sv9kVfal0MK0wBMCOGr+HeJm9v803BkJxGrk2au7j08=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw=
+go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y=
+go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 h1:9M3+rhx7kZCIQQhQRYaZCdNu1V73tm4TvXs2ntl98C4=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0/go.mod h1:noq80iT8rrHP1SfybmPiRGc9dc5M8RPmGvtwo7Oo7tc=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 h1:H2JFgRcGiyHg7H7bwcwaQJYrNFqCqrbTQ8K4p1OvDu8=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0/go.mod h1:WfCWp1bGoYK8MeULtI15MmQVczfR+bFkk0DF3h06QmQ=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0 h1:FyjCyI9jVEfqhUh2MoSkmolPjfh5fp2hnV0b0irxH4Q=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0/go.mod h1:hYwym2nDEeZfG/motx0p7L7J1N1vyzIThemQsb4g2qY=
+go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg=
+go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY=
+go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw=
+go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc=
+go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0=
+go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo=
 go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
 go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
@@ -847,8 +867,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb h1:c0vyKkb6yr3KR7jEfJaOSv4lG7xPkbN6r52aJz1d8a8=
-golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
+golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA=
+golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -926,8 +946,8 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ=
-golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM=
+golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ=
+golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -942,8 +962,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
-golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -998,7 +1018,6 @@ golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1008,6 +1027,7 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1096,8 +1116,8 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM=
-golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0=
+golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
+golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1117,8 +1137,8 @@ google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/
 google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
 google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.153.0 h1:N1AwGhielyKFaUqH07/ZSIQR3uNPcV7NVw0vj+j4iR4=
-google.golang.org/api v0.153.0/go.mod h1:3qNJX5eOmhiWYc67jRA/3GsDw97UFb5ivv7Y2PrriAY=
+google.golang.org/api v0.157.0 h1:ORAeqmbrrozeyw5NjnMxh7peHO0UzV4wWYSwZeCUb20=
+google.golang.org/api v0.157.0/go.mod h1:+z4v4ufbZ1WEpld6yMGHyggs+PmAHiaLNj5ytP3N01g=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1126,8 +1146,8 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
-google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
+google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -1157,12 +1177,12 @@ google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1m
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
 google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f h1:Vn+VyHU5guc9KjB5KrjI2q0wCOWEOIh0OEsleqakHJg=
-google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f/go.mod h1:nWSwAFPb+qfNJXsoeO3Io7zf4tMSfN8EA8RlDA04GhY=
-google.golang.org/genproto/googleapis/api v0.0.0-20231127180814-3a041ad873d4 h1:ZcOkrmX74HbKFYnpPY8Qsw93fC29TbJXspYKaBkSXDQ=
-google.golang.org/genproto/googleapis/api v0.0.0-20231127180814-3a041ad873d4/go.mod h1:k2dtGpRrbsSyKcNPKKI5sstZkrNCZwpU/ns96JoHbGg=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f h1:ultW7fxlIvee4HYrtnaRPon9HpEgFk5zYpmfMgtKB5I=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f/go.mod h1:L9KNLi232K1/xB6f7AlSX692koaRnKaWSR0stBki0Yc=
+google.golang.org/genproto v0.0.0-20240102182953-50ed04b92917 h1:nz5NESFLZbJGPFxDT/HCn+V1mZ8JGNoY4nUpmW/Y2eg=
+google.golang.org/genproto v0.0.0-20240102182953-50ed04b92917/go.mod h1:pZqR+glSb11aJ+JQcczCvgf47+duRuzNSKqE8YAQnV0=
+google.golang.org/genproto/googleapis/api v0.0.0-20240116215550-a9fa1716bcac h1:OZkkudMUu9LVQMCoRUbI/1p5VCo9BOrlvkqMvWtqa6s=
+google.golang.org/genproto/googleapis/api v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:B5xPO//w8qmBDjGReYLpR6UJPnkldGkCSMoH/2vxJeg=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac h1:nUQEQmH/csSvFECKYRv6HWEyypysidKl2I6Qpsglq/0=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:daQN87bsDqDoe316QbbvX60nMoJQa4r6Ds0ZuoAe5yA=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
@@ -1181,8 +1201,8 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji
 google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
-google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
-google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
+google.golang.org/grpc v1.61.0 h1:TOvOcuXn30kRao+gfcvsebNEa5iZIiLkisYEkf7R7o0=
+google.golang.org/grpc v1.61.0/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -1195,8 +1215,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
-google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
+google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -1248,17 +1268,17 @@ k8s.io/apimachinery v0.28.4 h1:zOSJe1mc+GxuMnFzD4Z/U1wst50X28ZNsn5bhgIIao8=
 k8s.io/apimachinery v0.28.4/go.mod h1:wI37ncBvfAoswfq626yPTe6Bz1c22L7uaJ8dho83mgg=
 k8s.io/client-go v0.28.4 h1:Np5ocjlZcTrkyRJ3+T3PkXDpe4UpatQxj85+xjaD2wY=
 k8s.io/client-go v0.28.4/go.mod h1:0VDZFpgoZfelyP5Wqu0/r/TRYcLYuJ2U1KEeoaPa1N4=
-k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ=
-k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM=
-k8s.io/utils v0.0.0-20230711102312-30195339c3c7 h1:ZgnF1KZsYxWIifwSNZFZgNtWE89WI5yiP5WwlfDoIyc=
-k8s.io/utils v0.0.0-20230711102312-30195339c3c7/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780=
+k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA=
+k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=
+k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
-sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk=
-sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
+sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=
+sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
 sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
diff --git a/tracing/tracing.go b/tracing/tracing.go
index d17540a91..6a1913737 100644
--- a/tracing/tracing.go
+++ b/tracing/tracing.go
@@ -30,7 +30,7 @@ import (
 	"go.opentelemetry.io/otel/propagation"
 	"go.opentelemetry.io/otel/sdk/resource"
 	tracesdk "go.opentelemetry.io/otel/sdk/trace"
-	semconv "go.opentelemetry.io/otel/semconv/v1.21.0"
+	semconv "go.opentelemetry.io/otel/semconv/v1.24.0"
 	"go.opentelemetry.io/otel/trace"
 	"go.opentelemetry.io/otel/trace/noop"
 	"google.golang.org/grpc/credentials"

From 94ced726e27629d25e5668db7eb51ba2b3aad923 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 25 Jan 2024 10:20:14 +0100
Subject: [PATCH 110/115] build(deps): bump the k8s-io group with 2 updates
 (#13454)

Bumps the k8s-io group with 2 updates: [k8s.io/api](https://github.com/kubernetes/api) and [k8s.io/client-go](https://github.com/kubernetes/client-go).


Updates `k8s.io/api` from 0.28.4 to 0.29.1
- [Commits](https://github.com/kubernetes/api/compare/v0.28.4...v0.29.1)

Updates `k8s.io/client-go` from 0.28.4 to 0.29.1
- [Changelog](https://github.com/kubernetes/client-go/blob/master/CHANGELOG.md)
- [Commits](https://github.com/kubernetes/client-go/compare/v0.28.4...v0.29.1)

---
updated-dependencies:
- dependency-name: k8s.io/api
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: k8s-io
- dependency-name: k8s.io/client-go
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: k8s-io
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod |  6 +++---
 go.sum | 20 ++++++++++----------
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/go.mod b/go.mod
index 5147eaa9e..c3c7facb5 100644
--- a/go.mod
+++ b/go.mod
@@ -86,9 +86,9 @@ require (
 	google.golang.org/protobuf v1.32.0
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
-	k8s.io/api v0.28.4
-	k8s.io/apimachinery v0.28.4
-	k8s.io/client-go v0.28.4
+	k8s.io/api v0.29.1
+	k8s.io/apimachinery v0.29.1
+	k8s.io/client-go v0.29.1
 	k8s.io/klog v1.0.0
 	k8s.io/klog/v2 v2.120.1
 )
diff --git a/go.sum b/go.sum
index 209f58cbb..f221d99d2 100644
--- a/go.sum
+++ b/go.sum
@@ -614,11 +614,11 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo/v2 v2.9.4 h1:xR7vG4IXt5RWx6FfIjyAtsoMAtnc3C/rFXBBd2AjZwE=
-github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLywzIhbKM=
+github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
+github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
-github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
+github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg=
+github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
@@ -1262,12 +1262,12 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.28.4 h1:8ZBrLjwosLl/NYgv1P7EQLqoO8MGQApnbgH8tu3BMzY=
-k8s.io/api v0.28.4/go.mod h1:axWTGrY88s/5YE+JSt4uUi6NMM+gur1en2REMR7IRj0=
-k8s.io/apimachinery v0.28.4 h1:zOSJe1mc+GxuMnFzD4Z/U1wst50X28ZNsn5bhgIIao8=
-k8s.io/apimachinery v0.28.4/go.mod h1:wI37ncBvfAoswfq626yPTe6Bz1c22L7uaJ8dho83mgg=
-k8s.io/client-go v0.28.4 h1:Np5ocjlZcTrkyRJ3+T3PkXDpe4UpatQxj85+xjaD2wY=
-k8s.io/client-go v0.28.4/go.mod h1:0VDZFpgoZfelyP5Wqu0/r/TRYcLYuJ2U1KEeoaPa1N4=
+k8s.io/api v0.29.1 h1:DAjwWX/9YT7NQD4INu49ROJuZAAAP/Ijki48GUPzxqw=
+k8s.io/api v0.29.1/go.mod h1:7Kl10vBRUXhnQQI8YR/R327zXC8eJ7887/+Ybta+RoQ=
+k8s.io/apimachinery v0.29.1 h1:KY4/E6km/wLBguvCZv8cKTeOwwOBqFNjwJIdMkMbbRc=
+k8s.io/apimachinery v0.29.1/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU=
+k8s.io/client-go v0.29.1 h1:19B/+2NGEwnFLzt0uB5kNJnfTsbV8w6TgQRz9l7ti7A=
+k8s.io/client-go v0.29.1/go.mod h1:TDG/psL9hdet0TI9mGyHJSgRkW3H9JZk2dNEUS7bRks=
 k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780=
 k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=

From 7f837694d9c3e9f308f87892a27bdcfd8e1a5bc0 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 25 Jan 2024 10:20:35 +0100
Subject: [PATCH 111/115] build(deps): bump the go-opentelemetry-io group with
 1 update (#13453)

Bumps the go-opentelemetry-io group with 1 update: [go.opentelemetry.io/collector/semconv](https://github.com/open-telemetry/opentelemetry-collector).


Updates `go.opentelemetry.io/collector/semconv` from 0.92.0 to 0.93.0
- [Release notes](https://github.com/open-telemetry/opentelemetry-collector/releases)
- [Changelog](https://github.com/open-telemetry/opentelemetry-collector/blob/main/CHANGELOG-API.md)
- [Commits](https://github.com/open-telemetry/opentelemetry-collector/compare/v0.92.0...v0.93.0)

---
updated-dependencies:
- dependency-name: go.opentelemetry.io/collector/semconv
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: go-opentelemetry-io
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index c3c7facb5..3c0d01e41 100644
--- a/go.mod
+++ b/go.mod
@@ -61,7 +61,7 @@ require (
 	github.com/vultr/govultr/v2 v2.17.2
 	go.opentelemetry.io/collector/featuregate v1.0.1
 	go.opentelemetry.io/collector/pdata v1.0.1
-	go.opentelemetry.io/collector/semconv v0.92.0
+	go.opentelemetry.io/collector/semconv v0.93.0
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0
 	go.opentelemetry.io/otel v1.22.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0
diff --git a/go.sum b/go.sum
index f221d99d2..c7850fd7b 100644
--- a/go.sum
+++ b/go.sum
@@ -805,8 +805,8 @@ go.opentelemetry.io/collector/featuregate v1.0.1 h1:ok//hLSXttBbyu4sSV1pTx1nKdr5
 go.opentelemetry.io/collector/featuregate v1.0.1/go.mod h1:QQXjP4etmJQhkQ20j4P/rapWuItYxoFozg/iIwuKnYg=
 go.opentelemetry.io/collector/pdata v1.0.1 h1:dGX2h7maA6zHbl5D3AsMnF1c3Nn+3EUftbVCLzeyNvA=
 go.opentelemetry.io/collector/pdata v1.0.1/go.mod h1:jutXeu0QOXYY8wcZ/hege+YAnSBP3+jpTqYU1+JTI5Y=
-go.opentelemetry.io/collector/semconv v0.92.0 h1:3+OGPPuVu4rtrz8qGbpbiw7eKKULj4iJaSDTV52HM40=
-go.opentelemetry.io/collector/semconv v0.92.0/go.mod h1:gZ0uzkXsN+J5NpiRcdp9xOhNGQDDui8Y62p15sKrlzo=
+go.opentelemetry.io/collector/semconv v0.93.0 h1:eBlMcVNTwYYsVdAsCVDs4wvVYs75K1xcIDpqj16PG4c=
+go.opentelemetry.io/collector/semconv v0.93.0/go.mod h1:gZ0uzkXsN+J5NpiRcdp9xOhNGQDDui8Y62p15sKrlzo=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 h1:sv9kVfal0MK0wBMCOGr+HeJm9v803BkJxGrk2au7j08=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw=
 go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y=

From 97c186e2c883f5d3c56277fd0f5b2c7d613661ef Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 25 Jan 2024 10:24:14 +0100
Subject: [PATCH 112/115] build(deps): bump actions/upload-artifact from 3.1.3
 to 4.0.0 (#13355)

Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 3.1.3 to 4.0.0.
- [Release notes](https://github.com/actions/upload-artifact/releases)
- [Commits](https://github.com/actions/upload-artifact/compare/a8a3f3ad30e3422c9c7b888a15615d19a852ae32...c7d193f32edcb7bfad88892161225aeda64e9392)

---
updated-dependencies:
- dependency-name: actions/upload-artifact
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/fuzzing.yml    | 2 +-
 .github/workflows/scorecards.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/fuzzing.yml b/.github/workflows/fuzzing.yml
index 13f04f772..599757060 100644
--- a/.github/workflows/fuzzing.yml
+++ b/.github/workflows/fuzzing.yml
@@ -21,7 +21,7 @@ jobs:
           fuzz-seconds: 600
           dry-run: false
       - name: Upload Crash
-        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
+        uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
         if: failure() && steps.build.outcome == 'success'
         with:
           name: artifacts
diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml
index 636dd1bd1..a668a4ceb 100644
--- a/.github/workflows/scorecards.yml
+++ b/.github/workflows/scorecards.yml
@@ -37,7 +37,7 @@ jobs:
       # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
       # format to the repository Actions tab.
       - name: "Upload artifact"
-        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # tag=v3.1.3
+        uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # tag=v4.0.0
         with:
           name: SARIF file
           path: results.sarif

From 1069c1f4950214f348f1bb2f6f4db7a2b1de41bf Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 25 Jan 2024 10:24:37 +0100
Subject: [PATCH 113/115] build(deps): bump bufbuild/buf-push-action (#13357)

Bumps [bufbuild/buf-push-action](https://github.com/bufbuild/buf-push-action) from 342fc4cdcf29115a01cf12a2c6dd6aac68dc51e1 to a654ff18effe4641ebea4a4ce242c49800728459.
- [Release notes](https://github.com/bufbuild/buf-push-action/releases)
- [Commits](https://github.com/bufbuild/buf-push-action/compare/342fc4cdcf29115a01cf12a2c6dd6aac68dc51e1...a654ff18effe4641ebea4a4ce242c49800728459)

---
updated-dependencies:
- dependency-name: bufbuild/buf-push-action
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/buf.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/buf.yml b/.github/workflows/buf.yml
index f6d5c9191..dc0694394 100644
--- a/.github/workflows/buf.yml
+++ b/.github/workflows/buf.yml
@@ -23,7 +23,7 @@ jobs:
         with:
           input: 'prompb'
           against: 'https://github.com/prometheus/prometheus.git#branch=main,ref=HEAD~1,subdir=prompb'
-      - uses: bufbuild/buf-push-action@342fc4cdcf29115a01cf12a2c6dd6aac68dc51e1 # v1.1.1
+      - uses: bufbuild/buf-push-action@a654ff18effe4641ebea4a4ce242c49800728459 # v1.1.1
         with:
           input: 'prompb'
           buf_token: ${{ secrets.BUF_TOKEN }}

From 74b73d1e2cd7ae97f08974234b25425c4241b1d5 Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Thu, 25 Jan 2024 10:48:49 +0000
Subject: [PATCH 114/115] Labels: Add DropMetricName function, used in PromQL
 (#13446)

This function is called very frequently when executing PromQL functions,
and we can do it much more efficiently inside Labels.

In the common case that `__name__` comes first in the labels, we simply
re-point to start at the next label, which is nearly free.

`DropMetricName` is now so cheap I removed the cache - benchmarks show
everything still goes faster.

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
---
 model/labels/labels.go              | 13 +++++++++++++
 model/labels/labels_stringlabels.go | 21 +++++++++++++++++++++
 model/labels/labels_test.go         |  6 ++++++
 promql/engine.go                    | 29 +++++------------------------
 promql/functions.go                 | 26 +++++++++++++-------------
 5 files changed, 58 insertions(+), 37 deletions(-)

diff --git a/model/labels/labels.go b/model/labels/labels.go
index bf67224bb..b6663dad2 100644
--- a/model/labels/labels.go
+++ b/model/labels/labels.go
@@ -342,6 +342,19 @@ func (ls Labels) Validate(f func(l Label) error) error {
 	return nil
 }
 
+// DropMetricName returns Labels with "__name__" removed.
+func (ls Labels) DropMetricName() Labels {
+	for i, l := range ls {
+		if l.Name == MetricName {
+			if i == 0 { // Make common case fast with no allocations.
+				return ls[1:]
+			}
+			return append(ls[:i], ls[i+1:]...)
+		}
+	}
+	return ls
+}
+
 // InternStrings calls intern on every string value inside ls, replacing them with what it returns.
 func (ls *Labels) InternStrings(intern func(string) string) {
 	for i, l := range *ls {
diff --git a/model/labels/labels_stringlabels.go b/model/labels/labels_stringlabels.go
index d79a83679..f53c3b2d0 100644
--- a/model/labels/labels_stringlabels.go
+++ b/model/labels/labels_stringlabels.go
@@ -429,6 +429,27 @@ func (ls Labels) Validate(f func(l Label) error) error {
 	return nil
 }
 
+// DropMetricName returns Labels with "__name__" removed.
+func (ls Labels) DropMetricName() Labels {
+	for i := 0; i < len(ls.data); {
+		lName, i2 := decodeString(ls.data, i)
+		size, i2 := decodeSize(ls.data, i2)
+		i2 += size
+		if lName == MetricName {
+			if i == 0 { // Make common case fast with no allocations.
+				ls.data = ls.data[i2:]
+			} else {
+				ls.data = ls.data[:i] + ls.data[i2:]
+			}
+			break
+		} else if lName[0] > MetricName[0] { // Stop looking if we've gone past.
+			break
+		}
+		i = i2
+	}
+	return ls
+}
+
 // InternStrings calls intern on every string value inside ls, replacing them with what it returns.
 func (ls *Labels) InternStrings(intern func(string) string) {
 	ls.data = intern(ls.data)
diff --git a/model/labels/labels_test.go b/model/labels/labels_test.go
index 44369278d..c497851d1 100644
--- a/model/labels/labels_test.go
+++ b/model/labels/labels_test.go
@@ -447,6 +447,12 @@ func TestLabels_Get(t *testing.T) {
 	require.Equal(t, "222", FromStrings("aaaa", "111", "bbb", "222").Get("bbb"))
 }
 
+func TestLabels_DropMetricName(t *testing.T) {
+	require.True(t, Equal(FromStrings("aaa", "111", "bbb", "222"), FromStrings("aaa", "111", "bbb", "222").DropMetricName()))
+	require.True(t, Equal(FromStrings("aaa", "111"), FromStrings(MetricName, "myname", "aaa", "111").DropMetricName()))
+	require.True(t, Equal(FromStrings("__aaa__", "111", "bbb", "222"), FromStrings("__aaa__", "111", MetricName, "myname", "bbb", "222").DropMetricName()))
+}
+
 // BenchmarkLabels_Get was written to check whether a binary search can improve the performance vs the linear search implementation
 // The results have shown that binary search would only be better when searching last labels in scenarios with more than 10 labels.
 // In the following list, `old` is the linear search while `new` is the binary search implementation (without calling sort.Search, which performs even worse here)
diff --git a/promql/engine.go b/promql/engine.go
index 8c8afd181..786b76e1f 100644
--- a/promql/engine.go
+++ b/promql/engine.go
@@ -1076,7 +1076,7 @@ type EvalNodeHelper struct {
 	Out Vector
 
 	// Caches.
-	// DropMetricName and label_*.
+	// label_*.
 	Dmn map[uint64]labels.Labels
 	// funcHistogramQuantile for classic histograms.
 	signatureToMetricWithBuckets map[string]*metricWithBuckets
@@ -1101,21 +1101,6 @@ func (enh *EvalNodeHelper) resetBuilder(lbls labels.Labels) {
 	}
 }
 
-// DropMetricName is a cached version of DropMetricName.
-func (enh *EvalNodeHelper) DropMetricName(l labels.Labels) labels.Labels {
-	if enh.Dmn == nil {
-		enh.Dmn = make(map[uint64]labels.Labels, len(enh.Out))
-	}
-	h := l.Hash()
-	ret, ok := enh.Dmn[h]
-	if ok {
-		return ret
-	}
-	ret = dropMetricName(l)
-	enh.Dmn[h] = ret
-	return ret
-}
-
 // rangeEval evaluates the given expressions, and then for each step calls
 // the given funcCall with the values computed for each expression at that
 // step. The return value is the combination into time series of all the
@@ -1492,7 +1477,7 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, annotations.Annotatio
 			// vector functions, the only change needed is to drop the
 			// metric name in the output.
 			if e.Func.Name != "last_over_time" {
-				metric = dropMetricName(metric)
+				metric = metric.DropMetricName()
 			}
 			ss := Series{
 				Metric: metric,
@@ -1624,7 +1609,7 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, annotations.Annotatio
 		mat := val.(Matrix)
 		if e.Op == parser.SUB {
 			for i := range mat {
-				mat[i].Metric = dropMetricName(mat[i].Metric)
+				mat[i].Metric = mat[i].Metric.DropMetricName()
 				for j := range mat[i].Floats {
 					mat[i].Floats[j].F = -mat[i].Floats[j].F
 				}
@@ -2370,7 +2355,7 @@ func (ev *evaluator) VectorBinop(op parser.ItemType, lhs, rhs Vector, matching *
 		}
 		metric := resultMetric(ls.Metric, rs.Metric, op, matching, enh)
 		if returnBool {
-			metric = enh.DropMetricName(metric)
+			metric = metric.DropMetricName()
 		}
 		insertedSigs, exists := matchedSigs[sig]
 		if matching.Card == parser.CardOneToOne {
@@ -2492,7 +2477,7 @@ func (ev *evaluator) VectorscalarBinop(op parser.ItemType, lhs Vector, rhs Scala
 			lhsSample.F = float
 			lhsSample.H = histogram
 			if shouldDropMetricName(op) || returnBool {
-				lhsSample.Metric = enh.DropMetricName(lhsSample.Metric)
+				lhsSample.Metric = lhsSample.Metric.DropMetricName()
 			}
 			enh.Out = append(enh.Out, lhsSample)
 		}
@@ -2500,10 +2485,6 @@ func (ev *evaluator) VectorscalarBinop(op parser.ItemType, lhs Vector, rhs Scala
 	return enh.Out
 }
 
-func dropMetricName(l labels.Labels) labels.Labels {
-	return labels.NewBuilder(l).Del(labels.MetricName).Labels()
-}
-
 // scalarBinop evaluates a binary operation between two Scalars.
 func scalarBinop(op parser.ItemType, lhs, rhs float64) float64 {
 	switch op {
diff --git a/promql/functions.go b/promql/functions.go
index 64cbd8e0f..aef73e7f2 100644
--- a/promql/functions.go
+++ b/promql/functions.go
@@ -445,7 +445,7 @@ func funcClamp(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper
 	}
 	for _, el := range vec {
 		enh.Out = append(enh.Out, Sample{
-			Metric: enh.DropMetricName(el.Metric),
+			Metric: el.Metric.DropMetricName(),
 			F:      math.Max(min, math.Min(max, el.F)),
 		})
 	}
@@ -458,7 +458,7 @@ func funcClampMax(vals []parser.Value, args parser.Expressions, enh *EvalNodeHel
 	max := vals[1].(Vector)[0].F
 	for _, el := range vec {
 		enh.Out = append(enh.Out, Sample{
-			Metric: enh.DropMetricName(el.Metric),
+			Metric: el.Metric.DropMetricName(),
 			F:      math.Min(max, el.F),
 		})
 	}
@@ -471,7 +471,7 @@ func funcClampMin(vals []parser.Value, args parser.Expressions, enh *EvalNodeHel
 	min := vals[1].(Vector)[0].F
 	for _, el := range vec {
 		enh.Out = append(enh.Out, Sample{
-			Metric: enh.DropMetricName(el.Metric),
+			Metric: el.Metric.DropMetricName(),
 			F:      math.Max(min, el.F),
 		})
 	}
@@ -493,7 +493,7 @@ func funcRound(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper
 	for _, el := range vec {
 		f := math.Floor(el.F*toNearestInverse+0.5) / toNearestInverse
 		enh.Out = append(enh.Out, Sample{
-			Metric: enh.DropMetricName(el.Metric),
+			Metric: el.Metric.DropMetricName(),
 			F:      f,
 		})
 	}
@@ -797,7 +797,7 @@ func simpleFunc(vals []parser.Value, enh *EvalNodeHelper, f func(float64) float6
 	for _, el := range vals[0].(Vector) {
 		if el.H == nil { // Process only float samples.
 			enh.Out = append(enh.Out, Sample{
-				Metric: enh.DropMetricName(el.Metric),
+				Metric: el.Metric.DropMetricName(),
 				F:      f(el.F),
 			})
 		}
@@ -943,7 +943,7 @@ func funcTimestamp(vals []parser.Value, args parser.Expressions, enh *EvalNodeHe
 	vec := vals[0].(Vector)
 	for _, el := range vec {
 		enh.Out = append(enh.Out, Sample{
-			Metric: enh.DropMetricName(el.Metric),
+			Metric: el.Metric.DropMetricName(),
 			F:      float64(el.T) / 1000,
 		})
 	}
@@ -1057,7 +1057,7 @@ func funcHistogramCount(vals []parser.Value, args parser.Expressions, enh *EvalN
 			continue
 		}
 		enh.Out = append(enh.Out, Sample{
-			Metric: enh.DropMetricName(sample.Metric),
+			Metric: sample.Metric.DropMetricName(),
 			F:      sample.H.Count,
 		})
 	}
@@ -1074,7 +1074,7 @@ func funcHistogramSum(vals []parser.Value, args parser.Expressions, enh *EvalNod
 			continue
 		}
 		enh.Out = append(enh.Out, Sample{
-			Metric: enh.DropMetricName(sample.Metric),
+			Metric: sample.Metric.DropMetricName(),
 			F:      sample.H.Sum,
 		})
 	}
@@ -1107,7 +1107,7 @@ func funcHistogramStdDev(vals []parser.Value, args parser.Expressions, enh *Eval
 		variance += cVariance
 		variance /= sample.H.Count
 		enh.Out = append(enh.Out, Sample{
-			Metric: enh.DropMetricName(sample.Metric),
+			Metric: sample.Metric.DropMetricName(),
 			F:      math.Sqrt(variance),
 		})
 	}
@@ -1140,7 +1140,7 @@ func funcHistogramStdVar(vals []parser.Value, args parser.Expressions, enh *Eval
 		variance += cVariance
 		variance /= sample.H.Count
 		enh.Out = append(enh.Out, Sample{
-			Metric: enh.DropMetricName(sample.Metric),
+			Metric: sample.Metric.DropMetricName(),
 			F:      variance,
 		})
 	}
@@ -1159,7 +1159,7 @@ func funcHistogramFraction(vals []parser.Value, args parser.Expressions, enh *Ev
 			continue
 		}
 		enh.Out = append(enh.Out, Sample{
-			Metric: enh.DropMetricName(sample.Metric),
+			Metric: sample.Metric.DropMetricName(),
 			F:      histogramFraction(lower, upper, sample.H),
 		})
 	}
@@ -1230,7 +1230,7 @@ func funcHistogramQuantile(vals []parser.Value, args parser.Expressions, enh *Ev
 		}
 
 		enh.Out = append(enh.Out, Sample{
-			Metric: enh.DropMetricName(sample.Metric),
+			Metric: sample.Metric.DropMetricName(),
 			F:      histogramQuantile(q, sample.H),
 		})
 	}
@@ -1440,7 +1440,7 @@ func dateWrapper(vals []parser.Value, enh *EvalNodeHelper, f func(time.Time) flo
 	for _, el := range vals[0].(Vector) {
 		t := time.Unix(int64(el.F), 0).UTC()
 		enh.Out = append(enh.Out, Sample{
-			Metric: enh.DropMetricName(el.Metric),
+			Metric: el.Metric.DropMetricName(),
 			F:      f(t),
 		})
 	}

From b9eab6e4b8436650559ff8ecf2a77cc31d24fc31 Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Thu, 25 Jan 2024 10:57:54 +0000
Subject: [PATCH 115/115] tsdb: simplify internal series delete function
 (#13261)

Lifting an optimisation from Agent code, `seriesHashmap.del` can use
the unique series reference, doesn't need to check Labels.
Also streamline the logic for deleting from `unique` and `conflicts` maps,
and add some comments to help the next person.

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
---
 tsdb/head.go | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/tsdb/head.go b/tsdb/head.go
index 692f7f77a..cdcd3ea56 100644
--- a/tsdb/head.go
+++ b/tsdb/head.go
@@ -1751,32 +1751,31 @@ func (m *seriesHashmap) set(hash uint64, s *memSeries) {
 	m.conflicts[hash] = append(l, s)
 }
 
-func (m *seriesHashmap) del(hash uint64, lset labels.Labels) {
+func (m *seriesHashmap) del(hash uint64, ref chunks.HeadSeriesRef) {
 	var rem []*memSeries
 	unique, found := m.unique[hash]
 	switch {
-	case !found:
+	case !found: // Supplied hash is not stored.
 		return
-	case labels.Equal(unique.lset, lset):
+	case unique.ref == ref:
 		conflicts := m.conflicts[hash]
-		if len(conflicts) == 0 {
+		if len(conflicts) == 0 { // Exactly one series with this hash was stored
 			delete(m.unique, hash)
 			return
 		}
-		rem = conflicts
-	default:
-		rem = append(rem, unique)
+		m.unique[hash] = conflicts[0] // First remaining series goes in 'unique'.
+		rem = conflicts[1:]           // Keep the rest.
+	default: // The series to delete is somewhere in 'conflicts'. Keep all the ones that don't match.
 		for _, s := range m.conflicts[hash] {
-			if !labels.Equal(s.lset, lset) {
+			if s.ref != ref {
 				rem = append(rem, s)
 			}
 		}
 	}
-	m.unique[hash] = rem[0]
-	if len(rem) == 1 {
+	if len(rem) == 0 {
 		delete(m.conflicts, hash)
 	} else {
-		m.conflicts[hash] = rem[1:]
+		m.conflicts[hash] = rem
 	}
 }
 
@@ -1891,7 +1890,7 @@ func (s *stripeSeries) gc(mint int64, minOOOMmapRef chunks.ChunkDiskMapperRef) (
 		}
 
 		deleted[storage.SeriesRef(series.ref)] = struct{}{}
-		s.hashes[hashShard].del(hash, series.lset)
+		s.hashes[hashShard].del(hash, series.ref)
 		delete(s.series[refShard], series.ref)
 		deletedForCallback[series.ref] = series.lset
 	}