Merge remote-tracking branch 'prometheus/main' into arve/close-engine

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## unreleased
 
+## 2.53.1 / 2024-07-10
+
+Fix a bug which would drop samples in remote-write if the sending flow stalled
+for longer than it takes to write one "WAL segment". How long this takes depends on the size
+of your Prometheus; as a rough guide with 10 million series it is about 2-3 minutes.
+
+* [BUGFIX] Remote-write: stop dropping samples in catch-up #14446
+
 ## 2.53.0 / 2024-06-16
 
 This release changes the default for GOGC, the Go runtime control for the trade-off between excess memory use and CPU usage. We have found that Prometheus operates with minimal additional CPU usage, but greatly reduced memory by adjusting the upstream Go default from 100 to 75.

RELEASE.md

@@ -57,7 +57,8 @@ Release cadence of first pre-releases being cut is 6 weeks.
 | v2.50          | 2024-01-16                                 | Augustin Husson (GitHub: @nexucis)          |
 | v2.51          | 2024-03-07                                 | Bryan Boreham (GitHub: @bboreham)           |
 | v2.52          | 2024-04-22                                 | Arthur Silva Sens (GitHub: @ArthurSens)     |
-| v2.53          | 2024-06-03                                 | George Krajcsovits (GitHub: @krajorama)     |
+| v2.53 LTS      | 2024-06-03                                 | George Krajcsovits (GitHub: @krajorama)     |
+| v2.54          | 2024-07-17                                 | Bryan Boreham (GitHub: @bboreham)           |
 
 If you are interested in volunteering please create a pull request against the [prometheus/prometheus](https://github.com/prometheus/prometheus) repository and propose yourself for the release series of your choice.
 

VERSION

@@ -1 +1 @@
-2.53.0
+2.53.1

cmd/prometheus/main.go

@@ -194,6 +194,9 @@ func (c *flagConfig) setFeatureListOptions(logger log.Logger) error {
 			case "extra-scrape-metrics":
 				c.scrape.ExtraMetrics = true
 				level.Info(logger).Log("msg", "Experimental additional scrape metrics enabled")
+			case "metadata-wal-records":
+				c.scrape.AppendMetadata = true
+				level.Info(logger).Log("msg", "Experimental metadata records in WAL enabled, required for remote write 2.0")
 			case "new-service-discovery-manager":
 				c.enableNewSDManager = true
 				level.Info(logger).Log("msg", "Experimental service discovery manager")
@@ -322,9 +325,15 @@ func main() {
 	a.Flag("web.enable-admin-api", "Enable API endpoints for admin control actions.").
 		Default("false").BoolVar(&cfg.web.EnableAdminAPI)
 
+	// TODO(bwplotka): Consider allowing those remote receive flags to be changed in config.
+	// See https://github.com/prometheus/prometheus/issues/14410
 	a.Flag("web.enable-remote-write-receiver", "Enable API endpoint accepting remote write requests.").
 		Default("false").BoolVar(&cfg.web.EnableRemoteWriteReceiver)
 
+	supportedRemoteWriteProtoMsgs := config.RemoteWriteProtoMsgs{config.RemoteWriteProtoMsgV1, config.RemoteWriteProtoMsgV2}
+	a.Flag("web.remote-write-receiver.accepted-protobuf-messages", fmt.Sprintf("List of the remote write protobuf messages to accept when receiving the remote writes. Supported values: %v", supportedRemoteWriteProtoMsgs.String())).
+		Default(supportedRemoteWriteProtoMsgs.Strings()...).SetValue(rwProtoMsgFlagValue(&cfg.web.AcceptRemoteWriteProtoMsgs))
+
 	a.Flag("web.console.templates", "Path to the console template directory, available at /consoles.").
 		Default("consoles").StringVar(&cfg.web.ConsoleTemplatesPath)
 
@@ -646,7 +655,7 @@ func main() {
 	var (
 		localStorage  = &readyStorage{stats: tsdb.NewDBStats()}
 		scraper       = &readyScrapeManager{}
-		remoteStorage = remote.NewStorage(log.With(logger, "component", "remote"), prometheus.DefaultRegisterer, localStorage.StartTime, localStoragePath, time.Duration(cfg.RemoteFlushDeadline), scraper)
+		remoteStorage = remote.NewStorage(log.With(logger, "component", "remote"), prometheus.DefaultRegisterer, localStorage.StartTime, localStoragePath, time.Duration(cfg.RemoteFlushDeadline), scraper, cfg.scrape.AppendMetadata)
 		fanoutStorage = storage.NewFanout(logger, localStorage, remoteStorage)
 	)
 
@@ -1776,3 +1785,39 @@ type discoveryManager interface {
 	Run() error
 	SyncCh() <-chan map[string][]*targetgroup.Group
 }
+
+// rwProtoMsgFlagParser is a custom parser for config.RemoteWriteProtoMsg enum.
+type rwProtoMsgFlagParser struct {
+	msgs *[]config.RemoteWriteProtoMsg
+}
+
+func rwProtoMsgFlagValue(msgs *[]config.RemoteWriteProtoMsg) kingpin.Value {
+	return &rwProtoMsgFlagParser{msgs: msgs}
+}
+
+// IsCumulative is used by kingpin to tell if it's an array or not.
+func (p *rwProtoMsgFlagParser) IsCumulative() bool {
+	return true
+}
+
+func (p *rwProtoMsgFlagParser) String() string {
+	ss := make([]string, 0, len(*p.msgs))
+	for _, t := range *p.msgs {
+		ss = append(ss, string(t))
+	}
+	return strings.Join(ss, ",")
+}
+
+func (p *rwProtoMsgFlagParser) Set(opt string) error {
+	t := config.RemoteWriteProtoMsg(opt)
+	if err := t.Validate(); err != nil {
+		return err
+	}
+	for _, prev := range *p.msgs {
+		if prev == t {
+			return fmt.Errorf("duplicated %v flag value, got %v already", t, *p.msgs)
+		}
+	}
+	*p.msgs = append(*p.msgs, t)
+	return nil
+}

cmd/prometheus/main_test.go

@@ -30,11 +30,13 @@ import (
 	"testing"
 	"time"
 
+	"github.com/alecthomas/kingpin/v2"
 	"github.com/go-kit/log"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 
+	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/notifier"
 	"github.com/prometheus/prometheus/rules"
@@ -499,3 +501,65 @@ func TestDocumentation(t *testing.T) {
 
 	require.Equal(t, string(expectedContent), generatedContent, "Generated content does not match documentation. Hint: run `make cli-documentation`.")
 }
+
+func TestRwProtoMsgFlagParser(t *testing.T) {
+	defaultOpts := config.RemoteWriteProtoMsgs{
+		config.RemoteWriteProtoMsgV1, config.RemoteWriteProtoMsgV2,
+	}
+
+	for _, tcase := range []struct {
+		args        []string
+		expected    []config.RemoteWriteProtoMsg
+		expectedErr error
+	}{
+		{
+			args:     nil,
+			expected: defaultOpts,
+		},
+		{
+			args:        []string{"--test-proto-msgs", "test"},
+			expectedErr: errors.New("unknown remote write protobuf message test, supported: prometheus.WriteRequest, io.prometheus.write.v2.Request"),
+		},
+		{
+			args:     []string{"--test-proto-msgs", "io.prometheus.write.v2.Request"},
+			expected: config.RemoteWriteProtoMsgs{config.RemoteWriteProtoMsgV2},
+		},
+		{
+			args: []string{
+				"--test-proto-msgs", "io.prometheus.write.v2.Request",
+				"--test-proto-msgs", "io.prometheus.write.v2.Request",
+			},
+			expectedErr: errors.New("duplicated io.prometheus.write.v2.Request flag value, got [io.prometheus.write.v2.Request] already"),
+		},
+		{
+			args: []string{
+				"--test-proto-msgs", "io.prometheus.write.v2.Request",
+				"--test-proto-msgs", "prometheus.WriteRequest",
+			},
+			expected: config.RemoteWriteProtoMsgs{config.RemoteWriteProtoMsgV2, config.RemoteWriteProtoMsgV1},
+		},
+		{
+			args: []string{
+				"--test-proto-msgs", "io.prometheus.write.v2.Request",
+				"--test-proto-msgs", "prometheus.WriteRequest",
+				"--test-proto-msgs", "io.prometheus.write.v2.Request",
+			},
+			expectedErr: errors.New("duplicated io.prometheus.write.v2.Request flag value, got [io.prometheus.write.v2.Request prometheus.WriteRequest] already"),
+		},
+	} {
+		t.Run(strings.Join(tcase.args, ","), func(t *testing.T) {
+			a := kingpin.New("test", "")
+			var opt []config.RemoteWriteProtoMsg
+			a.Flag("test-proto-msgs", "").Default(defaultOpts.Strings()...).SetValue(rwProtoMsgFlagValue(&opt))
+
+			_, err := a.Parse(tcase.args)
+			if tcase.expectedErr != nil {
+				require.Error(t, err)
+				require.Equal(t, tcase.expectedErr, err)
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, tcase.expected, opt)
+			}
+		})
+	}
+}

config/config.go

@@ -180,6 +180,7 @@ var (
 	// DefaultRemoteWriteConfig is the default remote write configuration.
 	DefaultRemoteWriteConfig = RemoteWriteConfig{
 		RemoteTimeout:    model.Duration(30 * time.Second),
+		ProtobufMessage:  RemoteWriteProtoMsgV1,
 		QueueConfig:      DefaultQueueConfig,
 		MetadataConfig:   DefaultMetadataConfig,
 		HTTPClientConfig: config.DefaultHTTPClientConfig,
@@ -279,7 +280,7 @@ func (c *Config) GetScrapeConfigs() ([]*ScrapeConfig, error) {
 
 	jobNames := map[string]string{}
 	for i, scfg := range c.ScrapeConfigs {
-		// We do these checks for library users that would not call Validate in
+		// We do these checks for library users that would not call validate in
 		// Unmarshal.
 		if err := scfg.Validate(c.GlobalConfig); err != nil {
 			return nil, err
@@ -1055,6 +1056,49 @@ func CheckTargetAddress(address model.LabelValue) error {
 	return nil
 }
 
+// RemoteWriteProtoMsg represents the known protobuf message for the remote write
+// 1.0 and 2.0 specs.
+type RemoteWriteProtoMsg string
+
+// Validate returns error if the given reference for the protobuf message is not supported.
+func (s RemoteWriteProtoMsg) Validate() error {
+	switch s {
+	case RemoteWriteProtoMsgV1, RemoteWriteProtoMsgV2:
+		return nil
+	default:
+		return fmt.Errorf("unknown remote write protobuf message %v, supported: %v", s, RemoteWriteProtoMsgs{RemoteWriteProtoMsgV1, RemoteWriteProtoMsgV2}.String())
+	}
+}
+
+type RemoteWriteProtoMsgs []RemoteWriteProtoMsg
+
+func (m RemoteWriteProtoMsgs) Strings() []string {
+	ret := make([]string, 0, len(m))
+	for _, typ := range m {
+		ret = append(ret, string(typ))
+	}
+	return ret
+}
+
+func (m RemoteWriteProtoMsgs) String() string {
+	return strings.Join(m.Strings(), ", ")
+}
+
+var (
+	// RemoteWriteProtoMsgV1 represents the deprecated `prometheus.WriteRequest` protobuf
+	// message introduced in the https://prometheus.io/docs/specs/remote_write_spec/.
+	//
+	// NOTE: This string is used for both HTTP header values and config value, so don't change
+	// this reference.
+	RemoteWriteProtoMsgV1 RemoteWriteProtoMsg = "prometheus.WriteRequest"
+	// RemoteWriteProtoMsgV2 represents the `io.prometheus.write.v2.Request` protobuf
+	// message introduced in https://prometheus.io/docs/specs/remote_write_spec_2_0/
+	//
+	// NOTE: This string is used for both HTTP header values and config value, so don't change
+	// this reference.
+	RemoteWriteProtoMsgV2 RemoteWriteProtoMsg = "io.prometheus.write.v2.Request"
+)
+
 // RemoteWriteConfig is the configuration for writing to remote storage.
 type RemoteWriteConfig struct {
 	URL                  *config.URL       `yaml:"url"`
@@ -1064,6 +1108,9 @@ type RemoteWriteConfig struct {
 	Name                 string            `yaml:"name,omitempty"`
 	SendExemplars        bool              `yaml:"send_exemplars,omitempty"`
 	SendNativeHistograms bool              `yaml:"send_native_histograms,omitempty"`
+	// ProtobufMessage specifies the protobuf message to use against the remote
+	// receiver as specified in https://prometheus.io/docs/specs/remote_write_spec_2_0/
+	ProtobufMessage RemoteWriteProtoMsg `yaml:"protobuf_message,omitempty"`
 
 	// We cannot do proper Go type embedding below as the parser will then parse
 	// values arbitrarily into the overflow maps of further-down types.
@@ -1098,6 +1145,10 @@ func (c *RemoteWriteConfig) UnmarshalYAML(unmarshal func(interface{}) error) err
 		return err
 	}
 
+	if err := c.ProtobufMessage.Validate(); err != nil {
+		return fmt.Errorf("invalid protobuf_message value: %w", err)
+	}
+
 	// The UnmarshalYAML method of HTTPClientConfig is not being called because it's not a pointer.
 	// We cannot make it a pointer as the parser panics for inlined pointer structs.
 	// Thus we just do its validation here.

config/config_test.go

@@ -108,9 +108,10 @@ var expectedConf = &Config{
 
 	RemoteWriteConfigs: []*RemoteWriteConfig{
 		{
-			URL:           mustParseURL("http://remote1/push"),
-			RemoteTimeout: model.Duration(30 * time.Second),
-			Name:          "drop_expensive",
+			URL:             mustParseURL("http://remote1/push"),
+			ProtobufMessage: RemoteWriteProtoMsgV1,
+			RemoteTimeout:   model.Duration(30 * time.Second),
+			Name:            "drop_expensive",
 			WriteRelabelConfigs: []*relabel.Config{
 				{
 					SourceLabels: model.LabelNames{"__name__"},
@@ -137,11 +138,12 @@ var expectedConf = &Config{
 			},
 		},
 		{
-			URL:            mustParseURL("http://remote2/push"),
-			RemoteTimeout:  model.Duration(30 * time.Second),
-			QueueConfig:    DefaultQueueConfig,
-			MetadataConfig: DefaultMetadataConfig,
-			Name:           "rw_tls",
+			URL:             mustParseURL("http://remote2/push"),
+			ProtobufMessage: RemoteWriteProtoMsgV2,
+			RemoteTimeout:   model.Duration(30 * time.Second),
+			QueueConfig:     DefaultQueueConfig,
+			MetadataConfig:  DefaultMetadataConfig,
+			Name:            "rw_tls",
 			HTTPClientConfig: config.HTTPClientConfig{
 				TLSConfig: config.TLSConfig{
 					CertFile: filepath.FromSlash("testdata/valid_cert_file"),
@@ -1800,6 +1802,10 @@ var expectedErrors = []struct {
 		filename: "remote_write_authorization_header.bad.yml",
 		errMsg:   `authorization header must be changed via the basic_auth, authorization, oauth2, sigv4, or azuread parameter`,
 	},
+	{
+		filename: "remote_write_wrong_msg.bad.yml",
+		errMsg:   `invalid protobuf_message value: unknown remote write protobuf message io.prometheus.writet.v2.Request, supported: prometheus.WriteRequest, io.prometheus.write.v2.Request`,
+	},
 	{
 		filename: "remote_write_url_missing.bad.yml",
 		errMsg:   `url for remote_write is empty`,

config/testdata/conf.good.yml

@@ -37,6 +37,7 @@ remote_write:
         key_file: valid_key_file
 
   - url: http://remote2/push
+    protobuf_message: io.prometheus.write.v2.Request
     name: rw_tls
     tls_config:
       cert_file: valid_cert_file

config/testdata/remote_write_wrong_msg.bad.yml

@@ -0,0 +1,3 @@
+remote_write:
+  - url: localhost:9090
+    protobuf_message: io.prometheus.writet.v2.Request # typo in 'write"

docs/command-line/prometheus.md

@@ -26,6 +26,7 @@ The Prometheus monitoring server
 | <code class="text-nowrap">--web.enable-lifecycle</code> | Enable shutdown and reload via HTTP request. | `false` |
 | <code class="text-nowrap">--web.enable-admin-api</code> | Enable API endpoints for admin control actions. | `false` |
 | <code class="text-nowrap">--web.enable-remote-write-receiver</code> | Enable API endpoint accepting remote write requests. | `false` |
+| <code class="text-nowrap">--web.remote-write-receiver.accepted-protobuf-messages</code> | List of the remote write protobuf messages to accept when receiving the remote writes. Supported values: prometheus.WriteRequest, io.prometheus.write.v2.Request | `prometheus.WriteRequest` |
 | <code class="text-nowrap">--web.console.templates</code> | Path to the console template directory, available at /consoles. | `consoles` |
 | <code class="text-nowrap">--web.console.libraries</code> | Path to the console library directory. | `console_libraries` |
 | <code class="text-nowrap">--web.page-title</code> | Document title of Prometheus instance. | `Prometheus Time Series Collection and Processing Server` |

docs/configuration/configuration.md

@@ -3575,6 +3575,17 @@ this functionality.
 # The URL of the endpoint to send samples to.
 url: <string>
 
+# protobuf message to use when writing to the remote write endpoint.
+#
+# * The `prometheus.WriteRequest` represents the message introduced in Remote Write 1.0, which
+# will be deprecated eventually.
+# * The `io.prometheus.write.v2.Request` was introduced in Remote Write 2.0 and replaces the former,
+# by improving efficiency and sending metadata, created timestamp and native histograms by default.
+#
+# Before changing this value, consult with your remote storage provider (or test) what message it supports.
+# Read more on https://prometheus.io/docs/specs/remote_write_spec_2_0/#io-prometheus-write-v2-request
+[ protobuf_message: <prometheus.WriteRequest | io.prometheus.write.v2.Request> | default = prometheus.WriteRequest ]
+
 # Timeout for requests to the remote write endpoint.
 [ remote_timeout: <duration> | default = 30s ]
 
@@ -3596,6 +3607,7 @@ write_relabel_configs:
 [ send_exemplars: <boolean> | default = false ]
 
 # Enables sending of native histograms, also known as sparse histograms, over remote write.
+# For the `io.prometheus.write.v2.Request` message, this option is noop (always true).
 [ send_native_histograms: <boolean> | default = false ]
 
 # Sets the `Authorization` header on every remote write request with the
@@ -3609,7 +3621,7 @@ basic_auth:
 # Optional `Authorization` header configuration.
 authorization:
   # Sets the authentication type.
-  [ type: <string> | default: Bearer ]
+  [ type: <string> | default = Bearer ]
   # Sets the credentials. It is mutually exclusive with
   # `credentials_file`.
   [ credentials: <secret> ]
@@ -3673,7 +3685,7 @@ tls_config:
 # contain port numbers.
 [ no_proxy: <string> ]
 # Use proxy URL indicated by environment variables (HTTP_PROXY, https_proxy, HTTPs_PROXY, https_proxy, and no_proxy)
-[ proxy_from_environment: <boolean> | default: false ]
+[ proxy_from_environment: <boolean> | default = false ]
 # Specifies headers to send to proxies during CONNECT requests.
 [ proxy_connect_header:
   [ <string>: [<secret>, ...] ] ]
@@ -3682,7 +3694,7 @@ tls_config:
 [ follow_redirects: <boolean> | default = true ]
 
 # Whether to enable HTTP2.
-[ enable_http2: <boolean> | default: true ]
+[ enable_http2: <boolean> | default = true ]
 
 # Configures the queue used to write to remote storage.
 queue_config:
@@ -3712,7 +3724,10 @@ queue_config:
   # which means that all samples are sent.
   [ sample_age_limit: <duration> | default = 0s ]
 
-# Configures the sending of series metadata to remote storage.
+# Configures the sending of series metadata to remote storage
+# if the `prometheus.WriteRequest` message was chosen. When
+# `io.prometheus.write.v2.Request` is used, metadata is always sent.
+#
 # Metadata configuration is subject to change at any point
 # or be removed in future releases.
 metadata_config:

docs/feature_flags.md

@@ -224,3 +224,13 @@ When the `concurrent-rule-eval` feature flag is enabled, rules without any depen
 This has the potential to improve rule group evaluation latency and resource utilization at the expense of adding more concurrent query load.
 
 The number of concurrent rule evaluations can be configured with `--rules.max-concurrent-rule-evals`, which is set to `4` by default.
+
+## Metadata WAL Records
+
+`--enable-feature=metadata-wal-records`
+
+When enabled, Prometheus will store metadata in-memory and keep track of
+metadata changes as WAL records on a per-series basis.
+
+This must be used if
+you are also using remote write 2.0 as it will only gather metadata from the WAL.

docs/querying/api.md

@@ -25,8 +25,10 @@ Other non-`2xx` codes may be returned for errors occurring before the API
 endpoint is reached.
 
 An array of warnings may be returned if there are errors that do
-not inhibit the request execution. All of the data that was successfully
-collected will be returned in the data field.
+not inhibit the request execution. An additional array of info-level
+annotations may be returned for potential query issues that may or may
+not be false positives. All of the data that was successfully collected
+will be returned in the data field.
 
 The JSON response envelope format is as follows:
 
@@ -40,9 +42,11 @@ The JSON response envelope format is as follows:
   "errorType": "<string>",
   "error": "<string>",
 
-  // Only if there were warnings while executing the request.
+  // Only set if there were warnings while executing the request.
   // There will still be data in the data field.
-  "warnings": ["<string>"]
+  "warnings": ["<string>"],
+  // Only set if there were info-level annnotations while executing the request.
+  "infos": ["<string>"]
 }
 ```
 
@@ -256,7 +260,7 @@ URL query parameters:
   series to return. At least one `match[]` argument must be provided.
 - `start=<rfc3339 | unix_timestamp>`: Start timestamp.
 - `end=<rfc3339 | unix_timestamp>`: End timestamp.
-- `limit=<number>`: Maximum number of returned series. Optional.
+- `limit=<number>`: Maximum number of returned series. Optional. 0 means disabled.
 
 You can URL-encode these parameters directly in the request body by using the `POST` method and
 `Content-Type: application/x-www-form-urlencoded` header. This is useful when specifying a large
@@ -307,7 +311,7 @@ URL query parameters:
 - `end=<rfc3339 | unix_timestamp>`: End timestamp. Optional.
 - `match[]=<series_selector>`: Repeated series selector argument that selects the
   series from which to read the label names. Optional.
-- `limit=<number>`: Maximum number of returned series. Optional.
+- `limit=<number>`: Maximum number of returned series. Optional. 0 means disabled.
 
 
 The `data` section of the JSON response is a list of string label names.
@@ -358,7 +362,7 @@ URL query parameters:
 - `end=<rfc3339 | unix_timestamp>`: End timestamp. Optional.
 - `match[]=<series_selector>`: Repeated series selector argument that selects the
   series from which to read the label values. Optional.
-- `limit=<number>`: Maximum number of returned series. Optional.
+- `limit=<number>`: Maximum number of returned series. Optional. 0 means disabled.
 
 
 The `data` section of the JSON response is a list of string label values.
@@ -452,7 +456,7 @@ raw numbers.
 
 The keys `"histogram"` and `"histograms"` only show up if the experimental
 native histograms are present in the response. Their placeholder `<histogram>`
-is explained in detail in its own section below. 
+is explained in detail in its own section below.
 
 ### Range vectors
 
@@ -470,7 +474,7 @@ Range vectors are returned as result type `matrix`. The corresponding
 ]
 ```
 
-Each series could have the `"values"` key, or the `"histograms"` key, or both. 
+Each series could have the `"values"` key, or the `"histograms"` key, or both.
 For a given timestamp, there will only be one sample of either float or histogram type.
 
 Series are returned sorted by `metric`. Functions such as [`sort`](functions.md#sort)
@@ -689,7 +693,8 @@ URL query parameters:
 - `rule_name[]=<string>`: only return rules with the given rule name. If the parameter is repeated, rules with any of the provided names are returned. If we've filtered out all the rules of a group, the group is not returned. When the parameter is absent or empty, no filtering is done.
 - `rule_group[]=<string>`: only return rules with the given rule group name. If the parameter is repeated, rules with any of the provided rule group names are returned. When the parameter is absent or empty, no filtering is done.
 - `file[]=<string>`: only return rules with the given filepath. If the parameter is repeated, rules with any of the provided filepaths are returned. When the parameter is absent or empty, no filtering is done.
-- `exclude_alerts=<bool>`: only return rules, do not return active alerts.
+- `exclude_alerts=<bool>`: only return rules, do not return active alerts. 
+- `match[]=<label_selector>`: only return rules that have configured labels that satisfy the label selectors. If the parameter is repeated, rules that match any of the sets of label selectors are returned. Note that matching is on the labels in the definition of each rule, not on the values after template expansion (for alerting rules). Optional.
 
 ```json
 $ curl http://localhost:9090/api/v1/rules
@@ -1309,7 +1314,7 @@ endpoint is `/api/v1/write`. Find more details [here](../storage.md#overview).
 
 ## OTLP Receiver
 
-Prometheus can be configured as a receiver for the OTLP Metrics protocol. This 
+Prometheus can be configured as a receiver for the OTLP Metrics protocol. This
 is not considered an efficient way of ingesting samples. Use it
 with caution for specific low-volume use cases. It is not suitable for
 replacing the ingestion via scraping.

docs/querying/basics.md

@@ -81,6 +81,16 @@ Examples:
     0x8f
     -Inf
     NaN
+	
+
+As of version 2.54, float literals can also be represented using the syntax of time durations, where the time duration is converted into a float value corresponding to the number of seconds the time duration represents. This is an experimental feature and might still change.
+
+Examples:
+
+    1s # Equivalent to 1.0
+    2m # Equivalent to 120.0
+    1ms # Equivalent to 0.001
+ 
 
 ## Time series selectors
 
@@ -224,6 +234,15 @@ Here are some examples of valid time durations:
     5m
     10s
 
+
+As of version 2.54, time durations can also be represented using the syntax of float literals, implying the number of seconds of the time duration. This is an experimental feature and might still change.
+
+Examples:
+
+    1.0 # Equivalent to 1s
+    0.001 # Equivalent to 1ms
+    120 # Equivalent to 2m
+
 ### Offset modifier
 
 The `offset` modifier allows changing the time offset for individual

docs/querying/examples.md

@@ -95,3 +95,13 @@ Assuming this metric contains one time series per running instance, you could
 count the number of running instances per application like this:
 
     count by (app) (instance_cpu_time_ns)
+
+If we are exploring some metrics for their labels, to e.g. be able to aggregate
+over some of them, we could use the following:
+
+    limitk(10, app_foo_metric_bar)
+
+Alternatively, if we wanted the returned timeseries to be more evenly sampled,
+we could use the following to get approximately 10% of them:
+
+    limit_ratio(0.1, app_foo_metric_bar)

docs/querying/functions.md

@@ -98,8 +98,9 @@ vector.
 clamps the sample values of all elements in `v` to have a lower limit of `min` and an upper limit of `max`.
 
 Special cases:
-- Return an empty vector if `min > max`
-- Return `NaN` if `min` or `max` is `NaN`
+
+* Return an empty vector if `min > max`
+* Return `NaN` if `min` or `max` is `NaN`
 
 ## `clamp_max()`
 
@@ -349,8 +350,8 @@ a histogram.
 
 Buckets of classic histograms are cumulative. Therefore, the following should always be the case:
 
-- The counts in the buckets are monotonically increasing (strictly non-decreasing).
-- A lack of observations between the upper limits of two consecutive buckets results in equal counts
+* The counts in the buckets are monotonically increasing (strictly non-decreasing).
+* A lack of observations between the upper limits of two consecutive buckets results in equal counts
 in those two buckets.
 
 However, floating point precision issues (e.g. small discrepancies introduced by computing of buckets
@@ -692,21 +693,21 @@ ignore histogram samples.
 
 The trigonometric functions work in radians:
 
-- `acos(v instant-vector)`: calculates the arccosine of all elements in `v` ([special cases](https://pkg.go.dev/math#Acos)).
-- `acosh(v instant-vector)`: calculates the inverse hyperbolic cosine of all elements in `v` ([special cases](https://pkg.go.dev/math#Acosh)).
-- `asin(v instant-vector)`: calculates the arcsine of all elements in `v` ([special cases](https://pkg.go.dev/math#Asin)).
-- `asinh(v instant-vector)`: calculates the inverse hyperbolic sine of all elements in `v` ([special cases](https://pkg.go.dev/math#Asinh)).
-- `atan(v instant-vector)`: calculates the arctangent of all elements in `v` ([special cases](https://pkg.go.dev/math#Atan)).
-- `atanh(v instant-vector)`: calculates the inverse hyperbolic tangent of all elements in `v` ([special cases](https://pkg.go.dev/math#Atanh)).
-- `cos(v instant-vector)`: calculates the cosine of all elements in `v` ([special cases](https://pkg.go.dev/math#Cos)).
-- `cosh(v instant-vector)`: calculates the hyperbolic cosine of all elements in `v` ([special cases](https://pkg.go.dev/math#Cosh)).
-- `sin(v instant-vector)`: calculates the sine of all elements in `v` ([special cases](https://pkg.go.dev/math#Sin)).
-- `sinh(v instant-vector)`: calculates the hyperbolic sine of all elements in `v` ([special cases](https://pkg.go.dev/math#Sinh)).
-- `tan(v instant-vector)`: calculates the tangent of all elements in `v` ([special cases](https://pkg.go.dev/math#Tan)).
-- `tanh(v instant-vector)`: calculates the hyperbolic tangent of all elements in `v` ([special cases](https://pkg.go.dev/math#Tanh)).
+* `acos(v instant-vector)`: calculates the arccosine of all elements in `v` ([special cases](https://pkg.go.dev/math#Acos)).
+* `acosh(v instant-vector)`: calculates the inverse hyperbolic cosine of all elements in `v` ([special cases](https://pkg.go.dev/math#Acosh)).
+* `asin(v instant-vector)`: calculates the arcsine of all elements in `v` ([special cases](https://pkg.go.dev/math#Asin)).
+* `asinh(v instant-vector)`: calculates the inverse hyperbolic sine of all elements in `v` ([special cases](https://pkg.go.dev/math#Asinh)).
+* `atan(v instant-vector)`: calculates the arctangent of all elements in `v` ([special cases](https://pkg.go.dev/math#Atan)).
+* `atanh(v instant-vector)`: calculates the inverse hyperbolic tangent of all elements in `v` ([special cases](https://pkg.go.dev/math#Atanh)).
+* `cos(v instant-vector)`: calculates the cosine of all elements in `v` ([special cases](https://pkg.go.dev/math#Cos)).
+* `cosh(v instant-vector)`: calculates the hyperbolic cosine of all elements in `v` ([special cases](https://pkg.go.dev/math#Cosh)).
+* `sin(v instant-vector)`: calculates the sine of all elements in `v` ([special cases](https://pkg.go.dev/math#Sin)).
+* `sinh(v instant-vector)`: calculates the hyperbolic sine of all elements in `v` ([special cases](https://pkg.go.dev/math#Sinh)).
+* `tan(v instant-vector)`: calculates the tangent of all elements in `v` ([special cases](https://pkg.go.dev/math#Tan)).
+* `tanh(v instant-vector)`: calculates the hyperbolic tangent of all elements in `v` ([special cases](https://pkg.go.dev/math#Tanh)).
 
 The following are useful for converting between degrees and radians:
 
-- `deg(v instant-vector)`: converts radians to degrees for all elements in `v`.
-- `pi()`: returns pi.
-- `rad(v instant-vector)`: converts degrees to radians for all elements in `v`.
+* `deg(v instant-vector)`: converts radians to degrees for all elements in `v`.
+* `pi()`: returns pi.
+* `rad(v instant-vector)`: converts degrees to radians for all elements in `v`.

docs/querying/operators.md

@@ -230,6 +230,8 @@ vector of fewer elements with aggregated values:
 * `bottomk` (smallest k elements by sample value)
 * `topk` (largest k elements by sample value)
 * `quantile` (calculate φ-quantile (0 ≤ φ ≤ 1) over dimensions)
+* `limitk` (sample n elements)
+* `limit_ratio` (sample elements with approximately 𝑟 ratio if `𝑟 > 0`, and the complement of such samples if `𝑟 = -(1.0 - 𝑟)`)
 
 These operators can either be used to aggregate over **all** label dimensions
 or preserve distinct dimensions by including a `without` or `by` clause. These
@@ -249,8 +251,8 @@ all other labels are preserved in the output. `by` does the opposite and drops
 labels that are not listed in the `by` clause, even if their label values are
 identical between all elements of the vector.
 
-`parameter` is only required for `count_values`, `quantile`, `topk` and
-`bottomk`.
+`parameter` is only required for `count_values`, `quantile`, `topk`,
+`bottomk`, `limitk` and `limit_ratio`.
 
 `count_values` outputs one time series per unique sample value. Each series has
 an additional label. The name of that label is given by the aggregation
@@ -261,11 +263,16 @@ time series is the number of times that sample value was present.
 the input samples, including the original labels, are returned in the result
 vector. `by` and `without` are only used to bucket the input vector.
 
+`limitk` and `limit_ratio` also return a subset of the input samples,
+including the original labels in the result vector, these are experimental
+operators that must be enabled with `--enable-feature=promql-experimental-functions`.
+
 `quantile` calculates the φ-quantile, the value that ranks at number φ*N among
 the N metric values of the dimensions aggregated over. φ is provided as the
 aggregation parameter. For example, `quantile(0.5, ...)` calculates the median,
 `quantile(0.95, ...)` the 95th percentile. For φ = `NaN`, `NaN` is returned. For φ < 0, `-Inf` is returned. For φ > 1, `+Inf` is returned.
 
+
 Example:
 
 If the metric `http_requests_total` had time series that fan out by
@@ -291,6 +298,33 @@ To get the 5 largest HTTP requests counts across all instances we could write:
 
     topk(5, http_requests_total)
 
+To sample 10 timeseries, for example to inspect labels and their values, we
+could write:
+
+    limitk(10, http_requests_total)
+
+To deterministically sample approximately 10% of timeseries we could write:
+
+    limit_ratio(0.1, http_requests_total)
+
+Given that `limit_ratio()` implements a deterministic sampling algorithm (based
+on labels' hash), you can get the _complement_ of the above samples, i.e.
+approximately 90%, but precisely those not returned by `limit_ratio(0.1, ...)`
+with:
+
+    limit_ratio(-0.9, http_requests_total)
+
+You can also use this feature to e.g. verify that `avg()` is a representative
+aggregation for your samples' values, by checking that the difference between
+averaging two samples' subsets is "small" when compared to the standard
+deviation.
+
+    abs(
+      avg(limit_ratio(0.5, http_requests_total))
+      -
+      avg(limit_ratio(-0.5, http_requests_total))
+    ) <= bool stddev(http_requests_total)
+
 ## Binary operator precedence
 
 The following list shows the precedence of binary operators in Prometheus, from

documentation/examples/remote_storage/example_write_adapter/README.md

@@ -7,6 +7,7 @@ To use it:
 
 ```
 go build
+
 ./example_write_adapter
 ```
 
@@ -15,10 +16,19 @@ go build
 ```yaml
 remote_write:
   - url: "http://localhost:1234/receive"
+    protobuf_message: "io.prometheus.write.v2.Request"
 ```
 
-Then start Prometheus:
+or for deprecated Remote Write 1.0 message:
+
+```yaml
+remote_write:
+  - url: "http://localhost:1234/receive"
+    protobuf_message: "prometheus.WriteRequest"
+```
+
+Then start Prometheus (in separate terminal):
 
 ```
-./prometheus
+./prometheus --enable-feature=metadata-wal-records
 ```

documentation/examples/remote_storage/example_write_adapter/server.go

@@ -18,44 +18,103 @@ import (
 	"log"
 	"net/http"
 
-	"github.com/prometheus/common/model"
-
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/prompb"
+	writev2 "github.com/prometheus/prometheus/prompb/io/prometheus/write/v2"
 	"github.com/prometheus/prometheus/storage/remote"
 )
 
 func main() {
 	http.HandleFunc("/receive", func(w http.ResponseWriter, r *http.Request) {
-		req, err := remote.DecodeWriteRequest(r.Body)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusBadRequest)
+		enc := r.Header.Get("Content-Encoding")
+		if enc == "" {
+			http.Error(w, "missing Content-Encoding header", http.StatusUnsupportedMediaType)
+			return
+		}
+		if enc != "snappy" {
+			http.Error(w, "unknown encoding, only snappy supported", http.StatusUnsupportedMediaType)
 			return
 		}
 
-		for _, ts := range req.Timeseries {
-			m := make(model.Metric, len(ts.Labels))
-			for _, l := range ts.Labels {
-				m[model.LabelName(l.Name)] = model.LabelValue(l.Value)
-			}
-			fmt.Println(m)
+		contentType := r.Header.Get("Content-Type")
+		if contentType == "" {
+			http.Error(w, "missing Content-Type header", http.StatusUnsupportedMediaType)
+		}
 
-			for _, s := range ts.Samples {
-				fmt.Printf("\tSample:  %f %d\n", s.Value, s.Timestamp)
-			}
+		defer func() { _ = r.Body.Close() }()
 
-			for _, e := range ts.Exemplars {
-				m := make(model.Metric, len(e.Labels))
-				for _, l := range e.Labels {
-					m[model.LabelName(l.Name)] = model.LabelValue(l.Value)
-				}
-				fmt.Printf("\tExemplar:  %+v %f %d\n", m, e.Value, e.Timestamp)
+		// Very simplistic content parsing, see
+		// storage/remote/write_handler.go#WriteHandler.ServeHTTP for production example.
+		switch contentType {
+		case "application/x-protobuf", "application/x-protobuf;proto=prometheus.WriteRequest":
+			req, err := remote.DecodeWriteRequest(r.Body)
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusBadRequest)
+				return
 			}
-
-			for _, hp := range ts.Histograms {
-				h := remote.HistogramProtoToHistogram(hp)
-				fmt.Printf("\tHistogram:  %s\n", h.String())
+			printV1(req)
+		case "application/x-protobuf;proto=io.prometheus.write.v2.Request":
+			req, err := remote.DecodeWriteV2Request(r.Body)
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusBadRequest)
+				return
 			}
+			printV2(req)
+		default:
+			msg := fmt.Sprintf("Unknown remote write content type: %s", contentType)
+			fmt.Println(msg)
+			http.Error(w, msg, http.StatusBadRequest)
 		}
 	})
-
 	log.Fatal(http.ListenAndServe(":1234", nil))
 }
+
+func printV1(req *prompb.WriteRequest) {
+	b := labels.NewScratchBuilder(0)
+	for _, ts := range req.Timeseries {
+		fmt.Println(ts.ToLabels(&b, nil))
+
+		for _, s := range ts.Samples {
+			fmt.Printf("\tSample:  %f %d\n", s.Value, s.Timestamp)
+		}
+		for _, ep := range ts.Exemplars {
+			e := ep.ToExemplar(&b, nil)
+			fmt.Printf("\tExemplar:  %+v %f %d\n", e.Labels, e.Value, ep.Timestamp)
+		}
+		for _, hp := range ts.Histograms {
+			if hp.IsFloatHistogram() {
+				h := hp.ToFloatHistogram()
+				fmt.Printf("\tHistogram:  %s\n", h.String())
+				continue
+			}
+			h := hp.ToIntHistogram()
+			fmt.Printf("\tHistogram:  %s\n", h.String())
+		}
+	}
+}
+
+func printV2(req *writev2.Request) {
+	b := labels.NewScratchBuilder(0)
+	for _, ts := range req.Timeseries {
+		l := ts.ToLabels(&b, req.Symbols)
+		m := ts.ToMetadata(req.Symbols)
+		fmt.Println(l, m)
+
+		for _, s := range ts.Samples {
+			fmt.Printf("\tSample:  %f %d\n", s.Value, s.Timestamp)
+		}
+		for _, ep := range ts.Exemplars {
+			e := ep.ToExemplar(&b, req.Symbols)
+			fmt.Printf("\tExemplar:  %+v %f %d\n", e.Labels, e.Value, ep.Timestamp)
+		}
+		for _, hp := range ts.Histograms {
+			if hp.IsFloatHistogram() {
+				h := hp.ToFloatHistogram()
+				fmt.Printf("\tHistogram:  %s\n", h.String())
+				continue
+			}
+			h := hp.ToIntHistogram()
+			fmt.Printf("\tHistogram:  %s\n", h.String())
+		}
+	}
+}

documentation/examples/remote_storage/go.mod

@@ -17,10 +17,10 @@ require (
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.6.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
 	github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect
-	github.com/aws/aws-sdk-go v1.51.25 // indirect
+	github.com/aws/aws-sdk-go v1.53.16 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
@@ -31,8 +31,7 @@ require (
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd // indirect
-	github.com/hashicorp/go-version v1.6.0 // indirect
+	github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
@@ -49,13 +48,12 @@ require (
 	github.com/prometheus/common/sigv4 v0.1.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
-	go.opentelemetry.io/collector/featuregate v1.5.0 // indirect
-	go.opentelemetry.io/collector/pdata v1.5.0 // indirect
-	go.opentelemetry.io/collector/semconv v0.98.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.50.0 // indirect
-	go.opentelemetry.io/otel v1.25.0 // indirect
-	go.opentelemetry.io/otel/metric v1.25.0 // indirect
-	go.opentelemetry.io/otel/trace v1.25.0 // indirect
+	go.opentelemetry.io/collector/pdata v1.8.0 // indirect
+	go.opentelemetry.io/collector/semconv v0.101.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 // indirect
+	go.opentelemetry.io/otel v1.27.0 // indirect
+	go.opentelemetry.io/otel/metric v1.27.0 // indirect
+	go.opentelemetry.io/otel/trace v1.27.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.24.0 // indirect
@@ -64,8 +62,8 @@ require (
 	golang.org/x/sys v0.21.0 // indirect
 	golang.org/x/text v0.16.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect
-	google.golang.org/grpc v1.63.2 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect
+	google.golang.org/grpc v1.64.0 // indirect
 	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
@@ -82,4 +80,10 @@ exclude (
 	cloud.google.com/go v0.34.0
 	cloud.google.com/go v0.65.0
 	cloud.google.com/go v0.82.0
+
+	// Fixing ambiguous import: found package google.golang.org/genproto/googleapis/api/annotations in multiple modules.
+	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1
 )
+
+// TODO(bwplotka): Move to main branch commit or perhaps released version.
+replace github.com/prometheus/prometheus => github.com/prometheus/prometheus v0.53.1-0.20240704074759-c137febfcf8c

documentation/examples/remote_storage/go.sum

@@ -2,10 +2,10 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqb
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2 h1:FDif4R1+UUR+00q6wquyX90K7A8dN+R5E8GEadoP7sU=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2/go.mod h1:aiYBYui4BJ/BJCAIKs92XiPyQfTaBWqvHujDwKb6CBU=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2/go.mod h1:yInRyqWXAuaPrgI7p70+lDDgh3mlBohis29jGMISnmc=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.6.0 h1:ui3YNbxfW7J3tTFIZMH6LIGRjCngp+J+nIFlnizfNTE=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.6.0/go.mod h1:gZmgV+qBqygoznvqo2J9oKZAFziqhLZ2xE/WVUmzkHA=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.6.0 h1:sUFnFjzDUie80h24I7mrKtwCKgLY9L8h5Tp2x9+TWqk=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.6.0/go.mod h1:52JbnQTp15qg5mRkMBHwp0j0ZFwHJ42Sx3zVV5RE9p0=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 h1:LkHbJbgF3YyvC53aqYGR+wWQDn2Rdp9AQdGndf9QvY4=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0/go.mod h1:QyiQdW4f4/BIfB8ZutZ2s+28RAgfa/pT+zS++ZHyM1I=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0 h1:bXwSugBiSbgtz7rOtbfGf+woewp4f06orW9OP5BjHLA=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0/go.mod h1:Y/HgrePTmGy9HjdSGTqZNa+apUpTVIEVKXJyARP2lrk=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
@@ -26,8 +26,8 @@ github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9/go.mod h1:OMCwj8V
 github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
 github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
 github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
-github.com/aws/aws-sdk-go v1.51.25 h1:DjTT8mtmsachhV6yrXR8+yhnG6120dazr720nopRsls=
-github.com/aws/aws-sdk-go v1.51.25/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
+github.com/aws/aws-sdk-go v1.53.16 h1:8oZjKQO/ml1WLUZw5hvF7pvYjPf8o9f57Wldoy/q9Qc=
+github.com/aws/aws-sdk-go v1.53.16/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 h1:6df1vn4bBlDDo4tARvBm7l6KA9iVMnE3NWizDeWSrps=
 github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3/go.mod h1:CIWtjkly68+yqLPbvwwR/fjNJA/idrtULjZWh2v1ys0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -37,8 +37,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ=
-github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM=
+github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50 h1:DBmgJDC9dTfkVyGgipamEh2BpGYxScCH1TOF1LL1cXc=
+github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50/go.mod h1:5e1+Vvlzido69INQaVO6d87Qn543Xr6nooe9Kz7oBFM=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -46,14 +46,14 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE=
 github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA=
-github.com/digitalocean/godo v1.113.0 h1:CLtCxlP4wDAjKIQ+Hshht/UNbgAp8/J/XBH1ZtDCF9Y=
-github.com/digitalocean/godo v1.113.0/go.mod h1:Z2mTP848Vi3IXXl5YbPekUgr4j4tOePomA+OE1Ag98w=
+github.com/digitalocean/godo v1.117.0 h1:WVlTe09melDYTd7VCVyvHcNWbgB+uI1O115+5LOtdSw=
+github.com/digitalocean/godo v1.117.0/go.mod h1:Vk0vpCot2HOAJwc5WE8wljZGtJ3ZtWIc8MQ8rF38sdo=
 github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0=
 github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
 github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
 github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
-github.com/docker/docker v26.0.1+incompatible h1:t39Hm6lpXuXtgkF0dm1t9a5HkbUfdGy6XbWexmGr+hA=
-github.com/docker/docker v26.0.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v26.1.3+incompatible h1:lLCzRbrVZrljpVNobJu1J2FHk8V0s4BawoZippkc+xo=
+github.com/docker/docker v26.1.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
@@ -68,8 +68,8 @@ github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU
 github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew=
 github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb h1:IT4JYU7k4ikYg1SCxNI1/Tieq/NFvh6dzLdgi7eu0tM=
 github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb/go.mod h1:bH6Xx7IW64qjjJq8M2u4dxNaBiDfKK+z/3eGDpXEQhc=
-github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
-github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
+github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
+github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
@@ -95,8 +95,8 @@ github.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdX
 github.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHauHj0V9Lhc506VOpw4=
 github.com/go-openapi/swag v0.22.9 h1:XX2DssF+mQKM2DHsbgZK74y/zj4mo9I99+89xUmuZCE=
 github.com/go-openapi/swag v0.22.9/go.mod h1:3/OXnFfnMAwBD099SwYRk7GD3xOrr1iL7d/XNLXVVwE=
-github.com/go-resty/resty/v2 v2.12.0 h1:rsVL8P90LFvkUYq/V5BTVe203WfRIU4gvcf+yfzJzGA=
-github.com/go-resty/resty/v2 v2.12.0/go.mod h1:o0yGPrkS3lOe1+eFajk6kBW8ScXzwU3hD69/gt2yB/0=
+github.com/go-resty/resty/v2 v2.13.1 h1:x+LHXBI2nMB1vqndymf26quycC4aggYJ7DECYbiz03g=
+github.com/go-resty/resty/v2 v2.13.1/go.mod h1:GznXlLxkq6Nh4sU59rPmUw3VtgpO3aS96ORAI6Q7d+0=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-zookeeper/zk v1.0.3 h1:7M2kwOsc//9VeeFiPtf+uSJlVpU66x9Ba5+8XK7/TDg=
 github.com/go-zookeeper/zk v1.0.3/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
@@ -135,40 +135,38 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gophercloud/gophercloud v1.11.0 h1:ls0O747DIq1D8SUHc7r2vI8BFbMLeLFuENaAIfEx7OM=
-github.com/gophercloud/gophercloud v1.11.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM=
+github.com/gophercloud/gophercloud v1.12.0 h1:Jrz16vPAL93l80q16fp8NplrTCp93y7rZh2P3Q4Yq7g=
+github.com/gophercloud/gophercloud v1.12.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM=
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd h1:PpuIBO5P3e9hpqBD0O/HjhShYuM6XE0i/lbE6J94kww=
-github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A=
-github.com/hashicorp/consul/api v1.28.2 h1:mXfkRHrpHN4YY3RqL09nXU1eHKLNiuAN4kHvDQ16k/8=
-github.com/hashicorp/consul/api v1.28.2/go.mod h1:KyzqzgMEya+IZPcD65YFoOVAgPpbfERu4I/tzG6/ueE=
+github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc h1:GN2Lv3MGO7AS6PrRoT6yV5+wkrOpcszoIsO4+4ds248=
+github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk=
+github.com/hashicorp/consul/api v1.29.1 h1:UEwOjYJrd3lG1x5w7HxDRMGiAUPrb3f103EoeKuuEcc=
+github.com/hashicorp/consul/api v1.29.1/go.mod h1:lumfRkY/coLuqMICkI7Fh3ylMG31mQSRZyef2c5YvJI=
 github.com/hashicorp/cronexpr v1.1.2 h1:wG/ZYIKT+RT3QkOdgYc+xsKWVRgnxJ1OJtjjy84fJ9A=
 github.com/hashicorp/cronexpr v1.1.2/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1Efb3g1PWA4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
-github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c=
-github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
+github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
+github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
 github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc=
 github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
-github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA=
-github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
+github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
+github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
 github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
 github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
-github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
-github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4=
 github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
-github.com/hashicorp/nomad/api v0.0.0-20240418183417-ea5f2f6748c7 h1:pjE59CS2C9Bg+Xby0ROrnZSSBWtKwx3Sf9gqsrvIFSA=
-github.com/hashicorp/nomad/api v0.0.0-20240418183417-ea5f2f6748c7/go.mod h1:svtxn6QnrQ69P23VvIWMR34tg3vmwLz4UdUzm1dSCgE=
+github.com/hashicorp/nomad/api v0.0.0-20240604134157-e73d8bb1140d h1:KHq+mAzWSkumj4PDoXc5VZbycPGcmYu8tohgVLQ6SIc=
+github.com/hashicorp/nomad/api v0.0.0-20240604134157-e73d8bb1140d/go.mod h1:svtxn6QnrQ69P23VvIWMR34tg3vmwLz4UdUzm1dSCgE=
 github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY=
 github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4=
-github.com/hetznercloud/hcloud-go/v2 v2.7.2 h1:UlE7n1GQZacCfyjv9tDVUN7HZfOXErPIfM/M039u9A0=
-github.com/hetznercloud/hcloud-go/v2 v2.7.2/go.mod h1:49tIV+pXRJTUC7fbFZ03s45LKqSQdOPP5y91eOnJo/k=
+github.com/hetznercloud/hcloud-go/v2 v2.9.0 h1:s0N6R7Zoi2DPfMtUF5o9VeUBzTtHVY6MIkHOQnfu/AY=
+github.com/hetznercloud/hcloud-go/v2 v2.9.0/go.mod h1:qtW/TuU7Bs16ibXl/ktJarWqU2LwHr7eGlwoilHxtgg=
 github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
 github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
 github.com/influxdata/influxdb v1.11.5 h1:+em5VOl6lhAZubXj5o6SobCwvrRs3XDlBx/MUI4schI=
@@ -208,14 +206,14 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/linode/linodego v1.32.0 h1:OmZzB3iON6uu84VtLFf64uKmAQqJJarvmsVguroioPI=
-github.com/linode/linodego v1.32.0/go.mod h1:y8GDP9uLVH4jTB9qyrgw79qfKdYJmNCGUOJmfuiOcmI=
+github.com/linode/linodego v1.35.0 h1:rIhUeCHBLEDlkoRnOTwzSGzljQ3ksXwLxacmXnrV+Do=
+github.com/linode/linodego v1.35.0/go.mod h1:JxuhOEAMfSxun6RU5/MgTKH2GGTmFrhKRj3wL1NFin0=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
-github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
-github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs=
 github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk=
@@ -243,8 +241,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=
 github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
-github.com/ovh/go-ovh v1.4.3 h1:Gs3V823zwTFpzgGLZNI6ILS4rmxZgJwJCz54Er9LwD0=
-github.com/ovh/go-ovh v1.4.3/go.mod h1:AkPXVtgwB6xlKblMjRKJJmjRp+ogrE7fz2lVgcQY8SY=
+github.com/ovh/go-ovh v1.5.1 h1:P8O+7H+NQuFK9P/j4sFW5C0fvSS2DnHYGPwdVCp45wI=
+github.com/ovh/go-ovh v1.5.1/go.mod h1:cTVDnl94z4tl8pP1uZ/8jlVxntjSIf09bNcQ5TJSC7c=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -279,12 +277,12 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
 github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
-github.com/prometheus/prometheus v0.52.1 h1:BrQ29YG+mzdGh8DgHPirHbeMGNqtL+INe0rqg7ttBJ4=
-github.com/prometheus/prometheus v0.52.1/go.mod h1:3z74cVsmVH0iXOR5QBjB7Pa6A0KJeEAK5A6UsmAFb1g=
+github.com/prometheus/prometheus v0.53.1-0.20240704074759-c137febfcf8c h1:6GEA48LnonkYZhQ654v7QTIP5uBTbCEVm49oIhif5lc=
+github.com/prometheus/prometheus v0.53.1-0.20240704074759-c137febfcf8c/go.mod h1:FcNs5wa7M9yV8IlxlB/05s5oy9vULUIlu/tZsviRIT8=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
-github.com/scaleway/scaleway-sdk-go v1.0.0-beta.26 h1:F+GIVtGqCFxPxO46ujf8cEOP574MBoRm3gNbPXECbxs=
-github.com/scaleway/scaleway-sdk-go v1.0.0-beta.26/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg=
+github.com/scaleway/scaleway-sdk-go v1.0.0-beta.27 h1:yGAraK1uUjlhSXgNMIy8o/J4LFNcy7yeipBqt9N9mVg=
+github.com/scaleway/scaleway-sdk-go v1.0.0-beta.27/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
@@ -306,20 +304,18 @@ github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8
 github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-go.opentelemetry.io/collector/featuregate v1.5.0 h1:uK8qnYQKz1TMkK+FDTFsywg/EybW/gbnOUaPNUkRznM=
-go.opentelemetry.io/collector/featuregate v1.5.0/go.mod h1:w7nUODKxEi3FLf1HslCiE6YWtMtOOrMnSwsDam8Mg9w=
-go.opentelemetry.io/collector/pdata v1.5.0 h1:1fKTmUpr0xCOhP/B0VEvtz7bYPQ45luQ8XFyA07j8LE=
-go.opentelemetry.io/collector/pdata v1.5.0/go.mod h1:TYj8aKRWZyT/KuKQXKyqSEvK/GV+slFaDMEI+Ke64Yw=
-go.opentelemetry.io/collector/semconv v0.98.0 h1:zO4L4TmlxXoYu8UgPeYElGY19BW7wPjM+quL5CzoOoY=
-go.opentelemetry.io/collector/semconv v0.98.0/go.mod h1:8ElcRZ8Cdw5JnvhTOQOdYizkJaQ10Z2fS+R6djOnj6A=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.50.0 h1:cEPbyTSEHlQR89XVlyo78gqluF8Y3oMeBkXGWzQsfXY=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.50.0/go.mod h1:DKdbWcT4GH1D0Y3Sqt/PFXt2naRKDWtU+eE6oLdFNA8=
-go.opentelemetry.io/otel v1.25.0 h1:gldB5FfhRl7OJQbUHt/8s0a7cE8fbsPAtdpRaApKy4k=
-go.opentelemetry.io/otel v1.25.0/go.mod h1:Wa2ds5NOXEMkCmUou1WA7ZBfLTHWIsp034OVD7AO+Vg=
-go.opentelemetry.io/otel/metric v1.25.0 h1:LUKbS7ArpFL/I2jJHdJcqMGxkRdxpPHE0VU/D4NuEwA=
-go.opentelemetry.io/otel/metric v1.25.0/go.mod h1:rkDLUSd2lC5lq2dFNrX9LGAbINP5B7WBkC78RXCpH5s=
-go.opentelemetry.io/otel/trace v1.25.0 h1:tqukZGLwQYRIFtSQM2u2+yfMVTgGVeqRLPUYx1Dq6RM=
-go.opentelemetry.io/otel/trace v1.25.0/go.mod h1:hCCs70XM/ljO+BeQkyFnbK28SBIJ/Emuha+ccrCRT7I=
+go.opentelemetry.io/collector/pdata v1.8.0 h1:d/QQgZxB4Y+d3mqLVh2ozvzujUhloD3P/fk7X+In764=
+go.opentelemetry.io/collector/pdata v1.8.0/go.mod h1:/W7clu0wFC4WSRp94Ucn6Vm36Wkrt+tmtlDb1aiNZCY=
+go.opentelemetry.io/collector/semconv v0.101.0 h1:tOe9iTe9dDCnvz/bqgfNRr4w80kXG8505tQJ5h5v08Q=
+go.opentelemetry.io/collector/semconv v0.101.0/go.mod h1:8ElcRZ8Cdw5JnvhTOQOdYizkJaQ10Z2fS+R6djOnj6A=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 h1:9l89oX4ba9kHbBol3Xin3leYJ+252h0zszDtBwyKe2A=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0/go.mod h1:XLZfZboOJWHNKUv7eH0inh0E9VV6eWDFB/9yJyTLPp0=
+go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg=
+go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ=
+go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik=
+go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak=
+go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw=
+go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4=
 go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
 go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
@@ -336,8 +332,8 @@ golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRj
 golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
-golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0=
+golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -397,21 +393,20 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
-golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
+golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA=
+golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY=
-google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be h1:Zz7rLWqp0ApfsR/l7+zSHhY3PMiH2xqgxlfYfAfNpoU=
-google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be/go.mod h1:dvdCTIoAGbkWbcIKBniID56/7XHTt6WfxXNMxuziJ+w=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be h1:LG9vZxsWGOmUKieR8wPAUR3u3MpnYFQZROPIMaXh7/A=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
-google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM=
-google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA=
+google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw=
+google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0=
+google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY=
+google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

model/labels/regexp.go

@@ -28,7 +28,7 @@ const (
 	maxSetMatches = 256
 
 	// The minimum number of alternate values a regex should have to trigger
-	// the optimization done by optimizeEqualStringMatchers() and so use a map
+	// the optimization done by optimizeEqualOrPrefixStringMatchers() and so use a map
 	// to match values instead of iterating over a list. This value has
 	// been computed running BenchmarkOptimizeEqualStringMatchers.
 	minEqualMultiStringMatcherMapThreshold = 16
@@ -337,7 +337,7 @@ func optimizeAlternatingLiterals(s string) (StringMatcher, []string) {
 		return nil, nil
 	}
 
-	multiMatcher := newEqualMultiStringMatcher(true, estimatedAlternates)
+	multiMatcher := newEqualMultiStringMatcher(true, estimatedAlternates, 0, 0)
 
 	for end := strings.IndexByte(s, '|'); end > -1; end = strings.IndexByte(s, '|') {
 		// Split the string into the next literal and the remainder
@@ -412,7 +412,7 @@ func stringMatcherFromRegexp(re *syntax.Regexp) StringMatcher {
 	clearBeginEndText(re)
 
 	m := stringMatcherFromRegexpInternal(re)
-	m = optimizeEqualStringMatchers(m, minEqualMultiStringMatcherMapThreshold)
+	m = optimizeEqualOrPrefixStringMatchers(m, minEqualMultiStringMatcherMapThreshold)
 
 	return m
 }
@@ -732,17 +732,20 @@ func (m *equalStringMatcher) Matches(s string) bool {
 type multiStringMatcherBuilder interface {
 	StringMatcher
 	add(s string)
+	addPrefix(prefix string, prefixCaseSensitive bool, matcher StringMatcher)
 	setMatches() []string
 }
 
-func newEqualMultiStringMatcher(caseSensitive bool, estimatedSize int) multiStringMatcherBuilder {
+func newEqualMultiStringMatcher(caseSensitive bool, estimatedSize, estimatedPrefixes, minPrefixLength int) multiStringMatcherBuilder {
 	// If the estimated size is low enough, it's faster to use a slice instead of a map.
-	if estimatedSize < minEqualMultiStringMatcherMapThreshold {
+	if estimatedSize < minEqualMultiStringMatcherMapThreshold && estimatedPrefixes == 0 {
 		return &equalMultiStringSliceMatcher{caseSensitive: caseSensitive, values: make([]string, 0, estimatedSize)}
 	}
 
 	return &equalMultiStringMapMatcher{
 		values:        make(map[string]struct{}, estimatedSize),
+		prefixes:      make(map[string][]StringMatcher, estimatedPrefixes),
+		minPrefixLen:  minPrefixLength,
 		caseSensitive: caseSensitive,
 	}
 }
@@ -758,6 +761,10 @@ func (m *equalMultiStringSliceMatcher) add(s string) {
 	m.values = append(m.values, s)
 }
 
+func (m *equalMultiStringSliceMatcher) addPrefix(_ string, _ bool, _ StringMatcher) {
+	panic("not implemented")
+}
+
 func (m *equalMultiStringSliceMatcher) setMatches() []string {
 	return m.values
 }
@@ -779,12 +786,17 @@ func (m *equalMultiStringSliceMatcher) Matches(s string) bool {
 	return false
 }
 
-// equalMultiStringMapMatcher matches a string exactly against a map of valid values.
+// equalMultiStringMapMatcher matches a string exactly against a map of valid values
+// or against a set of prefix matchers.
 type equalMultiStringMapMatcher struct {
 	// values contains values to match a string against. If the matching is case insensitive,
 	// the values here must be lowercase.
 	values map[string]struct{}
-
+	// prefixes maps strings, all of length minPrefixLen, to sets of matchers to check the rest of the string.
+	// If the matching is case insensitive, prefixes are all lowercase.
+	prefixes map[string][]StringMatcher
+	// minPrefixLen can be zero, meaning there are no prefix matchers.
+	minPrefixLen  int
 	caseSensitive bool
 }
 
@@ -796,8 +808,27 @@ func (m *equalMultiStringMapMatcher) add(s string) {
 	m.values[s] = struct{}{}
 }
 
+func (m *equalMultiStringMapMatcher) addPrefix(prefix string, prefixCaseSensitive bool, matcher StringMatcher) {
+	if m.minPrefixLen == 0 {
+		panic("addPrefix called when no prefix length defined")
+	}
+	if len(prefix) < m.minPrefixLen {
+		panic("addPrefix called with a too short prefix")
+	}
+	if m.caseSensitive != prefixCaseSensitive {
+		panic("addPrefix called with a prefix whose case sensitivity is different than the expected one")
+	}
+
+	s := prefix[:m.minPrefixLen]
+	if !m.caseSensitive {
+		s = strings.ToLower(s)
+	}
+
+	m.prefixes[s] = append(m.prefixes[s], matcher)
+}
+
 func (m *equalMultiStringMapMatcher) setMatches() []string {
-	if len(m.values) >= maxSetMatches {
+	if len(m.values) >= maxSetMatches || len(m.prefixes) > 0 {
 		return nil
 	}
 
@@ -813,8 +844,17 @@ func (m *equalMultiStringMapMatcher) Matches(s string) bool {
 		s = toNormalisedLower(s)
 	}
 
-	_, ok := m.values[s]
-	return ok
+	if _, ok := m.values[s]; ok {
+		return true
+	}
+	if m.minPrefixLen > 0 && len(s) >= m.minPrefixLen {
+		for _, matcher := range m.prefixes[s[:m.minPrefixLen]] {
+			if matcher.Matches(s) {
+				return true
+			}
+		}
+	}
+	return false
 }
 
 // toNormalisedLower normalise the input string using "Unicode Normalization Form D" and then convert
@@ -897,20 +937,24 @@ func (m trueMatcher) Matches(_ string) bool {
 	return true
 }
 
-// optimizeEqualStringMatchers optimize a specific case where all matchers are made by an
-// alternation (orStringMatcher) of strings checked for equality (equalStringMatcher). In
-// this specific case, when we have many strings to match against we can use a map instead
+// optimizeEqualOrPrefixStringMatchers optimize a specific case where all matchers are made by an
+// alternation (orStringMatcher) of strings checked for equality (equalStringMatcher) or
+// with a literal prefix (literalPrefixSensitiveStringMatcher or literalPrefixInsensitiveStringMatcher).
+//
+// In this specific case, when we have many strings to match against we can use a map instead
 // of iterating over the list of strings.
-func optimizeEqualStringMatchers(input StringMatcher, threshold int) StringMatcher {
+func optimizeEqualOrPrefixStringMatchers(input StringMatcher, threshold int) StringMatcher {
 	var (
 		caseSensitive    bool
 		caseSensitiveSet bool
 		numValues        int
+		numPrefixes      int
+		minPrefixLength  int
 	)
 
 	// Analyse the input StringMatcher to count the number of occurrences
 	// and ensure all of them have the same case sensitivity.
-	analyseCallback := func(matcher *equalStringMatcher) bool {
+	analyseEqualMatcherCallback := func(matcher *equalStringMatcher) bool {
 		// Ensure we don't have mixed case sensitivity.
 		if caseSensitiveSet && caseSensitive != matcher.caseSensitive {
 			return false
@@ -923,34 +967,55 @@ func optimizeEqualStringMatchers(input StringMatcher, threshold int) StringMatch
 		return true
 	}
 
-	if !findEqualStringMatchers(input, analyseCallback) {
+	analysePrefixMatcherCallback := func(prefix string, prefixCaseSensitive bool, matcher StringMatcher) bool {
+		// Ensure we don't have mixed case sensitivity.
+		if caseSensitiveSet && caseSensitive != prefixCaseSensitive {
+			return false
+		} else if !caseSensitiveSet {
+			caseSensitive = prefixCaseSensitive
+			caseSensitiveSet = true
+		}
+		if numPrefixes == 0 || len(prefix) < minPrefixLength {
+			minPrefixLength = len(prefix)
+		}
+
+		numPrefixes++
+		return true
+	}
+
+	if !findEqualOrPrefixStringMatchers(input, analyseEqualMatcherCallback, analysePrefixMatcherCallback) {
 		return input
 	}
 
-	// If the number of values found is less than the threshold, then we should skip the optimization.
-	if numValues < threshold {
+	// If the number of values and prefixes found is less than the threshold, then we should skip the optimization.
+	if (numValues + numPrefixes) < threshold {
 		return input
 	}
 
 	// Parse again the input StringMatcher to extract all values and storing them.
 	// We can skip the case sensitivity check because we've already checked it and
 	// if the code reach this point then it means all matchers have the same case sensitivity.
-	multiMatcher := newEqualMultiStringMatcher(caseSensitive, numValues)
+	multiMatcher := newEqualMultiStringMatcher(caseSensitive, numValues, numPrefixes, minPrefixLength)
 
 	// Ignore the return value because we already iterated over the input StringMatcher
 	// and it was all good.
-	findEqualStringMatchers(input, func(matcher *equalStringMatcher) bool {
+	findEqualOrPrefixStringMatchers(input, func(matcher *equalStringMatcher) bool {
 		multiMatcher.add(matcher.s)
 		return true
+	}, func(prefix string, prefixCaseSensitive bool, matcher StringMatcher) bool {
+		multiMatcher.addPrefix(prefix, caseSensitive, matcher)
+		return true
 	})
 
 	return multiMatcher
 }
 
-// findEqualStringMatchers analyze the input StringMatcher and calls the callback for each
-// equalStringMatcher found. Returns true if and only if the input StringMatcher is *only*
-// composed by an alternation of equalStringMatcher.
-func findEqualStringMatchers(input StringMatcher, callback func(matcher *equalStringMatcher) bool) bool {
+// findEqualOrPrefixStringMatchers analyze the input StringMatcher and calls the equalMatcherCallback for each
+// equalStringMatcher found, and prefixMatcherCallback for each literalPrefixSensitiveStringMatcher and literalPrefixInsensitiveStringMatcher found.
+//
+// Returns true if and only if the input StringMatcher is *only* composed by an alternation of equalStringMatcher and/or
+// literal prefix matcher. Returns false if prefixMatcherCallback is nil and a literal prefix matcher is encountered.
+func findEqualOrPrefixStringMatchers(input StringMatcher, equalMatcherCallback func(matcher *equalStringMatcher) bool, prefixMatcherCallback func(prefix string, prefixCaseSensitive bool, matcher StringMatcher) bool) bool {
 	orInput, ok := input.(orStringMatcher)
 	if !ok {
 		return false
@@ -959,17 +1024,27 @@ func findEqualStringMatchers(input StringMatcher, callback func(matcher *equalSt
 	for _, m := range orInput {
 		switch casted := m.(type) {
 		case orStringMatcher:
-			if !findEqualStringMatchers(m, callback) {
+			if !findEqualOrPrefixStringMatchers(m, equalMatcherCallback, prefixMatcherCallback) {
 				return false
 			}
 
 		case *equalStringMatcher:
-			if !callback(casted) {
+			if !equalMatcherCallback(casted) {
+				return false
+			}
+
+		case *literalPrefixSensitiveStringMatcher:
+			if prefixMatcherCallback == nil || !prefixMatcherCallback(casted.prefix, true, casted) {
+				return false
+			}
+
+		case *literalPrefixInsensitiveStringMatcher:
+			if prefixMatcherCallback == nil || !prefixMatcherCallback(casted.prefix, false, casted) {
 				return false
 			}
 
 		default:
-			// It's not an equal string matcher, so we have to stop searching
+			// It's not an equal or prefix string matcher, so we have to stop searching
 			// cause this optimization can't be applied.
 			return false
 		}

model/labels/regexp_test.go

@@ -71,6 +71,8 @@ var (
 		// A long case insensitive alternation.
 		"(?i:(zQPbMkNO|NNSPdvMi|iWuuSoAl|qbvKMimS|IecrXtPa|seTckYqt|NxnyHkgB|fIDlOgKb|UhlWIygH|OtNoJxHG|cUTkFVIV|mTgFIHjr|jQkoIDtE|PPMKxRXl|AwMfwVkQ|CQyMrTQJ|BzrqxVSi|nTpcWuhF|PertdywG|ZZDgCtXN|WWdDPyyE|uVtNQsKk|BdeCHvPZ|wshRnFlH|aOUIitIp|RxZeCdXT|CFZMslCj|AVBZRDxl|IzIGCnhw|ythYuWiz|oztXVXhl|VbLkwqQx|qvaUgyVC|VawUjPWC|ecloYJuj|boCLTdSU|uPrKeAZx|hrMWLWBq|JOnUNHRM|rYnujkPq|dDEdZhIj|DRrfvugG|yEGfDxVV|YMYdJWuP|PHUQZNWM|AmKNrLis|zTxndVfn|FPsHoJnc|EIulZTua|KlAPhdzg|ScHJJCLt|NtTfMzME|eMCwuFdo|SEpJVJbR|cdhXZeCx|sAVtBwRh|kVFEVcMI|jzJrxraA|tGLHTell|NNWoeSaw|DcOKSetX|UXZAJyka|THpMphDP|rizheevl|kDCBRidd|pCZZRqyu|pSygkitl|SwZGkAaW|wILOrfNX|QkwVOerj|kHOMxPDr|EwOVycJv|AJvtzQFS|yEOjKYYB|LizIINLL|JBRSsfcG|YPiUqqNl|IsdEbvee|MjEpGcBm|OxXZVgEQ|xClXGuxa|UzRCGFEb|buJbvfvA|IPZQxRet|oFYShsMc|oBHffuHO|bzzKrcBR|KAjzrGCl|IPUsAVls|OGMUMbIU|gyDccHuR|bjlalnDd|ZLWjeMna|fdsuIlxQ|dVXtiomV|XxedTjNg|XWMHlNoA|nnyqArQX|opfkWGhb|wYtnhdYb))",
 		"(?i:(AAAAAAAAAAAAAAAAAAAAAAAA|BBBBBBBBBBBBBBBBBBBBBBBB|cccccccccccccccccccccccC|ſſſſſſſſſſſſſſſſſſſſſſſſS|SSSSSSSSSSSSSSSSSSSSSSSSſ))",
+		// A short case insensitive alternation where each entry ends with ".*".
+		"(?i:(zQPbMkNO.*|NNSPdvMi.*|iWuuSoAl.*))",
 		// A long case insensitive alternation where each entry ends with ".*".
 		"(?i:(zQPbMkNO.*|NNSPdvMi.*|iWuuSoAl.*|qbvKMimS.*|IecrXtPa.*|seTckYqt.*|NxnyHkgB.*|fIDlOgKb.*|UhlWIygH.*|OtNoJxHG.*|cUTkFVIV.*|mTgFIHjr.*|jQkoIDtE.*|PPMKxRXl.*|AwMfwVkQ.*|CQyMrTQJ.*|BzrqxVSi.*|nTpcWuhF.*|PertdywG.*|ZZDgCtXN.*|WWdDPyyE.*|uVtNQsKk.*|BdeCHvPZ.*|wshRnFlH.*|aOUIitIp.*|RxZeCdXT.*|CFZMslCj.*|AVBZRDxl.*|IzIGCnhw.*|ythYuWiz.*|oztXVXhl.*|VbLkwqQx.*|qvaUgyVC.*|VawUjPWC.*|ecloYJuj.*|boCLTdSU.*|uPrKeAZx.*|hrMWLWBq.*|JOnUNHRM.*|rYnujkPq.*|dDEdZhIj.*|DRrfvugG.*|yEGfDxVV.*|YMYdJWuP.*|PHUQZNWM.*|AmKNrLis.*|zTxndVfn.*|FPsHoJnc.*|EIulZTua.*|KlAPhdzg.*|ScHJJCLt.*|NtTfMzME.*|eMCwuFdo.*|SEpJVJbR.*|cdhXZeCx.*|sAVtBwRh.*|kVFEVcMI.*|jzJrxraA.*|tGLHTell.*|NNWoeSaw.*|DcOKSetX.*|UXZAJyka.*|THpMphDP.*|rizheevl.*|kDCBRidd.*|pCZZRqyu.*|pSygkitl.*|SwZGkAaW.*|wILOrfNX.*|QkwVOerj.*|kHOMxPDr.*|EwOVycJv.*|AJvtzQFS.*|yEOjKYYB.*|LizIINLL.*|JBRSsfcG.*|YPiUqqNl.*|IsdEbvee.*|MjEpGcBm.*|OxXZVgEQ.*|xClXGuxa.*|UzRCGFEb.*|buJbvfvA.*|IPZQxRet.*|oFYShsMc.*|oBHffuHO.*|bzzKrcBR.*|KAjzrGCl.*|IPUsAVls.*|OGMUMbIU.*|gyDccHuR.*|bjlalnDd.*|ZLWjeMna.*|fdsuIlxQ.*|dVXtiomV.*|XxedTjNg.*|XWMHlNoA.*|nnyqArQX.*|opfkWGhb.*|wYtnhdYb.*))",
 		// A long case insensitive alternation where each entry starts with ".*".
@@ -686,7 +688,15 @@ func randStrings(randGenerator *rand.Rand, many, length int) []string {
 	return out
 }
 
-func TestOptimizeEqualStringMatchers(t *testing.T) {
+func randStringsWithSuffix(randGenerator *rand.Rand, many, length int, suffix string) []string {
+	out := randStrings(randGenerator, many, length)
+	for i := range out {
+		out[i] += suffix
+	}
+	return out
+}
+
+func TestOptimizeEqualOrPrefixStringMatchers(t *testing.T) {
 	tests := map[string]struct {
 		input                 StringMatcher
 		expectedValues        []string
@@ -767,7 +777,7 @@ func TestOptimizeEqualStringMatchers(t *testing.T) {
 
 	for testName, testData := range tests {
 		t.Run(testName, func(t *testing.T) {
-			actualMatcher := optimizeEqualStringMatchers(testData.input, 0)
+			actualMatcher := optimizeEqualOrPrefixStringMatchers(testData.input, 0)
 
 			if testData.expectedValues == nil {
 				require.IsType(t, testData.input, actualMatcher)
@@ -782,10 +792,12 @@ func TestOptimizeEqualStringMatchers(t *testing.T) {
 
 func TestNewEqualMultiStringMatcher(t *testing.T) {
 	tests := map[string]struct {
-		values             []string
-		caseSensitive      bool
-		expectedValuesMap  map[string]struct{}
-		expectedValuesList []string
+		values                []string
+		caseSensitivePrefixes []*literalPrefixSensitiveStringMatcher
+		caseSensitive         bool
+		expectedValuesMap     map[string]struct{}
+		expectedPrefixesMap   map[string][]StringMatcher
+		expectedValuesList    []string
 	}{
 		"few case sensitive values": {
 			values:             []string{"a", "B"},
@@ -797,27 +809,47 @@ func TestNewEqualMultiStringMatcher(t *testing.T) {
 			caseSensitive:      false,
 			expectedValuesList: []string{"a", "B"},
 		},
+		"few case sensitive values and prefixes": {
+			values:                []string{"a"},
+			caseSensitivePrefixes: []*literalPrefixSensitiveStringMatcher{{prefix: "B", right: anyStringWithoutNewlineMatcher{}}},
+			caseSensitive:         true,
+			expectedValuesMap:     map[string]struct{}{"a": {}},
+			expectedPrefixesMap:   map[string][]StringMatcher{"B": {&literalPrefixSensitiveStringMatcher{prefix: "B", right: anyStringWithoutNewlineMatcher{}}}},
+		},
 		"many case sensitive values": {
-			values:            []string{"a", "B", "c", "D", "e", "F", "g", "H", "i", "L", "m", "N", "o", "P", "q", "r"},
-			caseSensitive:     true,
-			expectedValuesMap: map[string]struct{}{"a": {}, "B": {}, "c": {}, "D": {}, "e": {}, "F": {}, "g": {}, "H": {}, "i": {}, "L": {}, "m": {}, "N": {}, "o": {}, "P": {}, "q": {}, "r": {}},
+			values:              []string{"a", "B", "c", "D", "e", "F", "g", "H", "i", "L", "m", "N", "o", "P", "q", "r"},
+			caseSensitive:       true,
+			expectedValuesMap:   map[string]struct{}{"a": {}, "B": {}, "c": {}, "D": {}, "e": {}, "F": {}, "g": {}, "H": {}, "i": {}, "L": {}, "m": {}, "N": {}, "o": {}, "P": {}, "q": {}, "r": {}},
+			expectedPrefixesMap: map[string][]StringMatcher{},
 		},
 		"many case insensitive values": {
-			values:            []string{"a", "B", "c", "D", "e", "F", "g", "H", "i", "L", "m", "N", "o", "P", "q", "r"},
-			caseSensitive:     false,
-			expectedValuesMap: map[string]struct{}{"a": {}, "b": {}, "c": {}, "d": {}, "e": {}, "f": {}, "g": {}, "h": {}, "i": {}, "l": {}, "m": {}, "n": {}, "o": {}, "p": {}, "q": {}, "r": {}},
+			values:              []string{"a", "B", "c", "D", "e", "F", "g", "H", "i", "L", "m", "N", "o", "P", "q", "r"},
+			caseSensitive:       false,
+			expectedValuesMap:   map[string]struct{}{"a": {}, "b": {}, "c": {}, "d": {}, "e": {}, "f": {}, "g": {}, "h": {}, "i": {}, "l": {}, "m": {}, "n": {}, "o": {}, "p": {}, "q": {}, "r": {}},
+			expectedPrefixesMap: map[string][]StringMatcher{},
 		},
 	}
 
 	for testName, testData := range tests {
 		t.Run(testName, func(t *testing.T) {
-			matcher := newEqualMultiStringMatcher(testData.caseSensitive, len(testData.values))
+			// To keep this test simple, we always assume a min prefix length of 1.
+			minPrefixLength := 0
+			if len(testData.caseSensitivePrefixes) > 0 {
+				minPrefixLength = 1
+			}
+
+			matcher := newEqualMultiStringMatcher(testData.caseSensitive, len(testData.values), len(testData.caseSensitivePrefixes), minPrefixLength)
 			for _, v := range testData.values {
 				matcher.add(v)
 			}
-			if testData.expectedValuesMap != nil {
+			for _, p := range testData.caseSensitivePrefixes {
+				matcher.addPrefix(p.prefix, true, p)
+			}
+
+			if testData.expectedValuesMap != nil || testData.expectedPrefixesMap != nil {
 				require.IsType(t, &equalMultiStringMapMatcher{}, matcher)
 				require.Equal(t, testData.expectedValuesMap, matcher.(*equalMultiStringMapMatcher).values)
+				require.Equal(t, testData.expectedPrefixesMap, matcher.(*equalMultiStringMapMatcher).prefixes)
 				require.Equal(t, testData.caseSensitive, matcher.(*equalMultiStringMapMatcher).caseSensitive)
 			}
 			if testData.expectedValuesList != nil {
@@ -829,9 +861,32 @@ func TestNewEqualMultiStringMatcher(t *testing.T) {
 	}
 }
 
+func TestEqualMultiStringMapMatcher_addPrefix(t *testing.T) {
+	t.Run("should panic if the matcher is case sensitive but the prefix is not case sensitive", func(t *testing.T) {
+		matcher := newEqualMultiStringMatcher(true, 0, 1, 1)
+
+		require.Panics(t, func() {
+			matcher.addPrefix("a", false, &literalPrefixInsensitiveStringMatcher{
+				prefix: "a",
+			})
+		})
+	})
+
+	t.Run("should panic if the matcher is not case sensitive but the prefix is case sensitive", func(t *testing.T) {
+		matcher := newEqualMultiStringMatcher(false, 0, 1, 1)
+
+		require.Panics(t, func() {
+			matcher.addPrefix("a", true, &literalPrefixSensitiveStringMatcher{
+				prefix: "a",
+			})
+		})
+	})
+}
+
 func TestEqualMultiStringMatcher_Matches(t *testing.T) {
 	tests := map[string]struct {
 		values             []string
+		prefixes           []StringMatcher
 		caseSensitive      bool
 		expectedMatches    []string
 		expectedNotMatches []string
@@ -848,6 +903,24 @@ func TestEqualMultiStringMatcher_Matches(t *testing.T) {
 			expectedMatches:    []string{"a", "A", "b", "B"},
 			expectedNotMatches: []string{"c", "C"},
 		},
+		"few case sensitive prefixes": {
+			prefixes: []StringMatcher{
+				&literalPrefixSensitiveStringMatcher{prefix: "a", right: anyStringWithoutNewlineMatcher{}},
+				&literalPrefixSensitiveStringMatcher{prefix: "B", right: anyStringWithoutNewlineMatcher{}},
+			},
+			caseSensitive:      true,
+			expectedMatches:    []string{"a", "aX", "B", "BX"},
+			expectedNotMatches: []string{"A", "b"},
+		},
+		"few case insensitive prefixes": {
+			prefixes: []StringMatcher{
+				&literalPrefixInsensitiveStringMatcher{prefix: "a", right: anyStringWithoutNewlineMatcher{}},
+				&literalPrefixInsensitiveStringMatcher{prefix: "B", right: anyStringWithoutNewlineMatcher{}},
+			},
+			caseSensitive:      false,
+			expectedMatches:    []string{"a", "aX", "A", "AX", "b", "bX", "B", "BX"},
+			expectedNotMatches: []string{"c", "cX", "C", "CX"},
+		},
 		"many case sensitive values": {
 			values:             []string{"a", "B", "c", "D", "e", "F", "g", "H", "i", "L", "m", "N", "o", "P", "q", "r"},
 			caseSensitive:      true,
@@ -860,14 +933,37 @@ func TestEqualMultiStringMatcher_Matches(t *testing.T) {
 			expectedMatches:    []string{"a", "A", "b", "B"},
 			expectedNotMatches: []string{"x", "X"},
 		},
+		"mixed values and prefixes": {
+			values:             []string{"a"},
+			prefixes:           []StringMatcher{&literalPrefixSensitiveStringMatcher{prefix: "B", right: anyStringWithoutNewlineMatcher{}}},
+			caseSensitive:      true,
+			expectedMatches:    []string{"a", "B", "BX"},
+			expectedNotMatches: []string{"aX", "A", "b", "bX"},
+		},
 	}
 
 	for testName, testData := range tests {
 		t.Run(testName, func(t *testing.T) {
-			matcher := newEqualMultiStringMatcher(testData.caseSensitive, len(testData.values))
+			// To keep this test simple, we always assume a min prefix length of 1.
+			minPrefixLength := 0
+			if len(testData.prefixes) > 0 {
+				minPrefixLength = 1
+			}
+
+			matcher := newEqualMultiStringMatcher(testData.caseSensitive, len(testData.values), len(testData.prefixes), minPrefixLength)
 			for _, v := range testData.values {
 				matcher.add(v)
 			}
+			for _, p := range testData.prefixes {
+				switch m := p.(type) {
+				case *literalPrefixSensitiveStringMatcher:
+					matcher.addPrefix(m.prefix, true, p)
+				case *literalPrefixInsensitiveStringMatcher:
+					matcher.addPrefix(m.prefix, false, p)
+				default:
+					panic("Unexpected type in test case")
+				}
+			}
 
 			for _, v := range testData.expectedMatches {
 				require.True(t, matcher.Matches(v), "value: %s", v)
@@ -879,29 +975,33 @@ func TestEqualMultiStringMatcher_Matches(t *testing.T) {
 	}
 }
 
-func TestFindEqualStringMatchers(t *testing.T) {
+func TestFindEqualOrPrefixStringMatchers(t *testing.T) {
 	type match struct {
 		s             string
 		caseSensitive bool
 	}
 
-	// Utility to call findEqualStringMatchers() and collect all callback invocations.
-	findEqualStringMatchersAndCollectMatches := func(input StringMatcher) (matches []match, ok bool) {
-		ok = findEqualStringMatchers(input, func(matcher *equalStringMatcher) bool {
+	// Utility to call findEqualOrPrefixStringMatchers() and collect all callback invocations.
+	findEqualOrPrefixStringMatchersAndCollectMatches := func(input StringMatcher) (matches []match, ok bool) {
+		ok = findEqualOrPrefixStringMatchers(input, func(matcher *equalStringMatcher) bool {
 			matches = append(matches, match{matcher.s, matcher.caseSensitive})
 			return true
+		}, func(prefix string, prefixCaseSensitive bool, right StringMatcher) bool {
+			matches = append(matches, match{prefix, prefixCaseSensitive})
+			return true
 		})
+
 		return
 	}
 
 	t.Run("empty matcher", func(t *testing.T) {
-		actualMatches, actualOk := findEqualStringMatchersAndCollectMatches(emptyStringMatcher{})
+		actualMatches, actualOk := findEqualOrPrefixStringMatchersAndCollectMatches(emptyStringMatcher{})
 		require.False(t, actualOk)
 		require.Empty(t, actualMatches)
 	})
 
 	t.Run("concat of literal matchers (case sensitive)", func(t *testing.T) {
-		actualMatches, actualOk := findEqualStringMatchersAndCollectMatches(
+		actualMatches, actualOk := findEqualOrPrefixStringMatchersAndCollectMatches(
 			orStringMatcher{
 				&equalStringMatcher{s: "test-1", caseSensitive: true},
 				&equalStringMatcher{s: "test-2", caseSensitive: true},
@@ -913,7 +1013,7 @@ func TestFindEqualStringMatchers(t *testing.T) {
 	})
 
 	t.Run("concat of literal matchers (case insensitive)", func(t *testing.T) {
-		actualMatches, actualOk := findEqualStringMatchersAndCollectMatches(
+		actualMatches, actualOk := findEqualOrPrefixStringMatchersAndCollectMatches(
 			orStringMatcher{
 				&equalStringMatcher{s: "test-1", caseSensitive: false},
 				&equalStringMatcher{s: "test-2", caseSensitive: false},
@@ -925,7 +1025,7 @@ func TestFindEqualStringMatchers(t *testing.T) {
 	})
 
 	t.Run("concat of literal matchers (mixed case)", func(t *testing.T) {
-		actualMatches, actualOk := findEqualStringMatchersAndCollectMatches(
+		actualMatches, actualOk := findEqualOrPrefixStringMatchersAndCollectMatches(
 			orStringMatcher{
 				&equalStringMatcher{s: "test-1", caseSensitive: false},
 				&equalStringMatcher{s: "test-2", caseSensitive: true},
@@ -935,11 +1035,59 @@ func TestFindEqualStringMatchers(t *testing.T) {
 		require.True(t, actualOk)
 		require.Equal(t, []match{{"test-1", false}, {"test-2", true}}, actualMatches)
 	})
+
+	t.Run("concat of literal prefix matchers (case sensitive)", func(t *testing.T) {
+		actualMatches, actualOk := findEqualOrPrefixStringMatchersAndCollectMatches(
+			orStringMatcher{
+				&literalPrefixSensitiveStringMatcher{prefix: "test-1"},
+				&literalPrefixSensitiveStringMatcher{prefix: "test-2"},
+			},
+		)
+
+		require.True(t, actualOk)
+		require.Equal(t, []match{{"test-1", true}, {"test-2", true}}, actualMatches)
+	})
+
+	t.Run("concat of literal prefix matchers (case insensitive)", func(t *testing.T) {
+		actualMatches, actualOk := findEqualOrPrefixStringMatchersAndCollectMatches(
+			orStringMatcher{
+				&literalPrefixInsensitiveStringMatcher{prefix: "test-1"},
+				&literalPrefixInsensitiveStringMatcher{prefix: "test-2"},
+			},
+		)
+
+		require.True(t, actualOk)
+		require.Equal(t, []match{{"test-1", false}, {"test-2", false}}, actualMatches)
+	})
+
+	t.Run("concat of literal prefix matchers (mixed case)", func(t *testing.T) {
+		actualMatches, actualOk := findEqualOrPrefixStringMatchersAndCollectMatches(
+			orStringMatcher{
+				&literalPrefixInsensitiveStringMatcher{prefix: "test-1"},
+				&literalPrefixSensitiveStringMatcher{prefix: "test-2"},
+			},
+		)
+
+		require.True(t, actualOk)
+		require.Equal(t, []match{{"test-1", false}, {"test-2", true}}, actualMatches)
+	})
+
+	t.Run("concat of literal string and prefix matchers (case sensitive)", func(t *testing.T) {
+		actualMatches, actualOk := findEqualOrPrefixStringMatchersAndCollectMatches(
+			orStringMatcher{
+				&equalStringMatcher{s: "test-1", caseSensitive: true},
+				&literalPrefixSensitiveStringMatcher{prefix: "test-2"},
+			},
+		)
+
+		require.True(t, actualOk)
+		require.Equal(t, []match{{"test-1", true}, {"test-2", true}}, actualMatches)
+	})
 }
 
 // This benchmark is used to find a good threshold to use to apply the optimization
-// done by optimizeEqualStringMatchers().
-func BenchmarkOptimizeEqualStringMatchers(b *testing.B) {
+// done by optimizeEqualOrPrefixStringMatchers().
+func BenchmarkOptimizeEqualOrPrefixStringMatchers(b *testing.B) {
 	randGenerator := rand.New(rand.NewSource(time.Now().UnixNano()))
 
 	// Generate variable lengths random texts to match against.
@@ -949,42 +1097,51 @@ func BenchmarkOptimizeEqualStringMatchers(b *testing.B) {
 
 	for numAlternations := 2; numAlternations <= 256; numAlternations *= 2 {
 		for _, caseSensitive := range []bool{true, false} {
-			b.Run(fmt.Sprintf("alternations: %d case sensitive: %t", numAlternations, caseSensitive), func(b *testing.B) {
-				// Generate a regex with the expected number of alternations.
-				re := strings.Join(randStrings(randGenerator, numAlternations, 10), "|")
-				if !caseSensitive {
-					re = "(?i:(" + re + "))"
-				}
-
-				parsed, err := syntax.Parse(re, syntax.Perl)
-				require.NoError(b, err)
-
-				unoptimized := stringMatcherFromRegexpInternal(parsed)
-				require.IsType(b, orStringMatcher{}, unoptimized)
-
-				optimized := optimizeEqualStringMatchers(unoptimized, 0)
-				if numAlternations < minEqualMultiStringMatcherMapThreshold {
-					require.IsType(b, &equalMultiStringSliceMatcher{}, optimized)
-				} else {
-					require.IsType(b, &equalMultiStringMapMatcher{}, optimized)
-				}
-
-				b.Run("without optimizeEqualStringMatchers()", func(b *testing.B) {
-					for n := 0; n < b.N; n++ {
-						for _, t := range texts {
-							unoptimized.Matches(t)
-						}
+			for _, prefixMatcher := range []bool{true, false} {
+				b.Run(fmt.Sprintf("alternations: %d case sensitive: %t prefix matcher: %t", numAlternations, caseSensitive, prefixMatcher), func(b *testing.B) {
+					// If the test should run on prefix matchers, we add a wildcard matcher as suffix (prefix will be a literal).
+					suffix := ""
+					if prefixMatcher {
+						suffix = ".*"
 					}
-				})
 
-				b.Run("with optimizeEqualStringMatchers()", func(b *testing.B) {
-					for n := 0; n < b.N; n++ {
-						for _, t := range texts {
-							optimized.Matches(t)
-						}
+					// Generate a regex with the expected number of alternations.
+					re := strings.Join(randStringsWithSuffix(randGenerator, numAlternations, 10, suffix), "|")
+					if !caseSensitive {
+						re = "(?i:(" + re + "))"
 					}
+					b.Logf("regexp: %s", re)
+
+					parsed, err := syntax.Parse(re, syntax.Perl)
+					require.NoError(b, err)
+
+					unoptimized := stringMatcherFromRegexpInternal(parsed)
+					require.IsType(b, orStringMatcher{}, unoptimized)
+
+					optimized := optimizeEqualOrPrefixStringMatchers(unoptimized, 0)
+					if numAlternations < minEqualMultiStringMatcherMapThreshold && !prefixMatcher {
+						require.IsType(b, &equalMultiStringSliceMatcher{}, optimized)
+					} else {
+						require.IsType(b, &equalMultiStringMapMatcher{}, optimized)
+					}
+
+					b.Run("without optimizeEqualOrPrefixStringMatchers()", func(b *testing.B) {
+						for n := 0; n < b.N; n++ {
+							for _, t := range texts {
+								unoptimized.Matches(t)
+							}
+						}
+					})
+
+					b.Run("with optimizeEqualOrPrefixStringMatchers()", func(b *testing.B) {
+						for n := 0; n < b.N; n++ {
+							for _, t := range texts {
+								optimized.Matches(t)
+							}
+						}
+					})
 				})
-			})
+			}
 		}
 	}
 }
@@ -1204,10 +1361,16 @@ func visitStringMatcher(matcher StringMatcher, callback func(matcher StringMatch
 		}
 
 	// No nested matchers for the following ones.
+	case *equalMultiStringMapMatcher:
+		for _, prefixes := range casted.prefixes {
+			for _, matcher := range prefixes {
+				visitStringMatcher(matcher, callback)
+			}
+		}
+
 	case emptyStringMatcher:
 	case *equalStringMatcher:
 	case *equalMultiStringSliceMatcher:
-	case *equalMultiStringMapMatcher:
 	case anyStringWithoutNewlineMatcher:
 	case *anyNonEmptyStringMatcher:
 	case trueMatcher:

prompb/codec.go

@@ -0,0 +1,201 @@
+// Copyright 2024 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package prompb
+
+import (
+	"strings"
+
+	"github.com/prometheus/common/model"
+
+	"github.com/prometheus/prometheus/model/exemplar"
+	"github.com/prometheus/prometheus/model/histogram"
+	"github.com/prometheus/prometheus/model/labels"
+)
+
+// NOTE(bwplotka): This file's code is tested in /prompb/rwcommon.
+
+// ToLabels return model labels.Labels from timeseries' remote labels.
+func (m TimeSeries) ToLabels(b *labels.ScratchBuilder, _ []string) labels.Labels {
+	return labelProtosToLabels(b, m.GetLabels())
+}
+
+// ToLabels return model labels.Labels from timeseries' remote labels.
+func (m ChunkedSeries) ToLabels(b *labels.ScratchBuilder, _ []string) labels.Labels {
+	return labelProtosToLabels(b, m.GetLabels())
+}
+
+func labelProtosToLabels(b *labels.ScratchBuilder, labelPairs []Label) labels.Labels {
+	b.Reset()
+	for _, l := range labelPairs {
+		b.Add(l.Name, l.Value)
+	}
+	b.Sort()
+	return b.Labels()
+}
+
+// FromLabels transforms labels into prompb labels. The buffer slice
+// will be used to avoid allocations if it is big enough to store the labels.
+func FromLabels(lbls labels.Labels, buf []Label) []Label {
+	result := buf[:0]
+	lbls.Range(func(l labels.Label) {
+		result = append(result, Label{
+			Name:  l.Name,
+			Value: l.Value,
+		})
+	})
+	return result
+}
+
+// FromMetadataType transforms a Prometheus metricType into prompb metricType. Since the former is a string we need to transform it to an enum.
+func FromMetadataType(t model.MetricType) MetricMetadata_MetricType {
+	mt := strings.ToUpper(string(t))
+	v, ok := MetricMetadata_MetricType_value[mt]
+	if !ok {
+		return MetricMetadata_UNKNOWN
+	}
+	return MetricMetadata_MetricType(v)
+}
+
+// IsFloatHistogram returns true if the histogram is float.
+func (h Histogram) IsFloatHistogram() bool {
+	_, ok := h.GetCount().(*Histogram_CountFloat)
+	return ok
+}
+
+// ToIntHistogram returns integer Prometheus histogram from the remote implementation
+// of integer histogram. If it's a float histogram, the method returns nil.
+func (h Histogram) ToIntHistogram() *histogram.Histogram {
+	if h.IsFloatHistogram() {
+		return nil
+	}
+	return &histogram.Histogram{
+		CounterResetHint: histogram.CounterResetHint(h.ResetHint),
+		Schema:           h.Schema,
+		ZeroThreshold:    h.ZeroThreshold,
+		ZeroCount:        h.GetZeroCountInt(),
+		Count:            h.GetCountInt(),
+		Sum:              h.Sum,
+		PositiveSpans:    spansProtoToSpans(h.GetPositiveSpans()),
+		PositiveBuckets:  h.GetPositiveDeltas(),
+		NegativeSpans:    spansProtoToSpans(h.GetNegativeSpans()),
+		NegativeBuckets:  h.GetNegativeDeltas(),
+	}
+}
+
+// ToFloatHistogram returns float Prometheus histogram from the remote implementation
+// of float histogram. If the underlying implementation is an integer histogram, a
+// conversion is performed.
+func (h Histogram) ToFloatHistogram() *histogram.FloatHistogram {
+	if h.IsFloatHistogram() {
+		return &histogram.FloatHistogram{
+			CounterResetHint: histogram.CounterResetHint(h.ResetHint),
+			Schema:           h.Schema,
+			ZeroThreshold:    h.ZeroThreshold,
+			ZeroCount:        h.GetZeroCountFloat(),
+			Count:            h.GetCountFloat(),
+			Sum:              h.Sum,
+			PositiveSpans:    spansProtoToSpans(h.GetPositiveSpans()),
+			PositiveBuckets:  h.GetPositiveCounts(),
+			NegativeSpans:    spansProtoToSpans(h.GetNegativeSpans()),
+			NegativeBuckets:  h.GetNegativeCounts(),
+		}
+	}
+	// Conversion from integer histogram.
+	return &histogram.FloatHistogram{
+		CounterResetHint: histogram.CounterResetHint(h.ResetHint),
+		Schema:           h.Schema,
+		ZeroThreshold:    h.ZeroThreshold,
+		ZeroCount:        float64(h.GetZeroCountInt()),
+		Count:            float64(h.GetCountInt()),
+		Sum:              h.Sum,
+		PositiveSpans:    spansProtoToSpans(h.GetPositiveSpans()),
+		PositiveBuckets:  deltasToCounts(h.GetPositiveDeltas()),
+		NegativeSpans:    spansProtoToSpans(h.GetNegativeSpans()),
+		NegativeBuckets:  deltasToCounts(h.GetNegativeDeltas()),
+	}
+}
+
+func spansProtoToSpans(s []BucketSpan) []histogram.Span {
+	spans := make([]histogram.Span, len(s))
+	for i := 0; i < len(s); i++ {
+		spans[i] = histogram.Span{Offset: s[i].Offset, Length: s[i].Length}
+	}
+
+	return spans
+}
+
+func deltasToCounts(deltas []int64) []float64 {
+	counts := make([]float64, len(deltas))
+	var cur float64
+	for i, d := range deltas {
+		cur += float64(d)
+		counts[i] = cur
+	}
+	return counts
+}
+
+// FromIntHistogram returns remote Histogram from the integer Histogram.
+func FromIntHistogram(timestamp int64, h *histogram.Histogram) Histogram {
+	return Histogram{
+		Count:          &Histogram_CountInt{CountInt: h.Count},
+		Sum:            h.Sum,
+		Schema:         h.Schema,
+		ZeroThreshold:  h.ZeroThreshold,
+		ZeroCount:      &Histogram_ZeroCountInt{ZeroCountInt: h.ZeroCount},
+		NegativeSpans:  spansToSpansProto(h.NegativeSpans),
+		NegativeDeltas: h.NegativeBuckets,
+		PositiveSpans:  spansToSpansProto(h.PositiveSpans),
+		PositiveDeltas: h.PositiveBuckets,
+		ResetHint:      Histogram_ResetHint(h.CounterResetHint),
+		Timestamp:      timestamp,
+	}
+}
+
+// FromFloatHistogram returns remote Histogram from the float Histogram.
+func FromFloatHistogram(timestamp int64, fh *histogram.FloatHistogram) Histogram {
+	return Histogram{
+		Count:          &Histogram_CountFloat{CountFloat: fh.Count},
+		Sum:            fh.Sum,
+		Schema:         fh.Schema,
+		ZeroThreshold:  fh.ZeroThreshold,
+		ZeroCount:      &Histogram_ZeroCountFloat{ZeroCountFloat: fh.ZeroCount},
+		NegativeSpans:  spansToSpansProto(fh.NegativeSpans),
+		NegativeCounts: fh.NegativeBuckets,
+		PositiveSpans:  spansToSpansProto(fh.PositiveSpans),
+		PositiveCounts: fh.PositiveBuckets,
+		ResetHint:      Histogram_ResetHint(fh.CounterResetHint),
+		Timestamp:      timestamp,
+	}
+}
+
+func spansToSpansProto(s []histogram.Span) []BucketSpan {
+	spans := make([]BucketSpan, len(s))
+	for i := 0; i < len(s); i++ {
+		spans[i] = BucketSpan{Offset: s[i].Offset, Length: s[i].Length}
+	}
+
+	return spans
+}
+
+// ToExemplar converts remote exemplar to model exemplar.
+func (m Exemplar) ToExemplar(b *labels.ScratchBuilder, _ []string) exemplar.Exemplar {
+	timestamp := m.Timestamp
+
+	return exemplar.Exemplar{
+		Labels: labelProtosToLabels(b, m.GetLabels()),
+		Value:  m.Value,
+		Ts:     timestamp,
+		HasTs:  timestamp != 0,
+	}
+}

prompb/custom.go

@@ -17,14 +17,6 @@ import (
 	"sync"
 )
 
-func (m Sample) T() int64   { return m.Timestamp }
-func (m Sample) V() float64 { return m.Value }
-
-func (h Histogram) IsFloatHistogram() bool {
-	_, ok := h.GetCount().(*Histogram_CountFloat)
-	return ok
-}
-
 func (r *ChunkedReadResponse) PooledMarshal(p *sync.Pool) ([]byte, error) {
 	size := r.Size()
 	data, ok := p.Get().(*[]byte)

prompb/io/prometheus/write/v2/codec.go

@@ -0,0 +1,216 @@
+// Copyright 2024 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package writev2
+
+import (
+	"github.com/prometheus/common/model"
+
+	"github.com/prometheus/prometheus/model/exemplar"
+	"github.com/prometheus/prometheus/model/histogram"
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/model/metadata"
+)
+
+// NOTE(bwplotka): This file's code is tested in /prompb/rwcommon.
+
+// ToLabels return model labels.Labels from timeseries' remote labels.
+func (m TimeSeries) ToLabels(b *labels.ScratchBuilder, symbols []string) labels.Labels {
+	return desymbolizeLabels(b, m.GetLabelsRefs(), symbols)
+}
+
+// ToMetadata return model metadata from timeseries' remote metadata.
+func (m TimeSeries) ToMetadata(symbols []string) metadata.Metadata {
+	typ := model.MetricTypeUnknown
+	switch m.Metadata.Type {
+	case Metadata_METRIC_TYPE_COUNTER:
+		typ = model.MetricTypeCounter
+	case Metadata_METRIC_TYPE_GAUGE:
+		typ = model.MetricTypeGauge
+	case Metadata_METRIC_TYPE_HISTOGRAM:
+		typ = model.MetricTypeHistogram
+	case Metadata_METRIC_TYPE_GAUGEHISTOGRAM:
+		typ = model.MetricTypeGaugeHistogram
+	case Metadata_METRIC_TYPE_SUMMARY:
+		typ = model.MetricTypeSummary
+	case Metadata_METRIC_TYPE_INFO:
+		typ = model.MetricTypeInfo
+	case Metadata_METRIC_TYPE_STATESET:
+		typ = model.MetricTypeStateset
+	}
+	return metadata.Metadata{
+		Type: typ,
+		Unit: symbols[m.Metadata.UnitRef],
+		Help: symbols[m.Metadata.HelpRef],
+	}
+}
+
+// FromMetadataType transforms a Prometheus metricType into writev2 metricType.
+// Since the former is a string we need to transform it to an enum.
+func FromMetadataType(t model.MetricType) Metadata_MetricType {
+	switch t {
+	case model.MetricTypeCounter:
+		return Metadata_METRIC_TYPE_COUNTER
+	case model.MetricTypeGauge:
+		return Metadata_METRIC_TYPE_GAUGE
+	case model.MetricTypeHistogram:
+		return Metadata_METRIC_TYPE_HISTOGRAM
+	case model.MetricTypeGaugeHistogram:
+		return Metadata_METRIC_TYPE_GAUGEHISTOGRAM
+	case model.MetricTypeSummary:
+		return Metadata_METRIC_TYPE_SUMMARY
+	case model.MetricTypeInfo:
+		return Metadata_METRIC_TYPE_INFO
+	case model.MetricTypeStateset:
+		return Metadata_METRIC_TYPE_STATESET
+	default:
+		return Metadata_METRIC_TYPE_UNSPECIFIED
+	}
+}
+
+// IsFloatHistogram returns true if the histogram is float.
+func (h Histogram) IsFloatHistogram() bool {
+	_, ok := h.GetCount().(*Histogram_CountFloat)
+	return ok
+}
+
+// ToIntHistogram returns integer Prometheus histogram from the remote implementation
+// of integer histogram. If it's a float histogram, the method returns nil.
+func (h Histogram) ToIntHistogram() *histogram.Histogram {
+	if h.IsFloatHistogram() {
+		return nil
+	}
+	return &histogram.Histogram{
+		CounterResetHint: histogram.CounterResetHint(h.ResetHint),
+		Schema:           h.Schema,
+		ZeroThreshold:    h.ZeroThreshold,
+		ZeroCount:        h.GetZeroCountInt(),
+		Count:            h.GetCountInt(),
+		Sum:              h.Sum,
+		PositiveSpans:    spansProtoToSpans(h.GetPositiveSpans()),
+		PositiveBuckets:  h.GetPositiveDeltas(),
+		NegativeSpans:    spansProtoToSpans(h.GetNegativeSpans()),
+		NegativeBuckets:  h.GetNegativeDeltas(),
+		CustomValues:     h.GetCustomValues(),
+	}
+}
+
+// ToFloatHistogram returns float Prometheus histogram from the remote implementation
+// of float histogram. If the underlying implementation is an integer histogram, a
+// conversion is performed.
+func (h Histogram) ToFloatHistogram() *histogram.FloatHistogram {
+	if h.IsFloatHistogram() {
+		return &histogram.FloatHistogram{
+			CounterResetHint: histogram.CounterResetHint(h.ResetHint),
+			Schema:           h.Schema,
+			ZeroThreshold:    h.ZeroThreshold,
+			ZeroCount:        h.GetZeroCountFloat(),
+			Count:            h.GetCountFloat(),
+			Sum:              h.Sum,
+			PositiveSpans:    spansProtoToSpans(h.GetPositiveSpans()),
+			PositiveBuckets:  h.GetPositiveCounts(),
+			NegativeSpans:    spansProtoToSpans(h.GetNegativeSpans()),
+			NegativeBuckets:  h.GetNegativeCounts(),
+			CustomValues:     h.GetCustomValues(),
+		}
+	}
+	// Conversion from integer histogram.
+	return &histogram.FloatHistogram{
+		CounterResetHint: histogram.CounterResetHint(h.ResetHint),
+		Schema:           h.Schema,
+		ZeroThreshold:    h.ZeroThreshold,
+		ZeroCount:        float64(h.GetZeroCountInt()),
+		Count:            float64(h.GetCountInt()),
+		Sum:              h.Sum,
+		PositiveSpans:    spansProtoToSpans(h.GetPositiveSpans()),
+		PositiveBuckets:  deltasToCounts(h.GetPositiveDeltas()),
+		NegativeSpans:    spansProtoToSpans(h.GetNegativeSpans()),
+		NegativeBuckets:  deltasToCounts(h.GetNegativeDeltas()),
+		CustomValues:     h.GetCustomValues(),
+	}
+}
+
+func spansProtoToSpans(s []BucketSpan) []histogram.Span {
+	spans := make([]histogram.Span, len(s))
+	for i := 0; i < len(s); i++ {
+		spans[i] = histogram.Span{Offset: s[i].Offset, Length: s[i].Length}
+	}
+
+	return spans
+}
+
+func deltasToCounts(deltas []int64) []float64 {
+	counts := make([]float64, len(deltas))
+	var cur float64
+	for i, d := range deltas {
+		cur += float64(d)
+		counts[i] = cur
+	}
+	return counts
+}
+
+// FromIntHistogram returns remote Histogram from the integer Histogram.
+func FromIntHistogram(timestamp int64, h *histogram.Histogram) Histogram {
+	return Histogram{
+		Count:          &Histogram_CountInt{CountInt: h.Count},
+		Sum:            h.Sum,
+		Schema:         h.Schema,
+		ZeroThreshold:  h.ZeroThreshold,
+		ZeroCount:      &Histogram_ZeroCountInt{ZeroCountInt: h.ZeroCount},
+		NegativeSpans:  spansToSpansProto(h.NegativeSpans),
+		NegativeDeltas: h.NegativeBuckets,
+		PositiveSpans:  spansToSpansProto(h.PositiveSpans),
+		PositiveDeltas: h.PositiveBuckets,
+		ResetHint:      Histogram_ResetHint(h.CounterResetHint),
+		CustomValues:   h.CustomValues,
+		Timestamp:      timestamp,
+	}
+}
+
+// FromFloatHistogram returns remote Histogram from the float Histogram.
+func FromFloatHistogram(timestamp int64, fh *histogram.FloatHistogram) Histogram {
+	return Histogram{
+		Count:          &Histogram_CountFloat{CountFloat: fh.Count},
+		Sum:            fh.Sum,
+		Schema:         fh.Schema,
+		ZeroThreshold:  fh.ZeroThreshold,
+		ZeroCount:      &Histogram_ZeroCountFloat{ZeroCountFloat: fh.ZeroCount},
+		NegativeSpans:  spansToSpansProto(fh.NegativeSpans),
+		NegativeCounts: fh.NegativeBuckets,
+		PositiveSpans:  spansToSpansProto(fh.PositiveSpans),
+		PositiveCounts: fh.PositiveBuckets,
+		ResetHint:      Histogram_ResetHint(fh.CounterResetHint),
+		CustomValues:   fh.CustomValues,
+		Timestamp:      timestamp,
+	}
+}
+
+func spansToSpansProto(s []histogram.Span) []BucketSpan {
+	spans := make([]BucketSpan, len(s))
+	for i := 0; i < len(s); i++ {
+		spans[i] = BucketSpan{Offset: s[i].Offset, Length: s[i].Length}
+	}
+
+	return spans
+}
+
+func (m Exemplar) ToExemplar(b *labels.ScratchBuilder, symbols []string) exemplar.Exemplar {
+	timestamp := m.Timestamp
+
+	return exemplar.Exemplar{
+		Labels: desymbolizeLabels(b, m.LabelsRefs, symbols),
+		Value:  m.Value,
+		Ts:     timestamp,
+		HasTs:  timestamp != 0,
+	}
+}

prompb/io/prometheus/write/v2/custom.go

@@ -0,0 +1,165 @@
+// Copyright 2024 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package writev2
+
+import (
+	"slices"
+)
+
+func (m Sample) T() int64   { return m.Timestamp }
+func (m Sample) V() float64 { return m.Value }
+
+func (m *Request) OptimizedMarshal(dst []byte) ([]byte, error) {
+	siz := m.Size()
+	if cap(dst) < siz {
+		dst = make([]byte, siz)
+	}
+	n, err := m.OptimizedMarshalToSizedBuffer(dst[:siz])
+	if err != nil {
+		return nil, err
+	}
+	return dst[:n], nil
+}
+
+// OptimizedMarshalToSizedBuffer is mostly a copy of the generated MarshalToSizedBuffer,
+// but calls OptimizedMarshalToSizedBuffer on the timeseries.
+func (m *Request) OptimizedMarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.XXX_unrecognized != nil {
+		i -= len(m.XXX_unrecognized)
+		copy(dAtA[i:], m.XXX_unrecognized)
+	}
+	if len(m.Timeseries) > 0 {
+		for iNdEx := len(m.Timeseries) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Timeseries[iNdEx].OptimizedMarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintTypes(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0x2a
+		}
+	}
+	if len(m.Symbols) > 0 {
+		for iNdEx := len(m.Symbols) - 1; iNdEx >= 0; iNdEx-- {
+			i -= len(m.Symbols[iNdEx])
+			copy(dAtA[i:], m.Symbols[iNdEx])
+			i = encodeVarintTypes(dAtA, i, uint64(len(m.Symbols[iNdEx])))
+			i--
+			dAtA[i] = 0x22
+		}
+	}
+	return len(dAtA) - i, nil
+}
+
+// OptimizedMarshalToSizedBuffer is mostly a copy of the generated MarshalToSizedBuffer,
+// but marshals m.LabelsRefs in place without extra allocations.
+func (m *TimeSeries) OptimizedMarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.XXX_unrecognized != nil {
+		i -= len(m.XXX_unrecognized)
+		copy(dAtA[i:], m.XXX_unrecognized)
+	}
+	if m.CreatedTimestamp != 0 {
+		i = encodeVarintTypes(dAtA, i, uint64(m.CreatedTimestamp))
+		i--
+		dAtA[i] = 0x30
+	}
+	{
+		size, err := m.Metadata.MarshalToSizedBuffer(dAtA[:i])
+		if err != nil {
+			return 0, err
+		}
+		i -= size
+		i = encodeVarintTypes(dAtA, i, uint64(size))
+	}
+	i--
+	dAtA[i] = 0x2a
+	if len(m.Histograms) > 0 {
+		for iNdEx := len(m.Histograms) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Histograms[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintTypes(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0x1a
+		}
+	}
+	if len(m.Exemplars) > 0 {
+		for iNdEx := len(m.Exemplars) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Exemplars[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintTypes(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0x22
+		}
+	}
+	if len(m.Samples) > 0 {
+		for iNdEx := len(m.Samples) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Samples[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintTypes(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0x12
+		}
+	}
+
+	if len(m.LabelsRefs) > 0 {
+		// This is the trick: encode the varints in reverse order to make it easier
+		// to do it in place. Then reverse the whole thing.
+		var j10 int
+		start := i
+		for _, num := range m.LabelsRefs {
+			for num >= 1<<7 {
+				dAtA[i-1] = uint8(uint64(num)&0x7f | 0x80)
+				num >>= 7
+				i--
+				j10++
+			}
+			dAtA[i-1] = uint8(num)
+			i--
+			j10++
+		}
+		slices.Reverse(dAtA[i:start])
+		// --- end of trick
+
+		i = encodeVarintTypes(dAtA, i, uint64(j10))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}

prompb/io/prometheus/write/v2/custom_test.go

@@ -0,0 +1,97 @@
+// Copyright 2023 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package writev2
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestOptimizedMarshal(t *testing.T) {
+	for _, tt := range []struct {
+		name string
+		m    *Request
+	}{
+		{
+			name: "empty",
+			m:    &Request{},
+		},
+		{
+			name: "simple",
+			m: &Request{
+				Timeseries: []TimeSeries{
+					{
+						LabelsRefs: []uint32{
+							0, 1,
+							2, 3,
+							4, 5,
+							6, 7,
+							8, 9,
+							10, 11,
+							12, 13,
+							14, 15,
+						},
+
+						Samples:    []Sample{{Value: 1, Timestamp: 0}},
+						Exemplars:  []Exemplar{{LabelsRefs: []uint32{0, 1}, Value: 1, Timestamp: 0}},
+						Histograms: nil,
+					},
+					{
+						LabelsRefs: []uint32{
+							0, 1,
+							2, 3,
+							4, 5,
+							6, 7,
+							8, 9,
+							10, 11,
+							12, 13,
+							14, 15,
+						},
+						Samples:    []Sample{{Value: 2, Timestamp: 1}},
+						Exemplars:  []Exemplar{{LabelsRefs: []uint32{0, 1}, Value: 2, Timestamp: 1}},
+						Histograms: nil,
+					},
+				},
+				Symbols: []string{
+					"a", "b",
+					"c", "d",
+					"e", "f",
+					"g", "h",
+					"i", "j",
+					"k", "l",
+					"m", "n",
+					"o", "p",
+				},
+			},
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			// Keep the slice allocated to mimic what std Marshal
+			// would give to sized Marshal.
+			got := make([]byte, 0)
+
+			// Should be the same as the standard marshal.
+			expected, err := tt.m.Marshal()
+			require.NoError(t, err)
+			got, err = tt.m.OptimizedMarshal(got)
+			require.NoError(t, err)
+			require.Equal(t, expected, got)
+
+			// Unmarshal should work too.
+			m := &Request{}
+			require.NoError(t, m.Unmarshal(got))
+			require.Equal(t, tt.m, m)
+		})
+	}
+}

prompb/io/prometheus/write/v2/symbols.go

@@ -0,0 +1,83 @@
+// Copyright 2024 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package writev2
+
+import "github.com/prometheus/prometheus/model/labels"
+
+// SymbolsTable implements table for easy symbol use.
+type SymbolsTable struct {
+	strings    []string
+	symbolsMap map[string]uint32
+}
+
+// NewSymbolTable returns a symbol table.
+func NewSymbolTable() SymbolsTable {
+	return SymbolsTable{
+		// Empty string is required as a first element.
+		symbolsMap: map[string]uint32{"": 0},
+		strings:    []string{""},
+	}
+}
+
+// Symbolize adds (if not added before) a string to the symbols table,
+// while returning its reference number.
+func (t *SymbolsTable) Symbolize(str string) uint32 {
+	if ref, ok := t.symbolsMap[str]; ok {
+		return ref
+	}
+	ref := uint32(len(t.strings))
+	t.strings = append(t.strings, str)
+	t.symbolsMap[str] = ref
+	return ref
+}
+
+// SymbolizeLabels symbolize Prometheus labels.
+func (t *SymbolsTable) SymbolizeLabels(lbls labels.Labels, buf []uint32) []uint32 {
+	result := buf[:0]
+	lbls.Range(func(l labels.Label) {
+		off := t.Symbolize(l.Name)
+		result = append(result, off)
+		off = t.Symbolize(l.Value)
+		result = append(result, off)
+	})
+	return result
+}
+
+// Symbols returns computes symbols table to put in e.g. Request.Symbols.
+// As per spec, order does not matter.
+func (t *SymbolsTable) Symbols() []string {
+	return t.strings
+}
+
+// Reset clears symbols table.
+func (t *SymbolsTable) Reset() {
+	// NOTE: Make sure to keep empty symbol.
+	t.strings = t.strings[:1]
+	for k := range t.symbolsMap {
+		if k == "" {
+			continue
+		}
+		delete(t.symbolsMap, k)
+	}
+}
+
+// desymbolizeLabels decodes label references, with given symbols to labels.
+func desymbolizeLabels(b *labels.ScratchBuilder, labelRefs []uint32, symbols []string) labels.Labels {
+	b.Reset()
+	for i := 0; i < len(labelRefs); i += 2 {
+		b.Add(symbols[labelRefs[i]], symbols[labelRefs[i+1]])
+	}
+	b.Sort()
+	return b.Labels()
+}

prompb/io/prometheus/write/v2/symbols_test.go

@@ -0,0 +1,60 @@
+// Copyright 2024 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package writev2
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/prometheus/model/labels"
+)
+
+func TestSymbolsTable(t *testing.T) {
+	s := NewSymbolTable()
+	require.Equal(t, []string{""}, s.Symbols(), "required empty reference does not exist")
+	require.Equal(t, uint32(0), s.Symbolize(""))
+	require.Equal(t, []string{""}, s.Symbols())
+
+	require.Equal(t, uint32(1), s.Symbolize("abc"))
+	require.Equal(t, []string{"", "abc"}, s.Symbols())
+
+	require.Equal(t, uint32(2), s.Symbolize("__name__"))
+	require.Equal(t, []string{"", "abc", "__name__"}, s.Symbols())
+
+	require.Equal(t, uint32(3), s.Symbolize("foo"))
+	require.Equal(t, []string{"", "abc", "__name__", "foo"}, s.Symbols())
+
+	s.Reset()
+	require.Equal(t, []string{""}, s.Symbols(), "required empty reference does not exist")
+	require.Equal(t, uint32(0), s.Symbolize(""))
+
+	require.Equal(t, uint32(1), s.Symbolize("__name__"))
+	require.Equal(t, []string{"", "__name__"}, s.Symbols())
+
+	require.Equal(t, uint32(2), s.Symbolize("abc"))
+	require.Equal(t, []string{"", "__name__", "abc"}, s.Symbols())
+
+	ls := labels.FromStrings("__name__", "qwer", "zxcv", "1234")
+	encoded := s.SymbolizeLabels(ls, nil)
+	require.Equal(t, []uint32{1, 3, 4, 5}, encoded)
+	b := labels.NewScratchBuilder(len(encoded))
+	decoded := desymbolizeLabels(&b, encoded, s.Symbols())
+	require.Equal(t, ls, decoded)
+
+	// Different buf.
+	ls = labels.FromStrings("__name__", "qwer", "zxcv2222", "1234")
+	encoded = s.SymbolizeLabels(ls, []uint32{1, 3, 4, 5})
+	require.Equal(t, []uint32{1, 3, 6, 5}, encoded)
+}

prompb/io/prometheus/write/v2/types.proto

@@ -0,0 +1,260 @@
+// Copyright 2024 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// NOTE: This file is also available on https://buf.build/prometheus/prometheus/docs/main:io.prometheus.write.v2
+
+syntax = "proto3";
+package io.prometheus.write.v2;
+
+option go_package = "writev2";
+
+import "gogoproto/gogo.proto";
+
+// Request represents a request to write the given timeseries to a remote destination.
+// This message was introduced in the Remote Write 2.0 specification:
+// https://prometheus.io/docs/concepts/remote_write_spec_2_0/
+//
+// The canonical Content-Type request header value for this message is
+// "application/x-protobuf;proto=io.prometheus.write.v2.Request"
+//
+// NOTE: gogoproto options might change in future for this file, they
+// are not part of the spec proto (they only modify the generated Go code, not
+// the serialized message). See: https://github.com/prometheus/prometheus/issues/11908
+message Request {
+  // Since Request supersedes 1.0 spec's prometheus.WriteRequest, we reserve the top-down message
+  // for the deterministic interop between those two, see types_test.go for details.
+  // Generally it's not needed, because Receivers must use the Content-Type header, but we want to
+  // be sympathetic to adopters with mistaken implementations and have deterministic error (empty
+  // message if you use the wrong proto schema).
+  reserved 1 to 3;
+
+  // symbols contains a de-duplicated array of string elements used for various
+  // items in a Request message, like labels and metadata items. For the sender's convenience
+  // around empty values for optional fields like unit_ref, symbols array MUST start with
+  // empty string.
+  //
+  // To decode each of the symbolized strings, referenced, by "ref(s)" suffix, you
+  // need to lookup the actual string by index from symbols array. The order of
+  // strings is up to the sender. The receiver should not assume any particular encoding.
+  repeated string symbols = 4;
+  // timeseries represents an array of distinct series with 0 or more samples.
+  repeated TimeSeries timeseries = 5 [(gogoproto.nullable) = false];
+}
+
+// TimeSeries represents a single series.
+message TimeSeries {
+  // labels_refs is a list of label name-value pair references, encoded
+  // as indices to the Request.symbols array. This list's length is always
+  // a multiple of two, and the underlying labels should be sorted lexicographically.
+  //
+  // Note that there might be multiple TimeSeries objects in the same
+  // Requests with the same labels e.g. for different exemplars, metadata
+  // or created timestamp.
+  repeated uint32 labels_refs = 1;
+
+  // Timeseries messages can either specify samples or (native) histogram samples
+  // (histogram field), but not both. For a typical sender (real-time metric
+  // streaming), in healthy cases, there will be only one sample or histogram.
+  //
+  // Samples and histograms are sorted by timestamp (older first).
+  repeated Sample samples = 2 [(gogoproto.nullable) = false];
+  repeated Histogram histograms = 3 [(gogoproto.nullable) = false];
+
+  // exemplars represents an optional set of exemplars attached to this series' samples.
+  repeated Exemplar exemplars = 4 [(gogoproto.nullable) = false];
+
+  // metadata represents the metadata associated with the given series' samples.
+  Metadata metadata = 5 [(gogoproto.nullable) = false];
+
+  // created_timestamp represents an optional created timestamp associated with
+  // this series' samples in ms format, typically for counter or histogram type
+  // metrics. Created timestamp represents the time when the counter started
+  // counting (sometimes referred to as start timestamp), which can increase
+  // the accuracy of query results.
+  //
+  // Note that some receivers might require this and in return fail to
+  // ingest such samples within the Request.
+  //
+  // For Go, see github.com/prometheus/prometheus/model/timestamp/timestamp.go
+  // for conversion from/to time.Time to Prometheus timestamp.
+  //
+  // Note that the "optional" keyword is omitted due to
+  // https://cloud.google.com/apis/design/design_patterns.md#optional_primitive_fields
+  // Zero value means value not set. If you need to use exactly zero value for
+  // the timestamp, use 1 millisecond before or after.
+  int64 created_timestamp = 6;
+}
+
+// Exemplar is an additional information attached to some series' samples.
+// It is typically used to attach an example trace or request ID associated with
+// the metric changes.
+message Exemplar {
+  // labels_refs is an optional list of label name-value pair references, encoded
+  // as indices to the Request.symbols array. This list's len is always
+  // a multiple of 2, and the underlying labels should be sorted lexicographically.
+  // If the exemplar references a trace it should use the `trace_id` label name, as a best practice.
+  repeated uint32 labels_refs = 1;
+  // value represents an exact example value. This can be useful when the exemplar
+  // is attached to a histogram, which only gives an estimated value through buckets.
+  double value = 2;
+  // timestamp represents an optional timestamp of the sample in ms.
+  //
+  // For Go, see github.com/prometheus/prometheus/model/timestamp/timestamp.go
+  // for conversion from/to time.Time to Prometheus timestamp.
+  //
+  // Note that the "optional" keyword is omitted due to
+  // https://cloud.google.com/apis/design/design_patterns.md#optional_primitive_fields
+  // Zero value means value not set. If you need to use exactly zero value for
+  // the timestamp, use 1 millisecond before or after.
+  int64 timestamp = 3;
+}
+
+// Sample represents series sample.
+message Sample {
+  // value of the sample.
+  double value = 1;
+  // timestamp represents timestamp of the sample in ms.
+  //
+  // For Go, see github.com/prometheus/prometheus/model/timestamp/timestamp.go
+  // for conversion from/to time.Time to Prometheus timestamp.
+  int64 timestamp = 2;
+}
+
+// Metadata represents the metadata associated with the given series' samples.
+message Metadata {
+  enum MetricType {
+    METRIC_TYPE_UNSPECIFIED    = 0;
+    METRIC_TYPE_COUNTER        = 1;
+    METRIC_TYPE_GAUGE          = 2;
+    METRIC_TYPE_HISTOGRAM      = 3;
+    METRIC_TYPE_GAUGEHISTOGRAM = 4;
+    METRIC_TYPE_SUMMARY        = 5;
+    METRIC_TYPE_INFO           = 6;
+    METRIC_TYPE_STATESET       = 7;
+  }
+  MetricType type = 1;
+  // help_ref is a reference to the Request.symbols array representing help
+  // text for the metric. Help is optional, reference should point to an empty string in
+  // such a case.
+  uint32 help_ref = 3;
+  // unit_ref is a reference to the Request.symbols array representing a unit
+  // for the metric. Unit is optional, reference should point to an empty string in
+  // such a case.
+  uint32 unit_ref = 4;
+}
+
+// A native histogram, also known as a sparse histogram.
+// Original design doc:
+// https://docs.google.com/document/d/1cLNv3aufPZb3fNfaJgdaRBZsInZKKIHo9E6HinJVbpM/edit
+// The appendix of this design doc also explains the concept of float
+// histograms. This Histogram message can represent both, the usual
+// integer histogram as well as a float histogram.
+message Histogram {
+  enum ResetHint {
+    RESET_HINT_UNSPECIFIED = 0; // Need to test for a counter reset explicitly.
+    RESET_HINT_YES     = 1; // This is the 1st histogram after a counter reset.
+    RESET_HINT_NO      = 2; // There was no counter reset between this and the previous Histogram.
+    RESET_HINT_GAUGE   = 3; // This is a gauge histogram where counter resets don't happen.
+  }
+
+  oneof count { // Count of observations in the histogram.
+    uint64 count_int   = 1;
+    double count_float = 2;
+  }
+  double sum = 3; // Sum of observations in the histogram.
+
+  // The schema defines the bucket schema. Currently, valid numbers
+  // are -53 and numbers in range of -4 <= n <= 8. More valid numbers might be
+  // added in future for new bucketing layouts.
+  //
+  // The schema equal to -53 means custom buckets. See
+  // custom_values field description for more details.
+  //
+  // Values between -4 and 8 represent base-2 bucket schema, where 1
+  // is a bucket boundary in each case, and then each power of two is
+  // divided into 2^n (n is schema value) logarithmic buckets. Or in other words,
+  // each bucket boundary is the previous boundary times 2^(2^-n).
+  sint32 schema             = 4;
+  double zero_threshold     = 5; // Breadth of the zero bucket.
+  oneof zero_count { // Count in zero bucket.
+    uint64 zero_count_int     = 6;
+    double zero_count_float   = 7;
+  }
+
+  // Negative Buckets.
+  repeated BucketSpan negative_spans =  8 [(gogoproto.nullable) = false];
+  // Use either "negative_deltas" or "negative_counts", the former for
+  // regular histograms with integer counts, the latter for
+  // float histograms.
+  repeated sint64 negative_deltas    =  9; // Count delta of each bucket compared to previous one (or to zero for 1st bucket).
+  repeated double negative_counts    = 10; // Absolute count of each bucket.
+
+  // Positive Buckets.
+  //
+  // In case of custom buckets (-53 schema value) the positive buckets are interpreted as follows:
+  // * The span offset+length points to an the index of the custom_values array
+  // or +Inf if pointing to the len of the array.
+  // * The counts and deltas have the same meaning as for exponential histograms.
+  repeated BucketSpan positive_spans = 11 [(gogoproto.nullable) = false];
+  // Use either "positive_deltas" or "positive_counts", the former for
+  // regular histograms with integer counts, the latter for
+  // float histograms.
+  repeated sint64 positive_deltas    = 12; // Count delta of each bucket compared to previous one (or to zero for 1st bucket).
+  repeated double positive_counts    = 13; // Absolute count of each bucket.
+
+  ResetHint reset_hint               = 14;
+  // timestamp represents timestamp of the sample in ms.
+  //
+  // For Go, see github.com/prometheus/prometheus/model/timestamp/timestamp.go
+  // for conversion from/to time.Time to Prometheus timestamp.
+  int64 timestamp = 15;
+
+  // custom_values is an additional field used by non-exponential bucketing layouts.
+  //
+  // For custom buckets (-53 schema value) custom_values specify monotonically
+  // increasing upper inclusive boundaries for the bucket counts with arbitrary
+  // widths for this histogram. In other words, custom_values represents custom,
+  // explicit bucketing that could have been converted from the classic histograms.
+  //
+  // Those bounds are then referenced by spans in positive_spans with corresponding positive
+  // counts of deltas (refer to positive_spans for more details). This way we can
+  // have encode sparse histograms with custom bucketing (many buckets are often
+  // not used).
+  //
+  // Note that for custom bounds, even negative observations are placed in the positive
+  // counts to simplify the implementation and avoid ambiguity of where to place
+  // an underflow bucket, e.g. (-2, 1]. Therefore negative buckets and
+  // the zero bucket are unused, if the schema indicates custom bucketing.
+  //
+  // For each upper boundary the previous boundary represent the lower exclusive
+  // boundary for that bucket. The first element is the upper inclusive boundary
+  // for the first bucket, which implicitly has a lower inclusive bound of -Inf.
+  // This is similar to "le" label semantics on classic histograms. You may add a
+  // bucket with an upper bound of 0 to make sure that you really have no negative
+  // observations, but in practice, native histogram rendering will show both with
+  // or without first upper boundary 0 and no negative counts as the same case.
+  //
+  // The last element is not only the upper inclusive bound of the last regular
+  // bucket, but implicitly the lower exclusive bound of the +Inf bucket.
+  repeated double custom_values = 16;
+}
+
+// A BucketSpan defines a number of consecutive buckets with their
+// offset. Logically, it would be more straightforward to include the
+// bucket counts in the Span. However, the protobuf representation is
+// more compact in the way the data is structured here (with all the
+// buckets in a single array separate from the Spans).
+message BucketSpan {
+  sint32 offset = 1; // Gap to previous span, or starting point for 1st span (which can be negative).
+  uint32 length = 2; // Length of consecutive buckets.
+}

prompb/io/prometheus/write/v2/types_test.go

@@ -0,0 +1,97 @@
+// Copyright 2024 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package writev2
+
+import (
+	"testing"
+	"time"
+
+	"github.com/gogo/protobuf/proto"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/prometheus/prompb"
+)
+
+func TestInteropV2UnmarshalWithV1_DeterministicEmpty(t *testing.T) {
+	expectedV1Empty := &prompb.WriteRequest{}
+	for _, tc := range []struct{ incoming *Request }{
+		{
+			incoming: &Request{}, // Technically wrong, should be at least empty string in symbol.
+		},
+		{
+			incoming: &Request{
+				Symbols: []string{""},
+			}, // NOTE: Without reserved fields, failed with "corrupted" ghost TimeSeries element.
+		},
+		{
+			incoming: &Request{
+				Symbols: []string{"", "__name__", "metric1"},
+				Timeseries: []TimeSeries{
+					{LabelsRefs: []uint32{1, 2}},
+					{Samples: []Sample{{Value: 21.4, Timestamp: time.Now().UnixMilli()}}},
+				}, // NOTE:  Without reserved fields, proto: illegal wireType 7
+			},
+		},
+	} {
+		t.Run("", func(t *testing.T) {
+			in, err := proto.Marshal(tc.incoming)
+			require.NoError(t, err)
+
+			// Test accidental unmarshal of v2 payload with v1 proto.
+			out := &prompb.WriteRequest{}
+			require.NoError(t, proto.Unmarshal(in, out))
+
+			// Drop unknowns, we expect them when incoming payload had some fields.
+			// This field & method will be likely gone after gogo removal.
+			out.XXX_unrecognized = nil // NOTE: out.XXX_DiscardUnknown() does not work with nullables.
+
+			require.Equal(t, expectedV1Empty, out)
+		})
+	}
+}
+
+func TestInteropV1UnmarshalWithV2_DeterministicEmpty(t *testing.T) {
+	expectedV2Empty := &Request{}
+	for _, tc := range []struct{ incoming *prompb.WriteRequest }{
+		{
+			incoming: &prompb.WriteRequest{},
+		},
+		{
+			incoming: &prompb.WriteRequest{
+				Timeseries: []prompb.TimeSeries{
+					{
+						Labels:  []prompb.Label{{Name: "__name__", Value: "metric1"}},
+						Samples: []prompb.Sample{{Value: 21.4, Timestamp: time.Now().UnixMilli()}},
+					},
+				},
+			},
+			// NOTE: Without reserved fields, results in corrupted v2.Request.Symbols.
+		},
+	} {
+		t.Run("", func(t *testing.T) {
+			in, err := proto.Marshal(tc.incoming)
+			require.NoError(t, err)
+
+			// Test accidental unmarshal of v1 payload with v2 proto.
+			out := &Request{}
+			require.NoError(t, proto.Unmarshal(in, out))
+
+			// Drop unknowns, we expect them when incoming payload had some fields.
+			// This field & method will be likely gone after gogo removal.
+			out.XXX_unrecognized = nil // NOTE: out.XXX_DiscardUnknown() does not work with nullables.
+
+			require.Equal(t, expectedV2Empty, out)
+		})
+	}
+}

prompb/rwcommon/codec_test.go

@@ -0,0 +1,298 @@
+// Copyright 2024 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rwcommon
+
+import (
+	"testing"
+
+	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/prometheus/model/histogram"
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/model/metadata"
+	"github.com/prometheus/prometheus/prompb"
+	writev2 "github.com/prometheus/prometheus/prompb/io/prometheus/write/v2"
+)
+
+func TestToLabels(t *testing.T) {
+	expected := labels.FromStrings("__name__", "metric1", "foo", "bar")
+
+	t.Run("v1", func(t *testing.T) {
+		ts := prompb.TimeSeries{Labels: []prompb.Label{{Name: "__name__", Value: "metric1"}, {Name: "foo", Value: "bar"}}}
+		b := labels.NewScratchBuilder(2)
+		require.Equal(t, expected, ts.ToLabels(&b, nil))
+		require.Equal(t, ts.Labels, prompb.FromLabels(expected, nil))
+		require.Equal(t, ts.Labels, prompb.FromLabels(expected, ts.Labels))
+	})
+	t.Run("v2", func(t *testing.T) {
+		v2Symbols := []string{"", "__name__", "metric1", "foo", "bar"}
+		ts := writev2.TimeSeries{LabelsRefs: []uint32{1, 2, 3, 4}}
+		b := labels.NewScratchBuilder(2)
+		require.Equal(t, expected, ts.ToLabels(&b, v2Symbols))
+		// No need for FromLabels in our prod code as we use symbol table to do so.
+	})
+}
+
+func TestFromMetadataType(t *testing.T) {
+	for _, tc := range []struct {
+		desc       string
+		input      model.MetricType
+		expectedV1 prompb.MetricMetadata_MetricType
+		expectedV2 writev2.Metadata_MetricType
+	}{
+		{
+			desc:       "with a single-word metric",
+			input:      model.MetricTypeCounter,
+			expectedV1: prompb.MetricMetadata_COUNTER,
+			expectedV2: writev2.Metadata_METRIC_TYPE_COUNTER,
+		},
+		{
+			desc:       "with a two-word metric",
+			input:      model.MetricTypeStateset,
+			expectedV1: prompb.MetricMetadata_STATESET,
+			expectedV2: writev2.Metadata_METRIC_TYPE_STATESET,
+		},
+		{
+			desc:       "with an unknown metric",
+			input:      "not-known",
+			expectedV1: prompb.MetricMetadata_UNKNOWN,
+			expectedV2: writev2.Metadata_METRIC_TYPE_UNSPECIFIED,
+		},
+	} {
+		t.Run(tc.desc, func(t *testing.T) {
+			t.Run("v1", func(t *testing.T) {
+				require.Equal(t, tc.expectedV1, prompb.FromMetadataType(tc.input))
+			})
+			t.Run("v2", func(t *testing.T) {
+				require.Equal(t, tc.expectedV2, writev2.FromMetadataType(tc.input))
+			})
+		})
+	}
+}
+
+func TestToMetadata(t *testing.T) {
+	sym := writev2.NewSymbolTable()
+
+	for _, tc := range []struct {
+		input    writev2.Metadata
+		expected metadata.Metadata
+	}{
+		{
+			input: writev2.Metadata{},
+			expected: metadata.Metadata{
+				Type: model.MetricTypeUnknown,
+			},
+		},
+		{
+			input: writev2.Metadata{
+				Type: 12414, // Unknown.
+			},
+			expected: metadata.Metadata{
+				Type: model.MetricTypeUnknown,
+			},
+		},
+		{
+			input: writev2.Metadata{
+				Type:    writev2.Metadata_METRIC_TYPE_COUNTER,
+				HelpRef: sym.Symbolize("help1"),
+				UnitRef: sym.Symbolize("unit1"),
+			},
+			expected: metadata.Metadata{
+				Type: model.MetricTypeCounter,
+				Help: "help1",
+				Unit: "unit1",
+			},
+		},
+		{
+			input: writev2.Metadata{
+				Type:    writev2.Metadata_METRIC_TYPE_STATESET,
+				HelpRef: sym.Symbolize("help2"),
+			},
+			expected: metadata.Metadata{
+				Type: model.MetricTypeStateset,
+				Help: "help2",
+			},
+		},
+	} {
+		t.Run("", func(t *testing.T) {
+			ts := writev2.TimeSeries{Metadata: tc.input}
+			require.Equal(t, tc.expected, ts.ToMetadata(sym.Symbols()))
+		})
+	}
+}
+
+func TestToHistogram_Empty(t *testing.T) {
+	t.Run("v1", func(t *testing.T) {
+		require.NotNilf(t, prompb.Histogram{}.ToIntHistogram(), "")
+		require.NotNilf(t, prompb.Histogram{}.ToFloatHistogram(), "")
+	})
+	t.Run("v2", func(t *testing.T) {
+		require.NotNilf(t, writev2.Histogram{}.ToIntHistogram(), "")
+		require.NotNilf(t, writev2.Histogram{}.ToFloatHistogram(), "")
+	})
+}
+
+// NOTE(bwplotka): This is technically not a valid histogram, but it represents
+// important cases to test when copying or converting to/from int/float histograms.
+func testIntHistogram() histogram.Histogram {
+	return histogram.Histogram{
+		CounterResetHint: histogram.GaugeType,
+		Schema:           1,
+		Count:            19,
+		Sum:              2.7,
+		ZeroThreshold:    1e-128,
+		PositiveSpans: []histogram.Span{
+			{Offset: 0, Length: 4},
+			{Offset: 0, Length: 0},
+			{Offset: 0, Length: 3},
+		},
+		PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 0},
+		NegativeSpans: []histogram.Span{
+			{Offset: 0, Length: 5},
+			{Offset: 1, Length: 0},
+			{Offset: 0, Length: 1},
+		},
+		NegativeBuckets: []int64{1, 2, -2, 1, -1, 0},
+		CustomValues:    []float64{21421, 523},
+	}
+}
+
+// NOTE(bwplotka): This is technically not a valid histogram, but it represents
+// important cases to test when copying or converting to/from int/float histograms.
+func testFloatHistogram() histogram.FloatHistogram {
+	return histogram.FloatHistogram{
+		CounterResetHint: histogram.GaugeType,
+		Schema:           1,
+		Count:            19,
+		Sum:              2.7,
+		ZeroThreshold:    1e-128,
+		PositiveSpans: []histogram.Span{
+			{Offset: 0, Length: 4},
+			{Offset: 0, Length: 0},
+			{Offset: 0, Length: 3},
+		},
+		PositiveBuckets: []float64{1, 3, 1, 2, 1, 1, 1},
+		NegativeSpans: []histogram.Span{
+			{Offset: 0, Length: 5},
+			{Offset: 1, Length: 0},
+			{Offset: 0, Length: 1},
+		},
+		NegativeBuckets: []float64{1, 3, 1, 2, 1, 1},
+		CustomValues:    []float64{21421, 523},
+	}
+}
+
+func TestFromIntToFloatOrIntHistogram(t *testing.T) {
+	t.Run("v1", func(t *testing.T) {
+		// v1 does not support nhcb.
+		testIntHistWithoutNHCB := testIntHistogram()
+		testIntHistWithoutNHCB.CustomValues = nil
+		testFloatHistWithoutNHCB := testFloatHistogram()
+		testFloatHistWithoutNHCB.CustomValues = nil
+
+		h := prompb.FromIntHistogram(123, &testIntHistWithoutNHCB)
+		require.False(t, h.IsFloatHistogram())
+		require.Equal(t, int64(123), h.Timestamp)
+		require.Equal(t, testIntHistWithoutNHCB, *h.ToIntHistogram())
+		require.Equal(t, testFloatHistWithoutNHCB, *h.ToFloatHistogram())
+	})
+	t.Run("v2", func(t *testing.T) {
+		testIntHist := testIntHistogram()
+		testFloatHist := testFloatHistogram()
+
+		h := writev2.FromIntHistogram(123, &testIntHist)
+		require.False(t, h.IsFloatHistogram())
+		require.Equal(t, int64(123), h.Timestamp)
+		require.Equal(t, testIntHist, *h.ToIntHistogram())
+		require.Equal(t, testFloatHist, *h.ToFloatHistogram())
+	})
+}
+
+func TestFromFloatToFloatHistogram(t *testing.T) {
+	t.Run("v1", func(t *testing.T) {
+		// v1 does not support nhcb.
+		testFloatHistWithoutNHCB := testFloatHistogram()
+		testFloatHistWithoutNHCB.CustomValues = nil
+
+		h := prompb.FromFloatHistogram(123, &testFloatHistWithoutNHCB)
+		require.True(t, h.IsFloatHistogram())
+		require.Equal(t, int64(123), h.Timestamp)
+		require.Nil(t, h.ToIntHistogram())
+		require.Equal(t, testFloatHistWithoutNHCB, *h.ToFloatHistogram())
+	})
+	t.Run("v2", func(t *testing.T) {
+		testFloatHist := testFloatHistogram()
+
+		h := writev2.FromFloatHistogram(123, &testFloatHist)
+		require.True(t, h.IsFloatHistogram())
+		require.Equal(t, int64(123), h.Timestamp)
+		require.Nil(t, h.ToIntHistogram())
+		require.Equal(t, testFloatHist, *h.ToFloatHistogram())
+	})
+}
+
+func TestFromIntOrFloatHistogram_ResetHint(t *testing.T) {
+	for _, tc := range []struct {
+		input      histogram.CounterResetHint
+		expectedV1 prompb.Histogram_ResetHint
+		expectedV2 writev2.Histogram_ResetHint
+	}{
+		{
+			input:      histogram.UnknownCounterReset,
+			expectedV1: prompb.Histogram_UNKNOWN,
+			expectedV2: writev2.Histogram_RESET_HINT_UNSPECIFIED,
+		},
+		{
+			input:      histogram.CounterReset,
+			expectedV1: prompb.Histogram_YES,
+			expectedV2: writev2.Histogram_RESET_HINT_YES,
+		},
+		{
+			input:      histogram.NotCounterReset,
+			expectedV1: prompb.Histogram_NO,
+			expectedV2: writev2.Histogram_RESET_HINT_NO,
+		},
+		{
+			input:      histogram.GaugeType,
+			expectedV1: prompb.Histogram_GAUGE,
+			expectedV2: writev2.Histogram_RESET_HINT_GAUGE,
+		},
+	} {
+		t.Run("", func(t *testing.T) {
+			t.Run("v1", func(t *testing.T) {
+				h := testIntHistogram()
+				h.CounterResetHint = tc.input
+				got := prompb.FromIntHistogram(1337, &h)
+				require.Equal(t, tc.expectedV1, got.GetResetHint())
+
+				fh := testFloatHistogram()
+				fh.CounterResetHint = tc.input
+				got2 := prompb.FromFloatHistogram(1337, &fh)
+				require.Equal(t, tc.expectedV1, got2.GetResetHint())
+			})
+			t.Run("v2", func(t *testing.T) {
+				h := testIntHistogram()
+				h.CounterResetHint = tc.input
+				got := writev2.FromIntHistogram(1337, &h)
+				require.Equal(t, tc.expectedV2, got.GetResetHint())
+
+				fh := testFloatHistogram()
+				fh.CounterResetHint = tc.input
+				got2 := writev2.FromFloatHistogram(1337, &fh)
+				require.Equal(t, tc.expectedV2, got2.GetResetHint())
+			})
+		})
+	}
+}

promql/bench_test.go

@@ -189,6 +189,21 @@ func rangeQueryCases() []benchCase {
 		{
 			expr: "topk(5, a_X)",
 		},
+		{
+			expr: "limitk(1, a_X)",
+		},
+		{
+			expr: "limitk(5, a_X)",
+		},
+		{
+			expr: "limit_ratio(0.1, a_X)",
+		},
+		{
+			expr: "limit_ratio(0.5, a_X)",
+		},
+		{
+			expr: "limit_ratio(-0.5, a_X)",
+		},
 		// Combinations.
 		{
 			expr: "rate(a_X[1m]) + rate(b_X[1m])",

promql/engine.go

@@ -1329,7 +1329,7 @@ func (ev *evaluator) rangeEvalAgg(aggExpr *parser.AggregateExpr, sortedGrouping
 		index, ok := groupToResultIndex[groupingKey]
 		// Add a new group if it doesn't exist.
 		if !ok {
-			if aggExpr.Op != parser.TOPK && aggExpr.Op != parser.BOTTOMK {
+			if aggExpr.Op != parser.TOPK && aggExpr.Op != parser.BOTTOMK && aggExpr.Op != parser.LIMITK && aggExpr.Op != parser.LIMIT_RATIO {
 				m := generateGroupingLabels(enh, series.Metric, aggExpr.Without, sortedGrouping)
 				result = append(result, Series{Metric: m})
 			}
@@ -1342,9 +1342,10 @@ func (ev *evaluator) rangeEvalAgg(aggExpr *parser.AggregateExpr, sortedGrouping
 	groups := make([]groupedAggregation, groupCount)
 
 	var k int
+	var ratio float64
 	var seriess map[uint64]Series
 	switch aggExpr.Op {
-	case parser.TOPK, parser.BOTTOMK:
+	case parser.TOPK, parser.BOTTOMK, parser.LIMITK:
 		if !convertibleToInt64(param) {
 			ev.errorf("Scalar value %v overflows int64", param)
 		}
@@ -1356,6 +1357,23 @@ func (ev *evaluator) rangeEvalAgg(aggExpr *parser.AggregateExpr, sortedGrouping
 			return nil, warnings
 		}
 		seriess = make(map[uint64]Series, len(inputMatrix)) // Output series by series hash.
+	case parser.LIMIT_RATIO:
+		if math.IsNaN(param) {
+			ev.errorf("Ratio value %v is NaN", param)
+		}
+		switch {
+		case param == 0:
+			return nil, warnings
+		case param < -1.0:
+			ratio = -1.0
+			warnings.Add(annotations.NewInvalidRatioWarning(param, ratio, aggExpr.Param.PositionRange()))
+		case param > 1.0:
+			ratio = 1.0
+			warnings.Add(annotations.NewInvalidRatioWarning(param, ratio, aggExpr.Param.PositionRange()))
+		default:
+			ratio = param
+		}
+		seriess = make(map[uint64]Series, len(inputMatrix)) // Output series by series hash.
 	case parser.QUANTILE:
 		if math.IsNaN(param) || param < 0 || param > 1 {
 			warnings.Add(annotations.NewInvalidQuantileWarning(param, aggExpr.Param.PositionRange()))
@@ -1373,11 +1391,12 @@ func (ev *evaluator) rangeEvalAgg(aggExpr *parser.AggregateExpr, sortedGrouping
 		enh.Ts = ts
 		var ws annotations.Annotations
 		switch aggExpr.Op {
-		case parser.TOPK, parser.BOTTOMK:
-			result, ws = ev.aggregationK(aggExpr, k, inputMatrix, seriesToResult, groups, enh, seriess)
+		case parser.TOPK, parser.BOTTOMK, parser.LIMITK, parser.LIMIT_RATIO:
+			result, ws = ev.aggregationK(aggExpr, k, ratio, inputMatrix, seriesToResult, groups, enh, seriess)
 			// If this could be an instant query, shortcut so as not to change sort order.
 			if ev.endTimestamp == ev.startTimestamp {
-				return result, ws
+				warnings.Merge(ws)
+				return result, warnings
 			}
 		default:
 			ws = ev.aggregation(aggExpr, param, inputMatrix, result, seriesToResult, groups, enh)
@@ -1392,7 +1411,7 @@ func (ev *evaluator) rangeEvalAgg(aggExpr *parser.AggregateExpr, sortedGrouping
 
 	// Assemble the output matrix. By the time we get here we know we don't have too many samples.
 	switch aggExpr.Op {
-	case parser.TOPK, parser.BOTTOMK:
+	case parser.TOPK, parser.BOTTOMK, parser.LIMITK, parser.LIMIT_RATIO:
 		result = make(Matrix, 0, len(seriess))
 		for _, ss := range seriess {
 			result = append(result, ss)
@@ -2765,14 +2784,15 @@ func vectorElemBinop(op parser.ItemType, lhs, rhs float64, hlhs, hrhs *histogram
 }
 
 type groupedAggregation struct {
-	seen           bool // Was this output groups seen in the input at this timestamp.
-	hasFloat       bool // Has at least 1 float64 sample aggregated.
-	hasHistogram   bool // Has at least 1 histogram sample aggregated.
-	floatValue     float64
-	histogramValue *histogram.FloatHistogram
-	floatMean      float64 // Mean, or "compensating value" for Kahan summation.
-	groupCount     int
-	heap           vectorByValueHeap
+	seen              bool // Was this output groups seen in the input at this timestamp.
+	hasFloat          bool // Has at least 1 float64 sample aggregated.
+	hasHistogram      bool // Has at least 1 histogram sample aggregated.
+	floatValue        float64
+	histogramValue    *histogram.FloatHistogram
+	floatMean         float64 // Mean, or "compensating value" for Kahan summation.
+	groupCount        int
+	groupAggrComplete bool // Used by LIMITK to short-cut series loop when we've reached K elem on every group
+	heap              vectorByValueHeap
 }
 
 // aggregation evaluates sum, avg, count, stdvar, stddev or quantile at one timestep on inputMatrix.
@@ -2969,19 +2989,22 @@ func (ev *evaluator) aggregation(e *parser.AggregateExpr, q float64, inputMatrix
 	return annos
 }
 
-// aggregationK evaluates topk or bottomk at one timestep on inputMatrix.
+// aggregationK evaluates topk, bottomk, limitk, or limit_ratio at one timestep on inputMatrix.
 // Output that has the same labels as the input, but just k of them per group.
 // seriesToResult maps inputMatrix indexes to groups indexes.
-// For an instant query, returns a Matrix in descending order for topk or ascending for bottomk.
+// For an instant query, returns a Matrix in descending order for topk or ascending for bottomk, or without any order for limitk / limit_ratio.
 // For a range query, aggregates output in the seriess map.
-func (ev *evaluator) aggregationK(e *parser.AggregateExpr, k int, inputMatrix Matrix, seriesToResult []int, groups []groupedAggregation, enh *EvalNodeHelper, seriess map[uint64]Series) (Matrix, annotations.Annotations) {
+func (ev *evaluator) aggregationK(e *parser.AggregateExpr, k int, r float64, inputMatrix Matrix, seriesToResult []int, groups []groupedAggregation, enh *EvalNodeHelper, seriess map[uint64]Series) (Matrix, annotations.Annotations) {
 	op := e.Op
 	var s Sample
 	var annos annotations.Annotations
+	// Used to short-cut the loop for LIMITK if we already collected k elements for every group
+	groupsRemaining := len(groups)
 	for i := range groups {
 		groups[i].seen = false
 	}
 
+seriesLoop:
 	for si := range inputMatrix {
 		f, _, ok := ev.nextValues(enh.Ts, &inputMatrix[si])
 		if !ok {
@@ -2992,11 +3015,23 @@ func (ev *evaluator) aggregationK(e *parser.AggregateExpr, k int, inputMatrix Ma
 		group := &groups[seriesToResult[si]]
 		// Initialize this group if it's the first time we've seen it.
 		if !group.seen {
-			*group = groupedAggregation{
-				seen: true,
-				heap: make(vectorByValueHeap, 1, k),
+			// LIMIT_RATIO is a special case, as we may not add this very sample to the heap,
+			// while we also don't know the final size of it.
+			if op == parser.LIMIT_RATIO {
+				*group = groupedAggregation{
+					seen: true,
+					heap: make(vectorByValueHeap, 0),
+				}
+				if ratiosampler.AddRatioSample(r, &s) {
+					heap.Push(&group.heap, &s)
+				}
+			} else {
+				*group = groupedAggregation{
+					seen: true,
+					heap: make(vectorByValueHeap, 1, k),
+				}
+				group.heap[0] = s
 			}
-			group.heap[0] = s
 			continue
 		}
 
@@ -3027,6 +3062,26 @@ func (ev *evaluator) aggregationK(e *parser.AggregateExpr, k int, inputMatrix Ma
 				}
 			}
 
+		case parser.LIMITK:
+			if len(group.heap) < k {
+				heap.Push(&group.heap, &s)
+			}
+			// LIMITK optimization: early break if we've added K elem to _every_ group,
+			// especially useful for large timeseries where the user is exploring labels via e.g.
+			// limitk(10, my_metric)
+			if !group.groupAggrComplete && len(group.heap) == k {
+				group.groupAggrComplete = true
+				groupsRemaining--
+				if groupsRemaining == 0 {
+					break seriesLoop
+				}
+			}
+
+		case parser.LIMIT_RATIO:
+			if ratiosampler.AddRatioSample(r, &s) {
+				heap.Push(&group.heap, &s)
+			}
+
 		default:
 			panic(fmt.Errorf("expected aggregation operator but got %q", op))
 		}
@@ -3076,6 +3131,11 @@ func (ev *evaluator) aggregationK(e *parser.AggregateExpr, k int, inputMatrix Ma
 			for _, v := range aggr.heap {
 				add(v.Metric, v.F)
 			}
+
+		case parser.LIMITK, parser.LIMIT_RATIO:
+			for _, v := range aggr.heap {
+				add(v.Metric, v.F)
+			}
 		}
 	}
 
@@ -3401,6 +3461,12 @@ func setOffsetForAtModifier(evalTime int64, expr parser.Expr) {
 // required for correctness.
 func detectHistogramStatsDecoding(expr parser.Expr) {
 	parser.Inspect(expr, func(node parser.Node, path []parser.Node) error {
+		if n, ok := node.(*parser.BinaryExpr); ok {
+			detectHistogramStatsDecoding(n.LHS)
+			detectHistogramStatsDecoding(n.RHS)
+			return fmt.Errorf("stop")
+		}
+
 		n, ok := (node).(*parser.VectorSelector)
 		if !ok {
 			return nil
@@ -3430,6 +3496,56 @@ func makeInt64Pointer(val int64) *int64 {
 	return valp
 }
 
+// Add RatioSampler interface to allow unit-testing (previously: Randomizer).
+type RatioSampler interface {
+	// Return this sample "offset" between [0.0, 1.0]
+	sampleOffset(ts int64, sample *Sample) float64
+	AddRatioSample(r float64, sample *Sample) bool
+}
+
+// Use Hash(labels.String()) / maxUint64 as a "deterministic"
+// value in [0.0, 1.0].
+type HashRatioSampler struct{}
+
+var ratiosampler RatioSampler = NewHashRatioSampler()
+
+func NewHashRatioSampler() *HashRatioSampler {
+	return &HashRatioSampler{}
+}
+
+func (s *HashRatioSampler) sampleOffset(ts int64, sample *Sample) float64 {
+	const (
+		float64MaxUint64 = float64(math.MaxUint64)
+	)
+	return float64(sample.Metric.Hash()) / float64MaxUint64
+}
+
+func (s *HashRatioSampler) AddRatioSample(ratioLimit float64, sample *Sample) bool {
+	// If ratioLimit >= 0: add sample if sampleOffset is lesser than ratioLimit
+	//
+	// 0.0        ratioLimit                1.0
+	//  [---------|--------------------------]
+	//  [#########...........................]
+	//
+	// e.g.:
+	//   sampleOffset==0.3 && ratioLimit==0.4
+	//     0.3 < 0.4 ? --> add sample
+	//
+	// Else if ratioLimit < 0: add sample if rand() return the "complement" of ratioLimit>=0 case
+	// (loosely similar behavior to negative array index in other programming languages)
+	//
+	// 0.0       1+ratioLimit               1.0
+	//  [---------|--------------------------]
+	//  [.........###########################]
+	//
+	// e.g.:
+	//   sampleOffset==0.3 && ratioLimit==-0.6
+	//     0.3 >= 0.4 ? --> don't add sample
+	sampleOffset := s.sampleOffset(sample.T, sample)
+	return (ratioLimit >= 0 && sampleOffset < ratioLimit) ||
+		(ratioLimit < 0 && sampleOffset >= (1.0+ratioLimit))
+}
+
 type histogramStatsSeries struct {
 	storage.Series
 }

promql/engine_test.go

@@ -48,6 +48,8 @@ const (
 )
 
 func TestMain(m *testing.M) {
+	// Enable experimental functions testing
+	parser.EnableExperimentalFunctions = true
 	goleak.VerifyTestMain(m)
 }
 
@@ -236,11 +238,11 @@ func (q *errQuerier) Select(context.Context, bool, *storage.SelectHints, ...*lab
 	return errSeriesSet{err: q.err}
 }
 
-func (*errQuerier) LabelValues(context.Context, string, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+func (*errQuerier) LabelValues(context.Context, string, *storage.LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
 	return nil, nil, nil
 }
 
-func (*errQuerier) LabelNames(context.Context, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+func (*errQuerier) LabelNames(context.Context, *storage.LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
 	return nil, nil, nil
 }
 func (*errQuerier) Close() error { return nil }

promql/parser/generated_parser.y

@@ -43,7 +43,6 @@ import (
     int         int64
     uint        uint64
     float       float64
-    duration    time.Duration
 }
 
 
@@ -126,6 +125,8 @@ STDDEV
 STDVAR
 SUM
 TOPK
+LIMITK
+LIMIT_RATIO
 %token	aggregatorsEnd
 
 // Keywords.
@@ -174,8 +175,7 @@ START_METRIC_SELECTOR
 %type <int> int
 %type <uint> uint
 %type <float> number series_value signed_number signed_or_unsigned_number
-%type <node> step_invariant_expr aggregate_expr aggregate_modifier bin_modifier binary_expr bool_modifier expr function_call function_call_args function_call_body group_modifiers label_matchers matrix_selector number_literal offset_expr on_or_ignoring paren_expr string_literal subquery_expr unary_expr vector_selector
-%type <duration> duration maybe_duration
+%type <node> step_invariant_expr aggregate_expr aggregate_modifier bin_modifier binary_expr bool_modifier expr function_call function_call_args function_call_body group_modifiers label_matchers matrix_selector number_duration_literal offset_expr on_or_ignoring paren_expr string_literal subquery_expr unary_expr vector_selector
 
 %start start
 
@@ -216,7 +216,7 @@ expr            :
                 | binary_expr
                 | function_call
                 | matrix_selector
-                | number_literal
+                | number_duration_literal
                 | offset_expr
                 | paren_expr
                 | string_literal
@@ -413,18 +413,22 @@ paren_expr      : LEFT_PAREN expr RIGHT_PAREN
  * Offset modifiers.
  */
 
-offset_expr: expr OFFSET duration
+offset_expr: expr OFFSET number_duration_literal
                         {
-                        yylex.(*parser).addOffset($1, $3)
-                        $$ = $1
+  		            numLit, _ := $3.(*NumberLiteral)
+      		            dur := time.Duration(numLit.Val * 1000) * time.Millisecond
+                 	    yylex.(*parser).addOffset($1, dur)
+                            $$ = $1
                         }
-                | expr OFFSET SUB duration
+                | expr OFFSET SUB number_duration_literal
                         {
-                        yylex.(*parser).addOffset($1, -$4)
-                        $$ = $1
+			    numLit, _ := $4.(*NumberLiteral)
+		            dur := time.Duration(numLit.Val * 1000) * time.Millisecond
+			    yylex.(*parser).addOffset($1, -dur)
+                            $$ = $1
                         }
                 | expr OFFSET error
-                        { yylex.(*parser).unexpected("offset", "duration"); $$ = $1 }
+                        { yylex.(*parser).unexpected("offset", "number or duration"); $$ = $1 }
                 ;
 /*
  * @ modifiers.
@@ -450,7 +454,7 @@ at_modifier_preprocessors: START | END;
  * Subquery and range selectors.
  */
 
-matrix_selector : expr LEFT_BRACKET duration RIGHT_BRACKET
+matrix_selector : expr LEFT_BRACKET number_duration_literal RIGHT_BRACKET
                         {
                         var errMsg string
                         vs, ok := $1.(*VectorSelector)
@@ -467,32 +471,44 @@ matrix_selector : expr LEFT_BRACKET duration RIGHT_BRACKET
                                 yylex.(*parser).addParseErrf(errRange, errMsg)
                         }
 
+			numLit, _ := $3.(*NumberLiteral)
                         $$ = &MatrixSelector{
                                 VectorSelector: $1.(Expr),
-                                Range: $3,
+                                Range: time.Duration(numLit.Val * 1000) * time.Millisecond,
                                 EndPos: yylex.(*parser).lastClosing,
                         }
                         }
                 ;
 
-subquery_expr   : expr LEFT_BRACKET duration COLON maybe_duration RIGHT_BRACKET
+subquery_expr   : expr LEFT_BRACKET number_duration_literal COLON number_duration_literal RIGHT_BRACKET
                         {
+			numLitRange, _ := $3.(*NumberLiteral)
+			numLitStep, _ := $5.(*NumberLiteral)
                         $$ = &SubqueryExpr{
                                 Expr:  $1.(Expr),
-                                Range: $3,
-                                Step:  $5,
-
+                                Range: time.Duration(numLitRange.Val * 1000) * time.Millisecond,
+                                Step:  time.Duration(numLitStep.Val * 1000) * time.Millisecond,
                                 EndPos: $6.Pos + 1,
                         }
                         }
-                | expr LEFT_BRACKET duration COLON duration error
+                | expr LEFT_BRACKET number_duration_literal COLON RIGHT_BRACKET
+		        {
+		          numLitRange, _ := $3.(*NumberLiteral)
+		          $$ = &SubqueryExpr{
+		          Expr:  $1.(Expr),
+		          Range: time.Duration(numLitRange.Val * 1000) * time.Millisecond,
+		          Step:  0,
+		          EndPos: $5.Pos + 1,
+		          }
+		        }
+                | expr LEFT_BRACKET number_duration_literal COLON number_duration_literal error
                         { yylex.(*parser).unexpected("subquery selector", "\"]\""); $$ = $1 }
-                | expr LEFT_BRACKET duration COLON error
-                        { yylex.(*parser).unexpected("subquery selector", "duration or \"]\""); $$ = $1 }
-                | expr LEFT_BRACKET duration error
+                | expr LEFT_BRACKET number_duration_literal COLON error
+                        { yylex.(*parser).unexpected("subquery selector", "number or duration or \"]\""); $$ = $1 }
+                | expr LEFT_BRACKET number_duration_literal error
                         { yylex.(*parser).unexpected("subquery or range", "\":\" or \"]\""); $$ = $1 }
                 | expr LEFT_BRACKET error
-                        { yylex.(*parser).unexpected("subquery selector", "duration"); $$ = $1 }
+		        { yylex.(*parser).unexpected("subquery selector", "number or duration"); $$ = $1 }
                 ;
 
 /*
@@ -609,7 +625,7 @@ metric          : metric_identifier label_set
                 ;
 
 
-metric_identifier: AVG | BOTTOMK | BY | COUNT | COUNT_VALUES | GROUP | IDENTIFIER |  LAND | LOR | LUNLESS | MAX | METRIC_IDENTIFIER | MIN | OFFSET | QUANTILE | STDDEV | STDVAR | SUM | TOPK | WITHOUT | START | END;
+metric_identifier: AVG | BOTTOMK | BY | COUNT | COUNT_VALUES | GROUP | IDENTIFIER |  LAND | LOR | LUNLESS | MAX | METRIC_IDENTIFIER | MIN | OFFSET | QUANTILE | STDDEV | STDVAR | SUM | TOPK | WITHOUT | START | END | LIMITK | LIMIT_RATIO;
 
 label_set       : LEFT_BRACE label_set_list RIGHT_BRACE
                         { $$ = labels.New($2...) }
@@ -851,10 +867,10 @@ bucket_set_list : bucket_set_list SPACE number
  * Keyword lists.
  */
 
-aggregate_op    : AVG | BOTTOMK | COUNT | COUNT_VALUES | GROUP | MAX | MIN | QUANTILE | STDDEV | STDVAR | SUM | TOPK ;
+aggregate_op    : AVG | BOTTOMK | COUNT | COUNT_VALUES | GROUP | MAX | MIN | QUANTILE | STDDEV | STDVAR | SUM | TOPK | LIMITK | LIMIT_RATIO;
 
 // Inside of grouping options label names can be recognized as keywords by the lexer. This is a list of keywords that could also be a label name.
-maybe_label     : AVG | BOOL | BOTTOMK | BY | COUNT | COUNT_VALUES | GROUP | GROUP_LEFT | GROUP_RIGHT | IDENTIFIER | IGNORING | LAND | LOR | LUNLESS | MAX | METRIC_IDENTIFIER | MIN | OFFSET | ON | QUANTILE | STDDEV | STDVAR | SUM | TOPK | START | END | ATAN2;
+maybe_label     : AVG | BOOL | BOTTOMK | BY | COUNT | COUNT_VALUES | GROUP | GROUP_LEFT | GROUP_RIGHT | IDENTIFIER | IGNORING | LAND | LOR | LUNLESS | MAX | METRIC_IDENTIFIER | MIN | OFFSET | ON | QUANTILE | STDDEV | STDVAR | SUM | TOPK | START | END | ATAN2 | LIMITK | LIMIT_RATIO;
 
 unary_op        : ADD | SUB;
 
@@ -864,16 +880,43 @@ match_op        : EQL | NEQ | EQL_REGEX | NEQ_REGEX ;
  * Literals.
  */
 
-number_literal  : NUMBER
+number_duration_literal  : NUMBER
                         {
-                        $$ = &NumberLiteral{
+                            $$ = &NumberLiteral{
                                 Val:           yylex.(*parser).number($1.Val),
                                 PosRange: $1.PositionRange(),
+                            }
                         }
+                        | DURATION
+			{
+                            var err error
+                            var dur time.Duration
+                            dur, err = parseDuration($1.Val)
+                            if err != nil {
+                                    yylex.(*parser).addParseErr($1.PositionRange(), err)
+                            }
+                            $$ = &NumberLiteral{
+			            Val:      dur.Seconds(),
+			            PosRange: $1.PositionRange(),
+                            }
                         }
                 ;
 
-number          : NUMBER { $$ = yylex.(*parser).number($1.Val) } ;
+number          : NUMBER
+                {
+		  $$ = yylex.(*parser).number($1.Val)
+		}
+                | DURATION
+		{
+		  var err error
+		  var dur time.Duration
+		  dur, err = parseDuration($1.Val)
+		  if err != nil {
+		      yylex.(*parser).addParseErr($1.PositionRange(), err)
+		  }
+		  $$ = dur.Seconds()
+		}
+                ;
 
 signed_number   : ADD number { $$ = $2 }
                 | SUB number { $$ = -$2 }
@@ -895,17 +938,6 @@ int             : SUB uint { $$ = -int64($2) }
                 | uint { $$ = int64($1) }
                 ;
 
-duration        : DURATION
-                        {
-                        var err error
-                        $$, err = parseDuration($1.Val)
-                        if err != nil {
-                                yylex.(*parser).addParseErr($1.PositionRange(), err)
-                        }
-                        }
-                ;
-
-
 string_literal  : STRING
                         {
                         $$ = &StringLiteral{
@@ -929,11 +961,6 @@ string_identifier  : STRING
  * Wrappers for optional arguments.
  */
 
-maybe_duration  : /* empty */
-                        {$$ = 0}
-                | duration
-                ;
-
 maybe_grouping_labels: /* empty */ { $$ = nil }
                 | grouping_labels
                 ;

promql/parser/lex.go

@@ -65,7 +65,7 @@ func (i ItemType) IsAggregator() bool { return i > aggregatorsStart && i < aggre
 // IsAggregatorWithParam returns true if the Item is an aggregator that takes a parameter.
 // Returns false otherwise.
 func (i ItemType) IsAggregatorWithParam() bool {
-	return i == TOPK || i == BOTTOMK || i == COUNT_VALUES || i == QUANTILE
+	return i == TOPK || i == BOTTOMK || i == COUNT_VALUES || i == QUANTILE || i == LIMITK || i == LIMIT_RATIO
 }
 
 // IsKeyword returns true if the Item corresponds to a keyword.
@@ -118,6 +118,8 @@ var key = map[string]ItemType{
 	"bottomk":      BOTTOMK,
 	"count_values": COUNT_VALUES,
 	"quantile":     QUANTILE,
+	"limitk":       LIMITK,
+	"limit_ratio":  LIMIT_RATIO,
 
 	// Keywords.
 	"offset":      OFFSET,
@@ -476,7 +478,7 @@ func lexStatements(l *Lexer) stateFn {
 			skipSpaces(l)
 		}
 		l.bracketOpen = true
-		return lexDuration
+		return lexNumberOrDuration
 	case r == ']':
 		if !l.bracketOpen {
 			return l.errorf("unexpected right bracket %q", r)
@@ -844,18 +846,6 @@ func lexLineComment(l *Lexer) stateFn {
 	return lexStatements
 }
 
-func lexDuration(l *Lexer) stateFn {
-	if l.scanNumber() {
-		return l.errorf("missing unit character in duration")
-	}
-	if !acceptRemainingDuration(l) {
-		return l.errorf("bad duration syntax: %q", l.input[l.start:l.pos])
-	}
-	l.backup()
-	l.emit(DURATION)
-	return lexStatements
-}
-
 // lexNumber scans a number: decimal, hex, oct or float.
 func lexNumber(l *Lexer) stateFn {
 	if !l.scanNumber() {
@@ -907,6 +897,7 @@ func acceptRemainingDuration(l *Lexer) bool {
 // scanNumber scans numbers of different formats. The scanned Item is
 // not necessarily a valid number. This case is caught by the parser.
 func (l *Lexer) scanNumber() bool {
+	initialPos := l.pos
 	// Modify the digit pattern if the number is hexadecimal.
 	digitPattern := "0123456789"
 	// Disallow hexadecimal in series descriptions as the syntax is ambiguous.
@@ -978,7 +969,10 @@ func (l *Lexer) scanNumber() bool {
 		// Handle digits at the end since we already consumed before this loop.
 		l.acceptRun(digitPattern)
 	}
-
+	// Empty string is not a valid number.
+	if l.pos == initialPos {
+		return false
+	}
 	// Next thing must not be alphanumeric unless it's the times token
 	// for series repetitions.
 	if r := l.peek(); (l.seriesDesc && r == 'x') || !isAlphaNumeric(r) {

promql/parser/parse.go

@@ -447,6 +447,10 @@ func (p *parser) newAggregateExpr(op Item, modifier, args Node) (ret *AggregateE
 
 	desiredArgs := 1
 	if ret.Op.IsAggregatorWithParam() {
+		if !EnableExperimentalFunctions && (ret.Op == LIMITK || ret.Op == LIMIT_RATIO) {
+			p.addParseErrf(ret.PositionRange(), "limitk() and limit_ratio() are experimental and must be enabled with --enable-feature=promql-experimental-functions")
+			return
+		}
 		desiredArgs = 2
 
 		ret.Param = arguments[0]
@@ -672,7 +676,7 @@ func (p *parser) checkAST(node Node) (typ ValueType) {
 			p.addParseErrf(n.PositionRange(), "aggregation operator expected in aggregation expression but got %q", n.Op)
 		}
 		p.expectType(n.Expr, ValueTypeVector, "aggregation expression")
-		if n.Op == TOPK || n.Op == BOTTOMK || n.Op == QUANTILE {
+		if n.Op == TOPK || n.Op == BOTTOMK || n.Op == QUANTILE || n.Op == LIMITK || n.Op == LIMIT_RATIO {
 			p.expectType(n.Param, ValueTypeScalar, "aggregation parameter")
 		}
 		if n.Op == COUNT_VALUES {

promql/parser/parse_test.go

@@ -2133,6 +2133,115 @@ var testExpr = []struct {
 			EndPos: 25,
 		},
 	},
+	{
+		input: `test{a="b"}[5m] OFFSET 3600`,
+		expected: &MatrixSelector{
+			VectorSelector: &VectorSelector{
+				Name:           "test",
+				OriginalOffset: 1 * time.Hour,
+				LabelMatchers: []*labels.Matcher{
+					MustLabelMatcher(labels.MatchEqual, "a", "b"),
+					MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "test"),
+				},
+				PosRange: posrange.PositionRange{
+					Start: 0,
+					End:   11,
+				},
+			},
+			Range:  5 * time.Minute,
+			EndPos: 27,
+		},
+	},
+	{
+		input: `foo[3ms] @ 2.345`,
+		expected: &MatrixSelector{
+			VectorSelector: &VectorSelector{
+				Name:      "foo",
+				Timestamp: makeInt64Pointer(2345),
+				LabelMatchers: []*labels.Matcher{
+					MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "foo"),
+				},
+				PosRange: posrange.PositionRange{
+					Start: 0,
+					End:   3,
+				},
+			},
+			Range:  3 * time.Millisecond,
+			EndPos: 16,
+		},
+	},
+	{
+		input: `foo[4s180ms] @ 2.345`,
+		expected: &MatrixSelector{
+			VectorSelector: &VectorSelector{
+				Name:      "foo",
+				Timestamp: makeInt64Pointer(2345),
+				LabelMatchers: []*labels.Matcher{
+					MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "foo"),
+				},
+				PosRange: posrange.PositionRange{
+					Start: 0,
+					End:   3,
+				},
+			},
+			Range:  4*time.Second + 180*time.Millisecond,
+			EndPos: 20,
+		},
+	},
+	{
+		input: `foo[4.18] @ 2.345`,
+		expected: &MatrixSelector{
+			VectorSelector: &VectorSelector{
+				Name:      "foo",
+				Timestamp: makeInt64Pointer(2345),
+				LabelMatchers: []*labels.Matcher{
+					MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "foo"),
+				},
+				PosRange: posrange.PositionRange{
+					Start: 0,
+					End:   3,
+				},
+			},
+			Range:  4*time.Second + 180*time.Millisecond,
+			EndPos: 17,
+		},
+	},
+	{
+		input: `foo[4s18ms] @ 2.345`,
+		expected: &MatrixSelector{
+			VectorSelector: &VectorSelector{
+				Name:      "foo",
+				Timestamp: makeInt64Pointer(2345),
+				LabelMatchers: []*labels.Matcher{
+					MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "foo"),
+				},
+				PosRange: posrange.PositionRange{
+					Start: 0,
+					End:   3,
+				},
+			},
+			Range:  4*time.Second + 18*time.Millisecond,
+			EndPos: 19,
+		},
+	},
+	{
+		input: `foo[4.018] @ 2.345`,
+		expected: &MatrixSelector{
+			VectorSelector: &VectorSelector{
+				Name:      "foo",
+				Timestamp: makeInt64Pointer(2345),
+				LabelMatchers: []*labels.Matcher{
+					MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "foo"),
+				},
+				PosRange: posrange.PositionRange{
+					Start: 0,
+					End:   3,
+				},
+			},
+			Range:  4*time.Second + 18*time.Millisecond,
+			EndPos: 18,
+		},
+	},
 	{
 		input: `test{a="b"}[5y] @ 1603774699`,
 		expected: &MatrixSelector{
@@ -2152,15 +2261,50 @@ var testExpr = []struct {
 			EndPos: 28,
 		},
 	},
+	{
+		input: "test[5]",
+		expected: &MatrixSelector{
+			VectorSelector: &VectorSelector{
+				Name: "test",
+				LabelMatchers: []*labels.Matcher{
+					MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "test"),
+				},
+				PosRange: posrange.PositionRange{
+					Start: 0,
+					End:   4,
+				},
+			},
+			Range:  5 * time.Second,
+			EndPos: 7,
+		},
+	},
+	{
+		input: `some_metric[5m] @ 1m`,
+		expected: &MatrixSelector{
+			VectorSelector: &VectorSelector{
+				Name:      "some_metric",
+				Timestamp: makeInt64Pointer(60000),
+				LabelMatchers: []*labels.Matcher{
+					MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "some_metric"),
+				},
+				PosRange: posrange.PositionRange{
+					Start: 0,
+					End:   11,
+				},
+			},
+			Range:  5 * time.Minute,
+			EndPos: 20,
+		},
+	},
 	{
 		input:  `foo[5mm]`,
 		fail:   true,
-		errMsg: "bad duration syntax: \"5mm\"",
+		errMsg: "bad number or duration syntax: \"5mm\"",
 	},
 	{
 		input:  `foo[5m1]`,
 		fail:   true,
-		errMsg: "bad duration syntax: \"5m1\"",
+		errMsg: "bad number or duration syntax: \"5m1\"",
 	},
 	{
 		input:  `foo[5m:1m1]`,
@@ -2194,17 +2338,12 @@ var testExpr = []struct {
 	{
 		input:  `foo[]`,
 		fail:   true,
-		errMsg: "missing unit character in duration",
+		errMsg: "bad number or duration syntax: \"\"",
 	},
 	{
-		input:  `foo[1]`,
+		input:  `foo[-1]`,
 		fail:   true,
-		errMsg: "missing unit character in duration",
-	},
-	{
-		input:  `some_metric[5m] OFFSET 1`,
-		fail:   true,
-		errMsg: "unexpected number \"1\" in offset, expected duration",
+		errMsg: "bad number or duration syntax: \"\"",
 	},
 	{
 		input:  `some_metric[5m] OFFSET 1mm`,
@@ -2214,18 +2353,13 @@ var testExpr = []struct {
 	{
 		input:  `some_metric[5m] OFFSET`,
 		fail:   true,
-		errMsg: "unexpected end of input in offset, expected duration",
+		errMsg: "unexpected end of input in offset, expected number or duration",
 	},
 	{
 		input:  `some_metric OFFSET 1m[5m]`,
 		fail:   true,
 		errMsg: "1:22: parse error: no offset modifiers allowed before range",
 	},
-	{
-		input:  `some_metric[5m] @ 1m`,
-		fail:   true,
-		errMsg: "1:19: parse error: unexpected duration \"1m\" in @, expected timestamp",
-	},
 	{
 		input:  `some_metric[5m] @`,
 		fail:   true,
@@ -2910,6 +3044,11 @@ var testExpr = []struct {
 		errMsg: "illegal character U+002E '.' in escape sequence",
 	},
 	// Subquery.
+	{
+		input:  `foo{bar="baz"}[`,
+		fail:   true,
+		errMsg: `1:16: parse error: bad number or duration syntax: ""`,
+	},
 	{
 		input: `foo{bar="baz"}[10m:6s]`,
 		expected: &SubqueryExpr{

promql/promql_test.go

@@ -23,6 +23,7 @@ import (
 	"golang.org/x/sync/errgroup"
 
 	"github.com/prometheus/prometheus/promql"
+	"github.com/prometheus/prometheus/promql/parser"
 	"github.com/prometheus/prometheus/promql/promqltest"
 	"github.com/prometheus/prometheus/util/teststorage"
 )
@@ -45,6 +46,8 @@ func TestConcurrentRangeQueries(t *testing.T) {
 		MaxSamples: 50000000,
 		Timeout:    100 * time.Second,
 	}
+	// Enable experimental functions testing
+	parser.EnableExperimentalFunctions = true
 	engine := promql.NewEngine(opts)
 	t.Cleanup(func() {
 		require.NoError(t, engine.Close())

promql/promqltest/testdata/at_modifier.test

@@ -10,22 +10,54 @@ eval instant at 10s metric @ 100
   metric{job="1"} 10
   metric{job="2"} 20
 
+eval instant at 10s metric @ 100s
+  metric{job="1"} 10
+  metric{job="2"} 20
+
+eval instant at 10s metric @ 1m40s
+  metric{job="1"} 10
+  metric{job="2"} 20
+
 eval instant at 10s metric @ 100 offset 50s
   metric{job="1"} 5
   metric{job="2"} 10
 
+eval instant at 10s metric @ 100 offset 50
+  metric{job="1"} 5
+  metric{job="2"} 10
+
 eval instant at 10s metric offset 50s @ 100
   metric{job="1"} 5
   metric{job="2"} 10
 
+eval instant at 10s metric offset 50 @ 100
+  metric{job="1"} 5
+  metric{job="2"} 10
+
 eval instant at 10s metric @ 0 offset -50s
   metric{job="1"} 5
   metric{job="2"} 10
 
+eval instant at 10s metric @ 0 offset -50
+  metric{job="1"} 5
+  metric{job="2"} 10
+
 eval instant at 10s metric offset -50s @ 0
   metric{job="1"} 5
   metric{job="2"} 10
 
+eval instant at 10s metric offset -50 @ 0
+  metric{job="1"} 5
+  metric{job="2"} 10
+
+eval instant at 10s metric @ 0 offset -50s
+  metric{job="1"} 5
+  metric{job="2"} 10
+
+eval instant at 10s metric @ 0 offset -50
+  metric{job="1"} 5
+  metric{job="2"} 10
+
 eval instant at 10s -metric @ 100
   {job="1"} -10
   {job="2"} -20
@@ -48,6 +80,12 @@ eval instant at 25s sum_over_time(metric{job="1"}[100s] @ 100 offset 50s)
 eval instant at 25s sum_over_time(metric{job="1"}[100s] offset 50s @ 100)
   {job="1"} 15
 
+eval instant at 25s sum_over_time(metric{job="1"}[100] @ 100 offset 50)
+  {job="1"} 15
+
+eval instant at 25s sum_over_time(metric{job="1"}[100] offset 50s @ 100)
+  {job="1"} 15
+
 # Different timestamps.
 eval instant at 25s metric{job="1"} @ 50 + metric{job="1"} @ 100
   {job="1"} 15
@@ -58,6 +96,9 @@ eval instant at 25s rate(metric{job="1"}[100s] @ 100) + label_replace(rate(metri
 eval instant at 25s sum_over_time(metric{job="1"}[100s] @ 100) + label_replace(sum_over_time(metric{job="2"}[100s] @ 100), "job", "1", "", "")
   {job="1"} 165
 
+eval instant at 25s sum_over_time(metric{job="1"}[100] @ 100) + label_replace(sum_over_time(metric{job="2"}[100] @ 100), "job", "1", "", "")
+  {job="1"} 165
+
 # Subqueries.
 
 # 10*(1+2+...+9) + 10.
@@ -72,6 +113,10 @@ eval instant at 25s sum_over_time(metric{job="1"}[100s:1s] @ 100 offset 20s)
 eval instant at 25s sum_over_time(metric{job="1"}[100s:1s] offset 20s @ 100)
   {job="1"} 288
 
+# 10*(1+2+...+7) + 8.
+eval instant at 25s sum_over_time(metric{job="1"}[100:1] offset 20 @ 100)
+  {job="1"} 288
+
 # Subquery with different timestamps.
 
 # Since vector selector has timestamp, the result value does not depend on the timestamp of subqueries.

promql/promqltest/testdata/functions.test

@@ -10,6 +10,11 @@ eval instant at 50m resets(http_requests[5m])
 	{path="/bar"} 0
 	{path="/biz"} 0
 
+eval instant at 50m resets(http_requests[300])
+	{path="/foo"} 0
+	{path="/bar"} 0
+	{path="/biz"} 0
+
 eval instant at 50m resets(http_requests[20m])
 	{path="/foo"} 1
 	{path="/bar"} 0
@@ -239,10 +244,16 @@ eval instant at 50m deriv(testcounter_reset_middle[100m])
 eval instant at 50m predict_linear(testcounter_reset_middle[50m], 3600)
 	{} 76.81818181818181
 
+eval instant at 50m predict_linear(testcounter_reset_middle[50m], 1h)
+	{} 76.81818181818181
+
 # intercept at t = 3000+3600 = 6600
 eval instant at 50m predict_linear(testcounter_reset_middle[50m] @ 3000, 3600)
 	{} 76.81818181818181
 
+eval instant at 50m predict_linear(testcounter_reset_middle[50m] @ 3000, 1h)
+	{} 76.81818181818181
+
 # intercept at t = 600+3600 = 4200
 eval instant at 10m predict_linear(testcounter_reset_middle[50m] @ 3000, 3600)
 	{} 51.36363636363637

promql/promqltest/testdata/limit.test

@@ -0,0 +1,119 @@
+# Tests for limitk
+#
+# NB: those many `and http_requests` are to ensure that the series _are_ indeed
+# a subset of the original series.
+load 5m
+	http_requests{job="api-server", instance="0", group="production"}	0+10x10
+	http_requests{job="api-server", instance="1", group="production"}	0+20x10
+	http_requests{job="api-server", instance="0", group="canary"}		0+30x10
+	http_requests{job="api-server", instance="1", group="canary"}		0+40x10
+	http_requests{job="api-server", instance="2", group="canary"}		0+50x10
+	http_requests{job="api-server", instance="3", group="canary"}		0+60x10
+
+eval instant at 50m count(limitk by (group) (0, http_requests))
+# empty
+
+eval instant at 50m count(limitk by (group) (-1, http_requests))
+# empty
+
+# Exercise k==1 special case (as sample is added before the main series loop
+eval instant at 50m count(limitk by (group) (1, http_requests) and http_requests)
+        {} 2
+
+eval instant at 50m count(limitk by (group) (2, http_requests) and http_requests)
+        {} 4
+
+eval instant at 50m count(limitk(100, http_requests) and http_requests)
+        {} 6
+
+# Exercise k==1 special case (as sample is added before the main series loop
+eval instant at 50m count(limitk by (group) (1, http_requests) and http_requests)
+        {} 2
+
+eval instant at 50m count(limitk by (group) (2, http_requests) and http_requests)
+        {} 4
+
+eval instant at 50m count(limitk(100, http_requests) and http_requests)
+        {} 6
+
+# limit_ratio
+eval range from 0 to 50m step 5m count(limit_ratio(0.0, http_requests))
+# empty
+
+# limitk(2, ...) should always return a 2-count subset of the timeseries (hence the AND'ing)
+eval range from 0 to 50m step 5m count(limitk(2, http_requests) and http_requests)
+        {} 2+0x10
+
+# Tests for limit_ratio
+#
+# NB: below 0.5 ratio will depend on some hashing "luck" (also there's no guarantee that
+# an integer comes from: total number of series * ratio), as it depends on:
+#
+# * ratioLimit = [0.0, 1.0]:	
+#     float64(sample.Metric.Hash()) / float64MaxUint64 < Ratio ?
+# * ratioLimit = [-1.0, 1.0):
+#     float64(sample.Metric.Hash()) / float64MaxUint64 >= (1.0 + Ratio) ?
+#
+# See `AddRatioSample()` in promql/engine.go for more details.
+
+# Half~ish samples: verify we get "near" 3 (of 0.5 * 6)
+eval range from 0 to 50m step 5m count(limit_ratio(0.5, http_requests) and http_requests) <= bool (3+1)
+        {} 1+0x10
+
+eval range from 0 to 50m step 5m count(limit_ratio(0.5, http_requests) and http_requests) >= bool (3-1)
+        {} 1+0x10
+
+# All samples
+eval range from 0 to 50m step 5m count(limit_ratio(1.0, http_requests) and http_requests)
+        {} 6+0x10
+
+# All samples
+eval range from 0 to 50m step 5m count(limit_ratio(-1.0, http_requests) and http_requests)
+        {} 6+0x10
+
+# Capped to 1.0 -> all samples
+eval_warn range from 0 to 50m step 5m count(limit_ratio(1.1, http_requests) and http_requests)
+        {} 6+0x10
+
+# Capped to -1.0 -> all samples
+eval_warn range from 0 to 50m step 5m count(limit_ratio(-1.1, http_requests) and http_requests)
+        {} 6+0x10
+
+# Verify that limit_ratio(value) and limit_ratio(1.0-value) return the "complement" of each other
+# Complement below for [0.2, -0.8]
+#
+# Complement 1of2: `or` should return all samples
+eval range from 0 to 50m step 5m count(limit_ratio(0.2, http_requests) or limit_ratio(-0.8, http_requests))
+        {} 6+0x10
+
+# Complement 2of2: `and` should return no samples
+eval range from 0 to 50m step 5m count(limit_ratio(0.2, http_requests) and limit_ratio(-0.8, http_requests))
+# empty
+
+# Complement below for [0.5, -0.5]
+eval range from 0 to 50m step 5m count(limit_ratio(0.5, http_requests) or limit_ratio(-0.5, http_requests))
+        {} 6+0x10
+
+eval range from 0 to 50m step 5m count(limit_ratio(0.5, http_requests) and limit_ratio(-0.5, http_requests))
+# empty
+
+# Complement below for [0.8, -0.2]
+eval range from 0 to 50m step 5m count(limit_ratio(0.8, http_requests) or limit_ratio(-0.2, http_requests))
+        {} 6+0x10
+
+eval range from 0 to 50m step 5m count(limit_ratio(0.8, http_requests) and limit_ratio(-0.2, http_requests))
+# empty
+
+# Complement below for [some_ratio, 1.0 - some_ratio], some_ratio derived from time(),
+# using a small prime number to avoid rounded ratio values, and a small set of them.
+eval range from 0 to 50m step 5m count(limit_ratio(time() % 17/17, http_requests) or limit_ratio(1.0 - (time() % 17/17), http_requests))
+        {} 6+0x10
+
+eval range from 0 to 50m step 5m count(limit_ratio(time() % 17/17, http_requests) and limit_ratio(1.0 - (time() % 17/17), http_requests))
+# empty
+
+# Poor man's normality check: ok (loaded samples follow a nice linearity over labels and time)
+# The check giving: 1 (i.e. true)
+eval range from 0 to 50m step 5m abs(avg(limit_ratio(0.5, http_requests)) - avg(limit_ratio(-0.5, http_requests))) <= bool stddev(http_requests)
+	{} 1+0x10
+

promql/promqltest/testdata/native_histograms.test

@@ -715,6 +715,9 @@ eval instant at 10m histogram_fraction(NaN, NaN, histogram_fraction_4)
 eval instant at 10m histogram_fraction(-Inf, +Inf, histogram_fraction_4)
     {} 1
 
+eval instant at 10m histogram_sum(scalar(histogram_fraction(-Inf, +Inf, sum(histogram_fraction_4))) * histogram_fraction_4)
+    {} 100
+
 clear
 
 # Counter reset only noticeable in a single bucket.

promql/promqltest/testdata/staleness.test

@@ -32,6 +32,9 @@ eval instant at 20s count_over_time(metric[1s])
 eval instant at 20s count_over_time(metric[10s])
   {} 1
 
+eval instant at 20s count_over_time(metric[10])
+  {} 1
+
 
 clear
 

promql/promqltest/testdata/subquery.test

@@ -76,6 +76,21 @@ eval instant at 1010s sum_over_time(metric1[30s:10s] offset 3s)
 eval instant at 1010s sum_over_time((metric1)[30s:10s] offset 3s)
   {} 297
 
+eval instant at 1010s sum_over_time(metric1[30:10] offset 3)
+  {} 297
+
+eval instant at 1010s sum_over_time((metric1)[30:10s] offset 3s)
+  {} 297
+
+eval instant at 1010s sum_over_time((metric1)[30:10s] offset 3s)
+  {} 297
+
+eval instant at 1010s sum_over_time((metric1)[30:10] offset 3s)
+  {} 297
+
+eval instant at 1010s sum_over_time((metric1)[30:10] offset 3)
+  {} 297
+
 # Nested subqueries
 eval instant at 1000s rate(sum_over_time(metric1[30s:10s])[50s:10s])
   {} 0.4

rules/group.go

@@ -151,7 +151,42 @@ func (g *Group) Name() string { return g.name }
 func (g *Group) File() string { return g.file }
 
 // Rules returns the group's rules.
-func (g *Group) Rules() []Rule { return g.rules }
+func (g *Group) Rules(matcherSets ...[]*labels.Matcher) []Rule {
+	if len(matcherSets) == 0 {
+		return g.rules
+	}
+	var rules []Rule
+	for _, rule := range g.rules {
+		if matchesMatcherSets(matcherSets, rule.Labels()) {
+			rules = append(rules, rule)
+		}
+	}
+	return rules
+}
+
+func matches(lbls labels.Labels, matchers ...*labels.Matcher) bool {
+	for _, m := range matchers {
+		if v := lbls.Get(m.Name); !m.Matches(v) {
+			return false
+		}
+	}
+	return true
+}
+
+// matchesMatcherSets ensures all matches in each matcher set are ANDed and the set of those is ORed.
+func matchesMatcherSets(matcherSets [][]*labels.Matcher, lbls labels.Labels) bool {
+	if len(matcherSets) == 0 {
+		return true
+	}
+
+	var ok bool
+	for _, matchers := range matcherSets {
+		if matches(lbls, matchers...) {
+			ok = true
+		}
+	}
+	return ok
+}
 
 // Queryable returns the group's querable.
 func (g *Group) Queryable() storage.Queryable { return g.opts.Queryable }

rules/manager.go

@@ -380,13 +380,13 @@ func (m *Manager) RuleGroups() []*Group {
 }
 
 // Rules returns the list of the manager's rules.
-func (m *Manager) Rules() []Rule {
+func (m *Manager) Rules(matcherSets ...[]*labels.Matcher) []Rule {
 	m.mtx.RLock()
 	defer m.mtx.RUnlock()
 
 	var rules []Rule
 	for _, g := range m.groups {
-		rules = append(rules, g.rules...)
+		rules = append(rules, g.Rules(matcherSets...)...)
 	}
 
 	return rules

scrape/manager.go

@@ -73,9 +73,11 @@ type Options struct {
 	// Option used by downstream scraper users like OpenTelemetry Collector
 	// to help lookup metric metadata. Should be false for Prometheus.
 	PassMetadataInContext bool
-	// Option to enable the experimental in-memory metadata storage and append
-	// metadata to the WAL.
-	EnableMetadataStorage bool
+	// Option to enable appending of scraped Metadata to the TSDB/other appenders. Individual appenders
+	// can decide what to do with metadata, but for practical purposes this flag exists so that metadata
+	// can be written to the WAL and thus read for remote write.
+	// TODO: implement some form of metadata storage
+	AppendMetadata bool
 	// Option to increase the interval used by scrape manager to throttle target groups updates.
 	DiscoveryReloadInterval model.Duration
 	// Option to enable the ingestion of the created timestamp as a synthetic zero sample.

scrape/metrics.go

@@ -34,6 +34,7 @@ type scrapeMetrics struct {
 	targetScrapePoolExceededTargetLimit prometheus.Counter
 	targetScrapePoolTargetLimit         *prometheus.GaugeVec
 	targetScrapePoolTargetsAdded        *prometheus.GaugeVec
+	targetScrapePoolSymbolTableItems    *prometheus.GaugeVec
 	targetSyncIntervalLength            *prometheus.SummaryVec
 	targetSyncFailed                    *prometheus.CounterVec
 
@@ -129,6 +130,13 @@ func newScrapeMetrics(reg prometheus.Registerer) (*scrapeMetrics, error) {
 		},
 		[]string{"scrape_job"},
 	)
+	sm.targetScrapePoolSymbolTableItems = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "prometheus_target_scrape_pool_symboltable_items",
+			Help: "Current number of symbols in table for this scrape pool.",
+		},
+		[]string{"scrape_job"},
+	)
 	sm.targetScrapePoolSyncsCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "prometheus_target_scrape_pool_sync_total",
@@ -234,6 +242,7 @@ func newScrapeMetrics(reg prometheus.Registerer) (*scrapeMetrics, error) {
 		sm.targetScrapePoolExceededTargetLimit,
 		sm.targetScrapePoolTargetLimit,
 		sm.targetScrapePoolTargetsAdded,
+		sm.targetScrapePoolSymbolTableItems,
 		sm.targetSyncFailed,
 		// Used by targetScraper.
 		sm.targetScrapeExceededBodySizeLimit,
@@ -274,6 +283,7 @@ func (sm *scrapeMetrics) Unregister() {
 	sm.reg.Unregister(sm.targetScrapePoolExceededTargetLimit)
 	sm.reg.Unregister(sm.targetScrapePoolTargetLimit)
 	sm.reg.Unregister(sm.targetScrapePoolTargetsAdded)
+	sm.reg.Unregister(sm.targetScrapePoolSymbolTableItems)
 	sm.reg.Unregister(sm.targetSyncFailed)
 	sm.reg.Unregister(sm.targetScrapeExceededBodySizeLimit)
 	sm.reg.Unregister(sm.targetScrapeCacheFlushForced)

scrape/scrape.go

@@ -181,7 +181,7 @@ func newScrapePool(cfg *config.ScrapeConfig, app storage.Appendable, offsetSeed
 			options.EnableNativeHistogramsIngestion,
 			options.EnableCreatedTimestampZeroIngestion,
 			options.ExtraMetrics,
-			options.EnableMetadataStorage,
+			options.AppendMetadata,
 			opts.target,
 			options.PassMetadataInContext,
 			metrics,
@@ -246,6 +246,7 @@ func (sp *scrapePool) stop() {
 		sp.metrics.targetScrapePoolSyncsCounter.DeleteLabelValues(sp.config.JobName)
 		sp.metrics.targetScrapePoolTargetLimit.DeleteLabelValues(sp.config.JobName)
 		sp.metrics.targetScrapePoolTargetsAdded.DeleteLabelValues(sp.config.JobName)
+		sp.metrics.targetScrapePoolSymbolTableItems.DeleteLabelValues(sp.config.JobName)
 		sp.metrics.targetSyncIntervalLength.DeleteLabelValues(sp.config.JobName)
 		sp.metrics.targetSyncFailed.DeleteLabelValues(sp.config.JobName)
 	}
@@ -273,6 +274,15 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) error {
 
 	sp.metrics.targetScrapePoolTargetLimit.WithLabelValues(sp.config.JobName).Set(float64(sp.config.TargetLimit))
 
+	sp.restartLoops(reuseCache)
+	oldClient.CloseIdleConnections()
+	sp.metrics.targetReloadIntervalLength.WithLabelValues(time.Duration(sp.config.ScrapeInterval).String()).Observe(
+		time.Since(start).Seconds(),
+	)
+	return nil
+}
+
+func (sp *scrapePool) restartLoops(reuseCache bool) {
 	var (
 		wg            sync.WaitGroup
 		interval      = time.Duration(sp.config.ScrapeInterval)
@@ -313,7 +323,7 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) error {
 				client:               sp.client,
 				timeout:              timeout,
 				bodySizeLimit:        bodySizeLimit,
-				acceptHeader:         acceptHeader(cfg.ScrapeProtocols),
+				acceptHeader:         acceptHeader(sp.config.ScrapeProtocols),
 				acceptEncodingHeader: acceptEncodingHeader(enableCompression),
 			}
 			newLoop = sp.newLoop(scrapeLoopOptions{
@@ -352,11 +362,10 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) error {
 	sp.targetMtx.Unlock()
 
 	wg.Wait()
-	oldClient.CloseIdleConnections()
-	sp.metrics.targetReloadIntervalLength.WithLabelValues(interval.String()).Observe(
-		time.Since(start).Seconds(),
-	)
+}
 
+// Must be called with sp.mtx held.
+func (sp *scrapePool) checkSymbolTable() {
 	// Here we take steps to clear out the symbol table if it has grown a lot.
 	// After waiting some time for things to settle, we take the size of the symbol-table.
 	// If, after some more time, the table has grown to twice that size, we start a new one.
@@ -367,11 +376,10 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) error {
 		} else if sp.symbolTable.Len() > 2*sp.initialSymbolTableLen {
 			sp.symbolTable = labels.NewSymbolTable()
 			sp.initialSymbolTableLen = 0
+			sp.restartLoops(false) // To drop all caches.
 		}
 		sp.lastSymbolTableCheck = time.Now()
 	}
-
-	return nil
 }
 
 // Sync converts target groups into actual scrape targets and synchronizes
@@ -408,8 +416,10 @@ func (sp *scrapePool) Sync(tgs []*targetgroup.Group) {
 			}
 		}
 	}
+	sp.metrics.targetScrapePoolSymbolTableItems.WithLabelValues(sp.config.JobName).Set(float64(sp.symbolTable.Len()))
 	sp.targetMtx.Unlock()
 	sp.sync(all)
+	sp.checkSymbolTable()
 
 	sp.metrics.targetSyncIntervalLength.WithLabelValues(sp.config.JobName).Observe(
 		time.Since(start).Seconds(),

scripts/genproto.sh

@@ -10,8 +10,9 @@ if ! [[ "$0" =~ "scripts/genproto.sh" ]]; then
 	exit 255
 fi
 
+# TODO(bwplotka): Move to buf, this is not OSS agnostic, likely won't work locally.
 if ! [[ $(protoc --version) =~ "3.15.8" ]]; then
-	echo "could not find protoc 3.15.8, is it installed + in PATH?"
+	echo "could not find protoc 3.15.8, is it installed + in PATH? Consider commenting out this check for local flow"
 	exit 255
 fi
 
@@ -40,6 +41,9 @@ for dir in ${DIRS}; do
             -I="${PROM_PATH}" \
             -I="${GRPC_GATEWAY_ROOT}/third_party/googleapis" \
             ./*.proto
+		protoc --gogofast_out=plugins=grpc:. -I=. \
+            -I="${GOGOPROTO_PATH}" \
+            ./io/prometheus/write/v2/*.proto
 		protoc --gogofast_out=Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types,paths=source_relative:. -I=. \
             -I="${GOGOPROTO_PATH}" \
             ./io/prometheus/client/*.proto

storage/fanout_test.go

@@ -181,7 +181,8 @@ func TestFanoutErrors(t *testing.T) {
 				require.NotEmpty(t, ss.Warnings(), "warnings expected")
 				w := ss.Warnings()
 				require.Error(t, w.AsErrors()[0])
-				require.Equal(t, tc.warning.Error(), w.AsStrings("", 0)[0])
+				warn, _ := w.AsStrings("", 0, 0)
+				require.Equal(t, tc.warning.Error(), warn[0])
 			}
 		})
 		t.Run("chunks", func(t *testing.T) {
@@ -207,7 +208,8 @@ func TestFanoutErrors(t *testing.T) {
 				require.NotEmpty(t, ss.Warnings(), "warnings expected")
 				w := ss.Warnings()
 				require.Error(t, w.AsErrors()[0])
-				require.Equal(t, tc.warning.Error(), w.AsStrings("", 0)[0])
+				warn, _ := w.AsStrings("", 0, 0)
+				require.Equal(t, tc.warning.Error(), warn[0])
 			}
 		})
 	}
@@ -236,11 +238,11 @@ func (errQuerier) Select(context.Context, bool, *storage.SelectHints, ...*labels
 	return storage.ErrSeriesSet(errSelect)
 }
 
-func (errQuerier) LabelValues(context.Context, string, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+func (errQuerier) LabelValues(context.Context, string, *storage.LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
 	return nil, nil, errors.New("label values error")
 }
 
-func (errQuerier) LabelNames(context.Context, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+func (errQuerier) LabelNames(context.Context, *storage.LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
 	return nil, nil, errors.New("label names error")
 }
 

storage/interface.go

@@ -122,11 +122,11 @@ type MockQuerier struct {
 	SelectMockFunction func(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) SeriesSet
 }
 
-func (q *MockQuerier) LabelValues(context.Context, string, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+func (q *MockQuerier) LabelValues(context.Context, string, *LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
 	return nil, nil, nil
 }
 
-func (q *MockQuerier) LabelNames(context.Context, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+func (q *MockQuerier) LabelNames(context.Context, *LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
 	return nil, nil, nil
 }
 
@@ -161,12 +161,12 @@ type LabelQuerier interface {
 	// It is not safe to use the strings beyond the lifetime of the querier.
 	// If matchers are specified the returned result set is reduced
 	// to label values of metrics matching the matchers.
-	LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error)
+	LabelValues(ctx context.Context, name string, hints *LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error)
 
 	// LabelNames returns all the unique label names present in the block in sorted order.
 	// If matchers are specified the returned result set is reduced
 	// to label names of metrics matching the matchers.
-	LabelNames(ctx context.Context, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error)
+	LabelNames(ctx context.Context, hints *LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error)
 
 	// Close releases the resources of the Querier.
 	Close() error
@@ -190,6 +190,9 @@ type SelectHints struct {
 	Start int64 // Start time in milliseconds for this select.
 	End   int64 // End time in milliseconds for this select.
 
+	// Maximum number of results returned. Use a value of 0 to disable.
+	Limit int
+
 	Step int64  // Query step size in milliseconds.
 	Func string // String representation of surrounding function or aggregation.
 
@@ -217,6 +220,13 @@ type SelectHints struct {
 	DisableTrimming bool
 }
 
+// LabelHints specifies hints passed for label reads.
+// This is used only as an option for implementation to use.
+type LabelHints struct {
+	// Maximum number of results returned. Use a value of 0 to disable.
+	Limit int
+}
+
 // TODO(bwplotka): Move to promql/engine_test.go?
 // QueryableFunc is an adapter to allow the use of ordinary functions as
 // Queryables. It follows the idea of http.HandlerFunc.

storage/merge.go

@@ -169,8 +169,8 @@ func (l labelGenericQueriers) SplitByHalf() (labelGenericQueriers, labelGenericQ
 // LabelValues returns all potential values for a label name.
 // If matchers are specified the returned result set is reduced
 // to label values of metrics matching the matchers.
-func (q *mergeGenericQuerier) LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
-	res, ws, err := q.lvals(ctx, q.queriers, name, matchers...)
+func (q *mergeGenericQuerier) LabelValues(ctx context.Context, name string, hints *LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+	res, ws, err := q.lvals(ctx, q.queriers, name, hints, matchers...)
 	if err != nil {
 		return nil, nil, fmt.Errorf("LabelValues() from merge generic querier for label %s: %w", name, err)
 	}
@@ -178,22 +178,22 @@ func (q *mergeGenericQuerier) LabelValues(ctx context.Context, name string, matc
 }
 
 // lvals performs merge sort for LabelValues from multiple queriers.
-func (q *mergeGenericQuerier) lvals(ctx context.Context, lq labelGenericQueriers, n string, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+func (q *mergeGenericQuerier) lvals(ctx context.Context, lq labelGenericQueriers, n string, hints *LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
 	if lq.Len() == 0 {
 		return nil, nil, nil
 	}
 	if lq.Len() == 1 {
-		return lq.Get(0).LabelValues(ctx, n, matchers...)
+		return lq.Get(0).LabelValues(ctx, n, hints, matchers...)
 	}
 	a, b := lq.SplitByHalf()
 
 	var ws annotations.Annotations
-	s1, w, err := q.lvals(ctx, a, n, matchers...)
+	s1, w, err := q.lvals(ctx, a, n, hints, matchers...)
 	ws.Merge(w)
 	if err != nil {
 		return nil, ws, err
 	}
-	s2, ws, err := q.lvals(ctx, b, n, matchers...)
+	s2, ws, err := q.lvals(ctx, b, n, hints, matchers...)
 	ws.Merge(w)
 	if err != nil {
 		return nil, ws, err
@@ -229,13 +229,13 @@ func mergeStrings(a, b []string) []string {
 }
 
 // LabelNames returns all the unique label names present in all queriers in sorted order.
-func (q *mergeGenericQuerier) LabelNames(ctx context.Context, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+func (q *mergeGenericQuerier) LabelNames(ctx context.Context, hints *LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
 	var (
 		labelNamesMap = make(map[string]struct{})
 		warnings      annotations.Annotations
 	)
 	for _, querier := range q.queriers {
-		names, wrn, err := querier.LabelNames(ctx, matchers...)
+		names, wrn, err := querier.LabelNames(ctx, hints, matchers...)
 		if wrn != nil {
 			// TODO(bwplotka): We could potentially wrap warnings.
 			warnings.Merge(wrn)

storage/merge_test.go

@@ -1361,7 +1361,7 @@ func (m *mockGenericQuerier) Select(_ context.Context, b bool, _ *SelectHints, _
 	return &mockGenericSeriesSet{resp: m.resp, warnings: m.warnings, err: m.err}
 }
 
-func (m *mockGenericQuerier) LabelValues(_ context.Context, name string, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+func (m *mockGenericQuerier) LabelValues(_ context.Context, name string, hints *LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
 	m.mtx.Lock()
 	m.labelNamesRequested = append(m.labelNamesRequested, labelNameRequest{
 		name:     name,
@@ -1371,7 +1371,7 @@ func (m *mockGenericQuerier) LabelValues(_ context.Context, name string, matcher
 	return m.resp, m.warnings, m.err
 }
 
-func (m *mockGenericQuerier) LabelNames(context.Context, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+func (m *mockGenericQuerier) LabelNames(context.Context, *LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
 	m.mtx.Lock()
 	m.labelNamesCalls++
 	m.mtx.Unlock()
@@ -1558,7 +1558,7 @@ func TestMergeGenericQuerierWithSecondaries_ErrorHandling(t *testing.T) {
 				}
 			})
 			t.Run("LabelNames", func(t *testing.T) {
-				res, w, err := q.LabelNames(ctx)
+				res, w, err := q.LabelNames(ctx, nil)
 				require.Subset(t, tcase.expectedWarnings, w)
 				require.ErrorIs(t, err, tcase.expectedErrs[1], "expected error doesn't match")
 				require.Equal(t, tcase.expectedLabels, res)
@@ -1573,7 +1573,7 @@ func TestMergeGenericQuerierWithSecondaries_ErrorHandling(t *testing.T) {
 				}
 			})
 			t.Run("LabelValues", func(t *testing.T) {
-				res, w, err := q.LabelValues(ctx, "test")
+				res, w, err := q.LabelValues(ctx, "test", nil)
 				require.Subset(t, tcase.expectedWarnings, w)
 				require.ErrorIs(t, err, tcase.expectedErrs[2], "expected error doesn't match")
 				require.Equal(t, tcase.expectedLabels, res)
@@ -1589,7 +1589,7 @@ func TestMergeGenericQuerierWithSecondaries_ErrorHandling(t *testing.T) {
 			})
 			t.Run("LabelValuesWithMatchers", func(t *testing.T) {
 				matcher := labels.MustNewMatcher(labels.MatchEqual, "otherLabel", "someValue")
-				res, w, err := q.LabelValues(ctx, "test2", matcher)
+				res, w, err := q.LabelValues(ctx, "test2", nil, matcher)
 				require.Subset(t, tcase.expectedWarnings, w)
 				require.ErrorIs(t, err, tcase.expectedErrs[3], "expected error doesn't match")
 				require.Equal(t, tcase.expectedLabels, res)

storage/noop.go

@@ -31,11 +31,11 @@ func (noopQuerier) Select(context.Context, bool, *SelectHints, ...*labels.Matche
 	return NoopSeriesSet()
 }
 
-func (noopQuerier) LabelValues(context.Context, string, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+func (noopQuerier) LabelValues(context.Context, string, *LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
 	return nil, nil, nil
 }
 
-func (noopQuerier) LabelNames(context.Context, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+func (noopQuerier) LabelNames(context.Context, *LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
 	return nil, nil, nil
 }
 
@@ -54,11 +54,11 @@ func (noopChunkQuerier) Select(context.Context, bool, *SelectHints, ...*labels.M
 	return NoopChunkedSeriesSet()
 }
 
-func (noopChunkQuerier) LabelValues(context.Context, string, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+func (noopChunkQuerier) LabelValues(context.Context, string, *LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
 	return nil, nil, nil
 }
 
-func (noopChunkQuerier) LabelNames(context.Context, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+func (noopChunkQuerier) LabelNames(context.Context, *LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
 	return nil, nil, nil
 }
 

storage/remote/client.go

@@ -35,13 +35,40 @@ import (
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/trace"
 
+	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/prompb"
 	"github.com/prometheus/prometheus/storage/remote/azuread"
 )
 
 const maxErrMsgLen = 1024
 
-var UserAgent = fmt.Sprintf("Prometheus/%s", version.Version)
+const (
+	RemoteWriteVersionHeader        = "X-Prometheus-Remote-Write-Version"
+	RemoteWriteVersion1HeaderValue  = "0.1.0"
+	RemoteWriteVersion20HeaderValue = "2.0.0"
+	appProtoContentType             = "application/x-protobuf"
+)
+
+// Compression represents the encoding. Currently remote storage supports only
+// one, but we experiment with more, thus leaving the compression scaffolding
+// for now.
+// NOTE(bwplotka): Keeping it public, as a non-stable help for importers to use.
+type Compression string
+
+const (
+	// SnappyBlockCompression represents https://github.com/google/snappy/blob/2c94e11145f0b7b184b831577c93e5a41c4c0346/format_description.txt
+	SnappyBlockCompression Compression = "snappy"
+)
+
+var (
+	// UserAgent represents Prometheus version to use for user agent header.
+	UserAgent = fmt.Sprintf("Prometheus/%s", version.Version)
+
+	remoteWriteContentTypeHeaders = map[config.RemoteWriteProtoMsg]string{
+		config.RemoteWriteProtoMsgV1: appProtoContentType, // Also application/x-protobuf;proto=prometheus.WriteRequest but simplified for compatibility with 1.x spec.
+		config.RemoteWriteProtoMsgV2: appProtoContentType + ";proto=io.prometheus.write.v2.Request",
+	}
+)
 
 var (
 	remoteReadQueriesTotal = prometheus.NewCounterVec(
@@ -93,6 +120,9 @@ type Client struct {
 	readQueries         prometheus.Gauge
 	readQueriesTotal    *prometheus.CounterVec
 	readQueriesDuration prometheus.Observer
+
+	writeProtoMsg    config.RemoteWriteProtoMsg
+	writeCompression Compression // Not exposed by ClientConfig for now.
 }
 
 // ClientConfig configures a client.
@@ -104,6 +134,7 @@ type ClientConfig struct {
 	AzureADConfig    *azuread.AzureADConfig
 	Headers          map[string]string
 	RetryOnRateLimit bool
+	WriteProtoMsg    config.RemoteWriteProtoMsg
 }
 
 // ReadClient uses the SAMPLES method of remote read to read series samples from remote server.
@@ -162,14 +193,20 @@ func NewWriteClient(name string, conf *ClientConfig) (WriteClient, error) {
 		}
 	}
 
-	httpClient.Transport = otelhttp.NewTransport(t)
+	writeProtoMsg := config.RemoteWriteProtoMsgV1
+	if conf.WriteProtoMsg != "" {
+		writeProtoMsg = conf.WriteProtoMsg
+	}
 
+	httpClient.Transport = otelhttp.NewTransport(t)
 	return &Client{
 		remoteName:       name,
 		urlString:        conf.URL.String(),
 		Client:           httpClient,
 		retryOnRateLimit: conf.RetryOnRateLimit,
 		timeout:          time.Duration(conf.Timeout),
+		writeProtoMsg:    writeProtoMsg,
+		writeCompression: SnappyBlockCompression,
 	}, nil
 }
 
@@ -206,10 +243,16 @@ func (c *Client) Store(ctx context.Context, req []byte, attempt int) error {
 		return err
 	}
 
-	httpReq.Header.Add("Content-Encoding", "snappy")
-	httpReq.Header.Set("Content-Type", "application/x-protobuf")
+	httpReq.Header.Add("Content-Encoding", string(c.writeCompression))
+	httpReq.Header.Set("Content-Type", remoteWriteContentTypeHeaders[c.writeProtoMsg])
 	httpReq.Header.Set("User-Agent", UserAgent)
-	httpReq.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
+	if c.writeProtoMsg == config.RemoteWriteProtoMsgV1 {
+		// Compatibility mode for 1.0.
+		httpReq.Header.Set(RemoteWriteVersionHeader, RemoteWriteVersion1HeaderValue)
+	} else {
+		httpReq.Header.Set(RemoteWriteVersionHeader, RemoteWriteVersion20HeaderValue)
+	}
+
 	if attempt > 0 {
 		httpReq.Header.Set("Retry-Attempt", strconv.Itoa(attempt))
 	}
@@ -265,12 +308,12 @@ func retryAfterDuration(t string) model.Duration {
 }
 
 // Name uniquely identifies the client.
-func (c Client) Name() string {
+func (c *Client) Name() string {
 	return c.remoteName
 }
 
 // Endpoint is the remote read or write endpoint.
-func (c Client) Endpoint() string {
+func (c *Client) Endpoint() string {
 	return c.urlString
 }
 

storage/remote/codec.go

@@ -22,7 +22,6 @@ import (
 	"net/http"
 	"slices"
 	"sort"
-	"strings"
 	"sync"
 
 	"github.com/gogo/protobuf/proto"
@@ -30,10 +29,10 @@ import (
 	"github.com/prometheus/common/model"
 	"go.opentelemetry.io/collector/pdata/pmetric/pmetricotlp"
 
-	"github.com/prometheus/prometheus/model/exemplar"
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/prompb"
+	writev2 "github.com/prometheus/prometheus/prompb/io/prometheus/write/v2"
 	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
 	"github.com/prometheus/prometheus/tsdb/chunks"
@@ -153,10 +152,10 @@ func ToQueryResult(ss storage.SeriesSet, sampleLimit int) (*prompb.QueryResult,
 				})
 			case chunkenc.ValHistogram:
 				ts, h := iter.AtHistogram(nil)
-				histograms = append(histograms, HistogramToHistogramProto(ts, h))
+				histograms = append(histograms, prompb.FromIntHistogram(ts, h))
 			case chunkenc.ValFloatHistogram:
 				ts, fh := iter.AtFloatHistogram(nil)
-				histograms = append(histograms, FloatHistogramToHistogramProto(ts, fh))
+				histograms = append(histograms, prompb.FromFloatHistogram(ts, fh))
 			default:
 				return nil, ss.Warnings(), fmt.Errorf("unrecognized value type: %s", valType)
 			}
@@ -166,7 +165,7 @@ func ToQueryResult(ss storage.SeriesSet, sampleLimit int) (*prompb.QueryResult,
 		}
 
 		resp.Timeseries = append(resp.Timeseries, &prompb.TimeSeries{
-			Labels:     LabelsToLabelsProto(series.Labels(), nil),
+			Labels:     prompb.FromLabels(series.Labels(), nil),
 			Samples:    samples,
 			Histograms: histograms,
 		})
@@ -182,7 +181,7 @@ func FromQueryResult(sortSeries bool, res *prompb.QueryResult) storage.SeriesSet
 		if err := validateLabelsAndMetricName(ts.Labels); err != nil {
 			return errSeriesSet{err: err}
 		}
-		lbls := LabelProtosToLabels(&b, ts.Labels)
+		lbls := ts.ToLabels(&b, nil)
 		series = append(series, &concreteSeries{labels: lbls, floats: ts.Samples, histograms: ts.Histograms})
 	}
 
@@ -235,7 +234,7 @@ func StreamChunkedReadResponses(
 	for ss.Next() {
 		series := ss.At()
 		iter = series.Iterator(iter)
-		lbls = MergeLabels(LabelsToLabelsProto(series.Labels(), lbls), sortedExternalLabels)
+		lbls = MergeLabels(prompb.FromLabels(series.Labels(), lbls), sortedExternalLabels)
 
 		maxDataLength := maxBytesInFrame
 		for _, lbl := range lbls {
@@ -481,21 +480,16 @@ func (c *concreteSeriesIterator) AtHistogram(*histogram.Histogram) (int64, *hist
 		panic("iterator is not on an integer histogram sample")
 	}
 	h := c.series.histograms[c.histogramsCur]
-	return h.Timestamp, HistogramProtoToHistogram(h)
+	return h.Timestamp, h.ToIntHistogram()
 }
 
 // AtFloatHistogram implements chunkenc.Iterator.
 func (c *concreteSeriesIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
-	switch c.curValType {
-	case chunkenc.ValHistogram:
+	if c.curValType == chunkenc.ValHistogram || c.curValType == chunkenc.ValFloatHistogram {
 		fh := c.series.histograms[c.histogramsCur]
-		return fh.Timestamp, HistogramProtoToFloatHistogram(fh)
-	case chunkenc.ValFloatHistogram:
-		fh := c.series.histograms[c.histogramsCur]
-		return fh.Timestamp, FloatHistogramProtoToFloatHistogram(fh)
-	default:
-		panic("iterator is not on a histogram sample")
+		return fh.Timestamp, fh.ToFloatHistogram() // integer will be auto-converted.
 	}
+	panic("iterator is not on a histogram sample")
 }
 
 // AtT implements chunkenc.Iterator.
@@ -618,141 +612,6 @@ func FromLabelMatchers(matchers []*prompb.LabelMatcher) ([]*labels.Matcher, erro
 	return result, nil
 }
 
-func exemplarProtoToExemplar(b *labels.ScratchBuilder, ep prompb.Exemplar) exemplar.Exemplar {
-	timestamp := ep.Timestamp
-
-	return exemplar.Exemplar{
-		Labels: LabelProtosToLabels(b, ep.Labels),
-		Value:  ep.Value,
-		Ts:     timestamp,
-		HasTs:  timestamp != 0,
-	}
-}
-
-// HistogramProtoToHistogram extracts a (normal integer) Histogram from the
-// provided proto message. The caller has to make sure that the proto message
-// represents an integer histogram and not a float histogram, or it panics.
-func HistogramProtoToHistogram(hp prompb.Histogram) *histogram.Histogram {
-	if hp.IsFloatHistogram() {
-		panic("HistogramProtoToHistogram called with a float histogram")
-	}
-	return &histogram.Histogram{
-		CounterResetHint: histogram.CounterResetHint(hp.ResetHint),
-		Schema:           hp.Schema,
-		ZeroThreshold:    hp.ZeroThreshold,
-		ZeroCount:        hp.GetZeroCountInt(),
-		Count:            hp.GetCountInt(),
-		Sum:              hp.Sum,
-		PositiveSpans:    spansProtoToSpans(hp.GetPositiveSpans()),
-		PositiveBuckets:  hp.GetPositiveDeltas(),
-		NegativeSpans:    spansProtoToSpans(hp.GetNegativeSpans()),
-		NegativeBuckets:  hp.GetNegativeDeltas(),
-	}
-}
-
-// FloatHistogramProtoToFloatHistogram extracts a float Histogram from the
-// provided proto message to a Float Histogram. The caller has to make sure that
-// the proto message represents a float histogram and not an integer histogram,
-// or it panics.
-func FloatHistogramProtoToFloatHistogram(hp prompb.Histogram) *histogram.FloatHistogram {
-	if !hp.IsFloatHistogram() {
-		panic("FloatHistogramProtoToFloatHistogram called with an integer histogram")
-	}
-	return &histogram.FloatHistogram{
-		CounterResetHint: histogram.CounterResetHint(hp.ResetHint),
-		Schema:           hp.Schema,
-		ZeroThreshold:    hp.ZeroThreshold,
-		ZeroCount:        hp.GetZeroCountFloat(),
-		Count:            hp.GetCountFloat(),
-		Sum:              hp.Sum,
-		PositiveSpans:    spansProtoToSpans(hp.GetPositiveSpans()),
-		PositiveBuckets:  hp.GetPositiveCounts(),
-		NegativeSpans:    spansProtoToSpans(hp.GetNegativeSpans()),
-		NegativeBuckets:  hp.GetNegativeCounts(),
-	}
-}
-
-// HistogramProtoToFloatHistogram extracts and converts a (normal integer) histogram from the provided proto message
-// to a float histogram. The caller has to make sure that the proto message represents an integer histogram and not a
-// float histogram, or it panics.
-func HistogramProtoToFloatHistogram(hp prompb.Histogram) *histogram.FloatHistogram {
-	if hp.IsFloatHistogram() {
-		panic("HistogramProtoToFloatHistogram called with a float histogram")
-	}
-	return &histogram.FloatHistogram{
-		CounterResetHint: histogram.CounterResetHint(hp.ResetHint),
-		Schema:           hp.Schema,
-		ZeroThreshold:    hp.ZeroThreshold,
-		ZeroCount:        float64(hp.GetZeroCountInt()),
-		Count:            float64(hp.GetCountInt()),
-		Sum:              hp.Sum,
-		PositiveSpans:    spansProtoToSpans(hp.GetPositiveSpans()),
-		PositiveBuckets:  deltasToCounts(hp.GetPositiveDeltas()),
-		NegativeSpans:    spansProtoToSpans(hp.GetNegativeSpans()),
-		NegativeBuckets:  deltasToCounts(hp.GetNegativeDeltas()),
-	}
-}
-
-func spansProtoToSpans(s []prompb.BucketSpan) []histogram.Span {
-	spans := make([]histogram.Span, len(s))
-	for i := 0; i < len(s); i++ {
-		spans[i] = histogram.Span{Offset: s[i].Offset, Length: s[i].Length}
-	}
-
-	return spans
-}
-
-func deltasToCounts(deltas []int64) []float64 {
-	counts := make([]float64, len(deltas))
-	var cur float64
-	for i, d := range deltas {
-		cur += float64(d)
-		counts[i] = cur
-	}
-	return counts
-}
-
-func HistogramToHistogramProto(timestamp int64, h *histogram.Histogram) prompb.Histogram {
-	return prompb.Histogram{
-		Count:          &prompb.Histogram_CountInt{CountInt: h.Count},
-		Sum:            h.Sum,
-		Schema:         h.Schema,
-		ZeroThreshold:  h.ZeroThreshold,
-		ZeroCount:      &prompb.Histogram_ZeroCountInt{ZeroCountInt: h.ZeroCount},
-		NegativeSpans:  spansToSpansProto(h.NegativeSpans),
-		NegativeDeltas: h.NegativeBuckets,
-		PositiveSpans:  spansToSpansProto(h.PositiveSpans),
-		PositiveDeltas: h.PositiveBuckets,
-		ResetHint:      prompb.Histogram_ResetHint(h.CounterResetHint),
-		Timestamp:      timestamp,
-	}
-}
-
-func FloatHistogramToHistogramProto(timestamp int64, fh *histogram.FloatHistogram) prompb.Histogram {
-	return prompb.Histogram{
-		Count:          &prompb.Histogram_CountFloat{CountFloat: fh.Count},
-		Sum:            fh.Sum,
-		Schema:         fh.Schema,
-		ZeroThreshold:  fh.ZeroThreshold,
-		ZeroCount:      &prompb.Histogram_ZeroCountFloat{ZeroCountFloat: fh.ZeroCount},
-		NegativeSpans:  spansToSpansProto(fh.NegativeSpans),
-		NegativeCounts: fh.NegativeBuckets,
-		PositiveSpans:  spansToSpansProto(fh.PositiveSpans),
-		PositiveCounts: fh.PositiveBuckets,
-		ResetHint:      prompb.Histogram_ResetHint(fh.CounterResetHint),
-		Timestamp:      timestamp,
-	}
-}
-
-func spansToSpansProto(s []histogram.Span) []prompb.BucketSpan {
-	spans := make([]prompb.BucketSpan, len(s))
-	for i := 0; i < len(s); i++ {
-		spans[i] = prompb.BucketSpan{Offset: s[i].Offset, Length: s[i].Length}
-	}
-
-	return spans
-}
-
 // LabelProtosToMetric unpack a []*prompb.Label to a model.Metric.
 func LabelProtosToMetric(labelPairs []*prompb.Label) model.Metric {
 	metric := make(model.Metric, len(labelPairs))
@@ -762,43 +621,9 @@ func LabelProtosToMetric(labelPairs []*prompb.Label) model.Metric {
 	return metric
 }
 
-// LabelProtosToLabels transforms prompb labels into labels. The labels builder
-// will be used to build the returned labels.
-func LabelProtosToLabels(b *labels.ScratchBuilder, labelPairs []prompb.Label) labels.Labels {
-	b.Reset()
-	for _, l := range labelPairs {
-		b.Add(l.Name, l.Value)
-	}
-	b.Sort()
-	return b.Labels()
-}
-
-// LabelsToLabelsProto transforms labels into prompb labels. The buffer slice
-// will be used to avoid allocations if it is big enough to store the labels.
-func LabelsToLabelsProto(lbls labels.Labels, buf []prompb.Label) []prompb.Label {
-	result := buf[:0]
-	lbls.Range(func(l labels.Label) {
-		result = append(result, prompb.Label{
-			Name:  l.Name,
-			Value: l.Value,
-		})
-	})
-	return result
-}
-
-// metricTypeToMetricTypeProto transforms a Prometheus metricType into prompb metricType. Since the former is a string we need to transform it to an enum.
-func metricTypeToMetricTypeProto(t model.MetricType) prompb.MetricMetadata_MetricType {
-	mt := strings.ToUpper(string(t))
-	v, ok := prompb.MetricMetadata_MetricType_value[mt]
-	if !ok {
-		return prompb.MetricMetadata_UNKNOWN
-	}
-
-	return prompb.MetricMetadata_MetricType(v)
-}
-
 // DecodeWriteRequest from an io.Reader into a prompb.WriteRequest, handling
 // snappy decompression.
+// Used also by documentation/examples/remote_storage.
 func DecodeWriteRequest(r io.Reader) (*prompb.WriteRequest, error) {
 	compressed, err := io.ReadAll(r)
 	if err != nil {
@@ -818,6 +643,28 @@ func DecodeWriteRequest(r io.Reader) (*prompb.WriteRequest, error) {
 	return &req, nil
 }
 
+// DecodeWriteV2Request from an io.Reader into a writev2.Request, handling
+// snappy decompression.
+// Used also by documentation/examples/remote_storage.
+func DecodeWriteV2Request(r io.Reader) (*writev2.Request, error) {
+	compressed, err := io.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+
+	reqBuf, err := snappy.Decode(nil, compressed)
+	if err != nil {
+		return nil, err
+	}
+
+	var req writev2.Request
+	if err := proto.Unmarshal(reqBuf, &req); err != nil {
+		return nil, err
+	}
+
+	return &req, nil
+}
+
 func DecodeOTLPWriteRequest(r *http.Request) (pmetricotlp.ExportRequest, error) {
 	contentType := r.Header.Get("Content-Type")
 	var decoderFunc func(buf []byte) (pmetricotlp.ExportRequest, error)

storage/remote/codec_test.go

@@ -19,13 +19,16 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/go-kit/log"
 	"github.com/gogo/protobuf/proto"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/model/metadata"
 	"github.com/prometheus/prometheus/prompb"
+	writev2 "github.com/prometheus/prometheus/prompb/io/prometheus/write/v2"
 	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
 	"github.com/prometheus/prometheus/tsdb/chunks"
@@ -33,45 +36,128 @@ import (
 	"github.com/prometheus/prometheus/util/annotations"
 )
 
-var testHistogram = histogram.Histogram{
-	Schema:          2,
-	ZeroThreshold:   1e-128,
-	ZeroCount:       0,
-	Count:           0,
-	Sum:             20,
-	PositiveSpans:   []histogram.Span{{Offset: 0, Length: 1}},
-	PositiveBuckets: []int64{1},
-	NegativeSpans:   []histogram.Span{{Offset: 0, Length: 1}},
-	NegativeBuckets: []int64{-1},
-}
+var (
+	testHistogram = histogram.Histogram{
+		Schema:          2,
+		ZeroThreshold:   1e-128,
+		ZeroCount:       0,
+		Count:           0,
+		Sum:             20,
+		PositiveSpans:   []histogram.Span{{Offset: 0, Length: 1}},
+		PositiveBuckets: []int64{1},
+		NegativeSpans:   []histogram.Span{{Offset: 0, Length: 1}},
+		NegativeBuckets: []int64{-1},
+	}
 
-var writeRequestFixture = &prompb.WriteRequest{
-	Timeseries: []prompb.TimeSeries{
-		{
-			Labels: []prompb.Label{
-				{Name: "__name__", Value: "test_metric1"},
-				{Name: "b", Value: "c"},
-				{Name: "baz", Value: "qux"},
-				{Name: "d", Value: "e"},
-				{Name: "foo", Value: "bar"},
+	writeRequestFixture = &prompb.WriteRequest{
+		Timeseries: []prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "__name__", Value: "test_metric1"},
+					{Name: "b", Value: "c"},
+					{Name: "baz", Value: "qux"},
+					{Name: "d", Value: "e"},
+					{Name: "foo", Value: "bar"},
+				},
+				Samples:    []prompb.Sample{{Value: 1, Timestamp: 1}},
+				Exemplars:  []prompb.Exemplar{{Labels: []prompb.Label{{Name: "f", Value: "g"}}, Value: 1, Timestamp: 1}},
+				Histograms: []prompb.Histogram{prompb.FromIntHistogram(1, &testHistogram), prompb.FromFloatHistogram(2, testHistogram.ToFloat(nil))},
 			},
-			Samples:    []prompb.Sample{{Value: 1, Timestamp: 0}},
-			Exemplars:  []prompb.Exemplar{{Labels: []prompb.Label{{Name: "f", Value: "g"}}, Value: 1, Timestamp: 0}},
-			Histograms: []prompb.Histogram{HistogramToHistogramProto(0, &testHistogram), FloatHistogramToHistogramProto(1, testHistogram.ToFloat(nil))},
-		},
-		{
-			Labels: []prompb.Label{
-				{Name: "__name__", Value: "test_metric1"},
-				{Name: "b", Value: "c"},
-				{Name: "baz", Value: "qux"},
-				{Name: "d", Value: "e"},
-				{Name: "foo", Value: "bar"},
+			{
+				Labels: []prompb.Label{
+					{Name: "__name__", Value: "test_metric1"},
+					{Name: "b", Value: "c"},
+					{Name: "baz", Value: "qux"},
+					{Name: "d", Value: "e"},
+					{Name: "foo", Value: "bar"},
+				},
+				Samples:    []prompb.Sample{{Value: 2, Timestamp: 2}},
+				Exemplars:  []prompb.Exemplar{{Labels: []prompb.Label{{Name: "h", Value: "i"}}, Value: 2, Timestamp: 2}},
+				Histograms: []prompb.Histogram{prompb.FromIntHistogram(3, &testHistogram), prompb.FromFloatHistogram(4, testHistogram.ToFloat(nil))},
 			},
-			Samples:    []prompb.Sample{{Value: 2, Timestamp: 1}},
-			Exemplars:  []prompb.Exemplar{{Labels: []prompb.Label{{Name: "h", Value: "i"}}, Value: 2, Timestamp: 1}},
-			Histograms: []prompb.Histogram{HistogramToHistogramProto(2, &testHistogram), FloatHistogramToHistogramProto(3, testHistogram.ToFloat(nil))},
 		},
-	},
+	}
+
+	writeV2RequestSeries1Metadata = metadata.Metadata{
+		Type: model.MetricTypeGauge,
+		Help: "Test gauge for test purposes",
+		Unit: "Maybe op/sec who knows (:",
+	}
+	writeV2RequestSeries2Metadata = metadata.Metadata{
+		Type: model.MetricTypeCounter,
+		Help: "Test counter for test purposes",
+	}
+
+	// writeV2RequestFixture represents the same request as writeRequestFixture,
+	// but using the v2 representation, plus includes writeV2RequestSeries1Metadata and writeV2RequestSeries2Metadata.
+	// NOTE: Use TestWriteV2RequestFixture and copy the diff to regenerate if needed.
+	writeV2RequestFixture = &writev2.Request{
+		Symbols: []string{"", "__name__", "test_metric1", "b", "c", "baz", "qux", "d", "e", "foo", "bar", "f", "g", "h", "i", "Test gauge for test purposes", "Maybe op/sec who knows (:", "Test counter for test purposes"},
+		Timeseries: []writev2.TimeSeries{
+			{
+				LabelsRefs: []uint32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, // Symbolized writeRequestFixture.Timeseries[0].Labels
+				Metadata: writev2.Metadata{
+					Type: writev2.Metadata_METRIC_TYPE_GAUGE, // writeV2RequestSeries1Metadata.Type.
+
+					HelpRef: 15, // Symbolized writeV2RequestSeries1Metadata.Help.
+					UnitRef: 16, // Symbolized writeV2RequestSeries1Metadata.Unit.
+				},
+				Samples:    []writev2.Sample{{Value: 1, Timestamp: 1}},
+				Exemplars:  []writev2.Exemplar{{LabelsRefs: []uint32{11, 12}, Value: 1, Timestamp: 1}},
+				Histograms: []writev2.Histogram{writev2.FromIntHistogram(1, &testHistogram), writev2.FromFloatHistogram(2, testHistogram.ToFloat(nil))},
+			},
+			{
+				LabelsRefs: []uint32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, // Same series as first.
+				Metadata: writev2.Metadata{
+					Type: writev2.Metadata_METRIC_TYPE_COUNTER, // writeV2RequestSeries2Metadata.Type.
+
+					HelpRef: 17, // Symbolized writeV2RequestSeries2Metadata.Help.
+					// No unit.
+				},
+				Samples:    []writev2.Sample{{Value: 2, Timestamp: 2}},
+				Exemplars:  []writev2.Exemplar{{LabelsRefs: []uint32{13, 14}, Value: 2, Timestamp: 2}},
+				Histograms: []writev2.Histogram{writev2.FromIntHistogram(3, &testHistogram), writev2.FromFloatHistogram(4, testHistogram.ToFloat(nil))},
+			},
+		},
+	}
+)
+
+func TestWriteV2RequestFixture(t *testing.T) {
+	// Generate dynamically writeV2RequestFixture, reusing v1 fixture elements.
+	st := writev2.NewSymbolTable()
+	b := labels.NewScratchBuilder(0)
+	labelRefs := st.SymbolizeLabels(writeRequestFixture.Timeseries[0].ToLabels(&b, nil), nil)
+	exemplar1LabelRefs := st.SymbolizeLabels(writeRequestFixture.Timeseries[0].Exemplars[0].ToExemplar(&b, nil).Labels, nil)
+	exemplar2LabelRefs := st.SymbolizeLabels(writeRequestFixture.Timeseries[1].Exemplars[0].ToExemplar(&b, nil).Labels, nil)
+	expected := &writev2.Request{
+		Timeseries: []writev2.TimeSeries{
+			{
+				LabelsRefs: labelRefs,
+				Metadata: writev2.Metadata{
+					Type:    writev2.Metadata_METRIC_TYPE_GAUGE,
+					HelpRef: st.Symbolize(writeV2RequestSeries1Metadata.Help),
+					UnitRef: st.Symbolize(writeV2RequestSeries1Metadata.Unit),
+				},
+				Samples:    []writev2.Sample{{Value: 1, Timestamp: 1}},
+				Exemplars:  []writev2.Exemplar{{LabelsRefs: exemplar1LabelRefs, Value: 1, Timestamp: 1}},
+				Histograms: []writev2.Histogram{writev2.FromIntHistogram(1, &testHistogram), writev2.FromFloatHistogram(2, testHistogram.ToFloat(nil))},
+			},
+			{
+				LabelsRefs: labelRefs,
+				Metadata: writev2.Metadata{
+					Type:    writev2.Metadata_METRIC_TYPE_COUNTER,
+					HelpRef: st.Symbolize(writeV2RequestSeries2Metadata.Help),
+					// No unit.
+				},
+				Samples:    []writev2.Sample{{Value: 2, Timestamp: 2}},
+				Exemplars:  []writev2.Exemplar{{LabelsRefs: exemplar2LabelRefs, Value: 2, Timestamp: 2}},
+				Histograms: []writev2.Histogram{writev2.FromIntHistogram(3, &testHistogram), writev2.FromFloatHistogram(4, testHistogram.ToFloat(nil))},
+			},
+		},
+		Symbols: st.Symbols(),
+	}
+	// Check if it matches static writeV2RequestFixture.
+	require.Equal(t, expected, writeV2RequestFixture)
 }
 
 func TestValidateLabelsAndMetricName(t *testing.T) {
@@ -268,7 +354,7 @@ func TestConcreteSeriesIterator_HistogramSamples(t *testing.T) {
 		} else {
 			ts = int64(i)
 		}
-		histProtos[i] = HistogramToHistogramProto(ts, h)
+		histProtos[i] = prompb.FromIntHistogram(ts, h)
 	}
 	series := &concreteSeries{
 		labels:     labels.FromStrings("foo", "bar"),
@@ -319,9 +405,9 @@ func TestConcreteSeriesIterator_FloatAndHistogramSamples(t *testing.T) {
 	histProtos := make([]prompb.Histogram, len(histograms))
 	for i, h := range histograms {
 		if i < 10 {
-			histProtos[i] = HistogramToHistogramProto(int64(i+1), h)
+			histProtos[i] = prompb.FromIntHistogram(int64(i+1), h)
 		} else {
-			histProtos[i] = HistogramToHistogramProto(int64(i+6), h)
+			histProtos[i] = prompb.FromIntHistogram(int64(i+6), h)
 		}
 	}
 	series := &concreteSeries{
@@ -401,7 +487,7 @@ func TestConcreteSeriesIterator_FloatAndHistogramSamples(t *testing.T) {
 	require.Equal(t, chunkenc.ValHistogram, it.Next())
 	ts, fh = it.AtFloatHistogram(nil)
 	require.Equal(t, int64(17), ts)
-	expected := HistogramProtoToFloatHistogram(HistogramToHistogramProto(int64(17), histograms[11]))
+	expected := prompb.FromIntHistogram(int64(17), histograms[11]).ToFloatHistogram()
 	require.Equal(t, expected, fh)
 
 	// Keep calling Next() until the end.
@@ -485,39 +571,8 @@ func TestMergeLabels(t *testing.T) {
 	}
 }
 
-func TestMetricTypeToMetricTypeProto(t *testing.T) {
-	tc := []struct {
-		desc     string
-		input    model.MetricType
-		expected prompb.MetricMetadata_MetricType
-	}{
-		{
-			desc:     "with a single-word metric",
-			input:    model.MetricTypeCounter,
-			expected: prompb.MetricMetadata_COUNTER,
-		},
-		{
-			desc:     "with a two-word metric",
-			input:    model.MetricTypeStateset,
-			expected: prompb.MetricMetadata_STATESET,
-		},
-		{
-			desc:     "with an unknown metric",
-			input:    "not-known",
-			expected: prompb.MetricMetadata_UNKNOWN,
-		},
-	}
-
-	for _, tt := range tc {
-		t.Run(tt.desc, func(t *testing.T) {
-			m := metricTypeToMetricTypeProto(tt.input)
-			require.Equal(t, tt.expected, m)
-		})
-	}
-}
-
 func TestDecodeWriteRequest(t *testing.T) {
-	buf, _, _, err := buildWriteRequest(nil, writeRequestFixture.Timeseries, nil, nil, nil, nil)
+	buf, _, _, err := buildWriteRequest(nil, writeRequestFixture.Timeseries, nil, nil, nil, nil, "snappy")
 	require.NoError(t, err)
 
 	actual, err := DecodeWriteRequest(bytes.NewReader(buf))
@@ -525,212 +580,18 @@ func TestDecodeWriteRequest(t *testing.T) {
 	require.Equal(t, writeRequestFixture, actual)
 }
 
-func TestNilHistogramProto(*testing.T) {
-	// This function will panic if it impromperly handles nil
-	// values, causing the test to fail.
-	HistogramProtoToHistogram(prompb.Histogram{})
-	HistogramProtoToFloatHistogram(prompb.Histogram{})
-}
+func TestDecodeWriteV2Request(t *testing.T) {
+	buf, _, _, err := buildV2WriteRequest(log.NewNopLogger(), writeV2RequestFixture.Timeseries, writeV2RequestFixture.Symbols, nil, nil, nil, "snappy")
+	require.NoError(t, err)
 
-func exampleHistogram() histogram.Histogram {
-	return histogram.Histogram{
-		CounterResetHint: histogram.GaugeType,
-		Schema:           0,
-		Count:            19,
-		Sum:              2.7,
-		PositiveSpans: []histogram.Span{
-			{Offset: 0, Length: 4},
-			{Offset: 0, Length: 0},
-			{Offset: 0, Length: 3},
-		},
-		PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 0},
-		NegativeSpans: []histogram.Span{
-			{Offset: 0, Length: 5},
-			{Offset: 1, Length: 0},
-			{Offset: 0, Length: 1},
-		},
-		NegativeBuckets: []int64{1, 2, -2, 1, -1, 0},
-	}
-}
-
-func exampleHistogramProto() prompb.Histogram {
-	return prompb.Histogram{
-		Count:         &prompb.Histogram_CountInt{CountInt: 19},
-		Sum:           2.7,
-		Schema:        0,
-		ZeroThreshold: 0,
-		ZeroCount:     &prompb.Histogram_ZeroCountInt{ZeroCountInt: 0},
-		NegativeSpans: []prompb.BucketSpan{
-			{
-				Offset: 0,
-				Length: 5,
-			},
-			{
-				Offset: 1,
-				Length: 0,
-			},
-			{
-				Offset: 0,
-				Length: 1,
-			},
-		},
-		NegativeDeltas: []int64{1, 2, -2, 1, -1, 0},
-		PositiveSpans: []prompb.BucketSpan{
-			{
-				Offset: 0,
-				Length: 4,
-			},
-			{
-				Offset: 0,
-				Length: 0,
-			},
-			{
-				Offset: 0,
-				Length: 3,
-			},
-		},
-		PositiveDeltas: []int64{1, 2, -2, 1, -1, 0, 0},
-		ResetHint:      prompb.Histogram_GAUGE,
-		Timestamp:      1337,
-	}
-}
-
-func TestHistogramToProtoConvert(t *testing.T) {
-	tests := []struct {
-		input    histogram.CounterResetHint
-		expected prompb.Histogram_ResetHint
-	}{
-		{
-			input:    histogram.UnknownCounterReset,
-			expected: prompb.Histogram_UNKNOWN,
-		},
-		{
-			input:    histogram.CounterReset,
-			expected: prompb.Histogram_YES,
-		},
-		{
-			input:    histogram.NotCounterReset,
-			expected: prompb.Histogram_NO,
-		},
-		{
-			input:    histogram.GaugeType,
-			expected: prompb.Histogram_GAUGE,
-		},
-	}
-
-	for _, test := range tests {
-		h := exampleHistogram()
-		h.CounterResetHint = test.input
-		p := exampleHistogramProto()
-		p.ResetHint = test.expected
-
-		require.Equal(t, p, HistogramToHistogramProto(1337, &h))
-
-		require.Equal(t, h, *HistogramProtoToHistogram(p))
-	}
-}
-
-func exampleFloatHistogram() histogram.FloatHistogram {
-	return histogram.FloatHistogram{
-		CounterResetHint: histogram.GaugeType,
-		Schema:           0,
-		Count:            19,
-		Sum:              2.7,
-		PositiveSpans: []histogram.Span{
-			{Offset: 0, Length: 4},
-			{Offset: 0, Length: 0},
-			{Offset: 0, Length: 3},
-		},
-		PositiveBuckets: []float64{1, 2, -2, 1, -1, 0, 0},
-		NegativeSpans: []histogram.Span{
-			{Offset: 0, Length: 5},
-			{Offset: 1, Length: 0},
-			{Offset: 0, Length: 1},
-		},
-		NegativeBuckets: []float64{1, 2, -2, 1, -1, 0},
-	}
-}
-
-func exampleFloatHistogramProto() prompb.Histogram {
-	return prompb.Histogram{
-		Count:         &prompb.Histogram_CountFloat{CountFloat: 19},
-		Sum:           2.7,
-		Schema:        0,
-		ZeroThreshold: 0,
-		ZeroCount:     &prompb.Histogram_ZeroCountFloat{ZeroCountFloat: 0},
-		NegativeSpans: []prompb.BucketSpan{
-			{
-				Offset: 0,
-				Length: 5,
-			},
-			{
-				Offset: 1,
-				Length: 0,
-			},
-			{
-				Offset: 0,
-				Length: 1,
-			},
-		},
-		NegativeCounts: []float64{1, 2, -2, 1, -1, 0},
-		PositiveSpans: []prompb.BucketSpan{
-			{
-				Offset: 0,
-				Length: 4,
-			},
-			{
-				Offset: 0,
-				Length: 0,
-			},
-			{
-				Offset: 0,
-				Length: 3,
-			},
-		},
-		PositiveCounts: []float64{1, 2, -2, 1, -1, 0, 0},
-		ResetHint:      prompb.Histogram_GAUGE,
-		Timestamp:      1337,
-	}
-}
-
-func TestFloatHistogramToProtoConvert(t *testing.T) {
-	tests := []struct {
-		input    histogram.CounterResetHint
-		expected prompb.Histogram_ResetHint
-	}{
-		{
-			input:    histogram.UnknownCounterReset,
-			expected: prompb.Histogram_UNKNOWN,
-		},
-		{
-			input:    histogram.CounterReset,
-			expected: prompb.Histogram_YES,
-		},
-		{
-			input:    histogram.NotCounterReset,
-			expected: prompb.Histogram_NO,
-		},
-		{
-			input:    histogram.GaugeType,
-			expected: prompb.Histogram_GAUGE,
-		},
-	}
-
-	for _, test := range tests {
-		h := exampleFloatHistogram()
-		h.CounterResetHint = test.input
-		p := exampleFloatHistogramProto()
-		p.ResetHint = test.expected
-
-		require.Equal(t, p, FloatHistogramToHistogramProto(1337, &h))
-
-		require.Equal(t, h, *FloatHistogramProtoToFloatHistogram(p))
-	}
+	actual, err := DecodeWriteV2Request(bytes.NewReader(buf))
+	require.NoError(t, err)
+	require.Equal(t, writeV2RequestFixture, actual)
 }
 
 func TestStreamResponse(t *testing.T) {
-	lbs1 := LabelsToLabelsProto(labels.FromStrings("instance", "localhost1", "job", "demo1"), nil)
-	lbs2 := LabelsToLabelsProto(labels.FromStrings("instance", "localhost2", "job", "demo2"), nil)
+	lbs1 := prompb.FromLabels(labels.FromStrings("instance", "localhost1", "job", "demo1"), nil)
+	lbs2 := prompb.FromLabels(labels.FromStrings("instance", "localhost2", "job", "demo2"), nil)
 	chunk := prompb.Chunk{
 		Type: prompb.Chunk_XOR,
 		Data: make([]byte, 100),
@@ -802,7 +663,7 @@ func (c *mockChunkSeriesSet) Next() bool {
 
 func (c *mockChunkSeriesSet) At() storage.ChunkSeries {
 	return &storage.ChunkSeriesEntry{
-		Lset: LabelProtosToLabels(&c.builder, c.chunkedSeries[c.index].Labels),
+		Lset: c.chunkedSeries[c.index].ToLabels(&c.builder, nil),
 		ChunkIteratorFn: func(chunks.Iterator) chunks.Iterator {
 			return &mockChunkIterator{
 				chunks: c.chunkedSeries[c.index].Chunks,

storage/remote/max_timestamp.go

@@ -39,9 +39,3 @@ func (m *maxTimestamp) Get() float64 {
 	defer m.mtx.Unlock()
 	return m.value
 }
-
-func (m *maxTimestamp) Collect(c chan<- prometheus.Metric) {
-	if m.Get() > 0 {
-		m.Gauge.Collect(c)
-	}
-}

storage/remote/metadata_watcher.go

@@ -27,7 +27,7 @@ import (
 
 // MetadataAppender is an interface used by the Metadata Watcher to send metadata, It is read from the scrape manager, on to somewhere else.
 type MetadataAppender interface {
-	AppendMetadata(context.Context, []scrape.MetricMetadata)
+	AppendWatcherMetadata(context.Context, []scrape.MetricMetadata)
 }
 
 // Watchable represents from where we fetch active targets for metadata.
@@ -146,7 +146,7 @@ func (mw *MetadataWatcher) collect() {
 	}
 
 	// Blocks until the metadata is sent to the remote write endpoint or hardShutdownContext is expired.
-	mw.writer.AppendMetadata(mw.hardShutdownCtx, metadata)
+	mw.writer.AppendWatcherMetadata(mw.hardShutdownCtx, metadata)
 }
 
 func (mw *MetadataWatcher) ready() bool {

storage/remote/metadata_watcher_test.go

@@ -57,7 +57,7 @@ type writeMetadataToMock struct {
 	metadataAppended int
 }
 
-func (mwtm *writeMetadataToMock) AppendMetadata(_ context.Context, m []scrape.MetricMetadata) {
+func (mwtm *writeMetadataToMock) AppendWatcherMetadata(_ context.Context, m []scrape.MetricMetadata) {
 	mwtm.metadataAppended += len(m)
 }
 

storage/remote/otlptranslator/prometheus/normalize_name.go

@@ -29,7 +29,6 @@ import (
 // Prometheus best practices for units: https://prometheus.io/docs/practices/naming/#base-units
 // OpenMetrics specification for units: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#units-and-base-units
 var unitMap = map[string]string{
-
 	// Time
 	"d":   "days",
 	"h":   "hours",
@@ -111,7 +110,6 @@ func BuildCompliantName(metric pmetric.Metric, namespace string, addMetricSuffix
 
 // Build a normalized name for the specified metric
 func normalizeName(metric pmetric.Metric, namespace string) string {
-
 	// Split metric name in "tokens" (remove all non-alphanumeric)
 	nameTokens := strings.FieldsFunc(
 		metric.Name(),

storage/remote/otlptranslator/prometheus/unit_to_ucum.go

@@ -19,7 +19,6 @@ package prometheus
 import "strings"
 
 var wordToUCUM = map[string]string{
-
 	// Time
 	"days":         "d",
 	"hours":        "h",

storage/remote/queue_manager.go

@@ -36,9 +36,11 @@ import (
 	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/model/metadata"
 	"github.com/prometheus/prometheus/model/relabel"
 	"github.com/prometheus/prometheus/model/timestamp"
 	"github.com/prometheus/prometheus/prompb"
+	writev2 "github.com/prometheus/prometheus/prompb/io/prometheus/write/v2"
 	"github.com/prometheus/prometheus/scrape"
 	"github.com/prometheus/prometheus/tsdb/chunks"
 	"github.com/prometheus/prometheus/tsdb/record"
@@ -230,7 +232,7 @@ func newQueueManagerMetrics(r prometheus.Registerer, rn, e string) *queueManager
 			Namespace:   namespace,
 			Subsystem:   subsystem,
 			Name:        "queue_highest_sent_timestamp_seconds",
-			Help:        "Timestamp from a WAL sample, the highest timestamp successfully sent by this queue, in seconds since epoch.",
+			Help:        "Timestamp from a WAL sample, the highest timestamp successfully sent by this queue, in seconds since epoch. Initialized to 0 when no data has been sent yet.",
 			ConstLabels: constLabels,
 		}),
 	}
@@ -389,7 +391,7 @@ func (m *queueManagerMetrics) unregister() {
 // external timeseries database.
 type WriteClient interface {
 	// Store stores the given samples in the remote storage.
-	Store(context.Context, []byte, int) error
+	Store(ctx context.Context, req []byte, retryAttempt int) error
 	// Name uniquely identifies the remote storage.
 	Name() string
 	// Endpoint is the remote read or write endpoint for the storage client.
@@ -418,11 +420,14 @@ type QueueManager struct {
 
 	clientMtx   sync.RWMutex
 	storeClient WriteClient
+	protoMsg    config.RemoteWriteProtoMsg
+	enc         Compression
 
-	seriesMtx     sync.Mutex // Covers seriesLabels, droppedSeries and builder.
-	seriesLabels  map[chunks.HeadSeriesRef]labels.Labels
-	droppedSeries map[chunks.HeadSeriesRef]struct{}
-	builder       *labels.Builder
+	seriesMtx      sync.Mutex // Covers seriesLabels, seriesMetadata, droppedSeries and builder.
+	seriesLabels   map[chunks.HeadSeriesRef]labels.Labels
+	seriesMetadata map[chunks.HeadSeriesRef]*metadata.Metadata
+	droppedSeries  map[chunks.HeadSeriesRef]struct{}
+	builder        *labels.Builder
 
 	seriesSegmentMtx     sync.Mutex // Covers seriesSegmentIndexes - if you also lock seriesMtx, take seriesMtx first.
 	seriesSegmentIndexes map[chunks.HeadSeriesRef]int
@@ -463,6 +468,7 @@ func NewQueueManager(
 	sm ReadyScrapeManager,
 	enableExemplarRemoteWrite bool,
 	enableNativeHistogramRemoteWrite bool,
+	protoMsg config.RemoteWriteProtoMsg,
 ) *QueueManager {
 	if logger == nil {
 		logger = log.NewNopLogger()
@@ -487,6 +493,7 @@ func NewQueueManager(
 		sendNativeHistograms: enableNativeHistogramRemoteWrite,
 
 		seriesLabels:         make(map[chunks.HeadSeriesRef]labels.Labels),
+		seriesMetadata:       make(map[chunks.HeadSeriesRef]*metadata.Metadata),
 		seriesSegmentIndexes: make(map[chunks.HeadSeriesRef]int),
 		droppedSeries:        make(map[chunks.HeadSeriesRef]struct{}),
 		builder:              labels.NewBuilder(labels.EmptyLabels()),
@@ -503,9 +510,26 @@ func NewQueueManager(
 		metrics:              metrics,
 		interner:             interner,
 		highestRecvTimestamp: highestRecvTimestamp,
+
+		protoMsg: protoMsg,
+		enc:      SnappyBlockCompression, // Hardcoded for now, but scaffolding exists for likely future use.
+	}
+
+	walMetadata := false
+	if t.protoMsg != config.RemoteWriteProtoMsgV1 {
+		walMetadata = true
+	}
+	t.watcher = wlog.NewWatcher(watcherMetrics, readerMetrics, logger, client.Name(), t, dir, enableExemplarRemoteWrite, enableNativeHistogramRemoteWrite, walMetadata)
+
+	// The current MetadataWatcher implementation is mutually exclusive
+	// with the new approach, which stores metadata as WAL records and
+	// ships them alongside series. If both mechanisms are set, the new one
+	// takes precedence by implicitly disabling the older one.
+	if t.mcfg.Send && t.protoMsg != config.RemoteWriteProtoMsgV1 {
+		level.Warn(logger).Log("msg", "usage of 'metadata_config.send' is redundant when using remote write v2 (or higher) as metadata will always be gathered from the WAL and included for every series within each write request")
+		t.mcfg.Send = false
 	}
 
-	t.watcher = wlog.NewWatcher(watcherMetrics, readerMetrics, logger, client.Name(), t, dir, enableExemplarRemoteWrite, enableNativeHistogramRemoteWrite)
 	if t.mcfg.Send {
 		t.metadataWatcher = NewMetadataWatcher(logger, sm, client.Name(), t, t.mcfg.SendInterval, flushDeadline)
 	}
@@ -514,14 +538,21 @@ func NewQueueManager(
 	return t
 }
 
-// AppendMetadata sends metadata to the remote storage. Metadata is sent in batches, but is not parallelized.
-func (t *QueueManager) AppendMetadata(ctx context.Context, metadata []scrape.MetricMetadata) {
+// AppendWatcherMetadata sends metadata to the remote storage. Metadata is sent in batches, but is not parallelized.
+// This is only used for the metadata_config.send setting and 1.x Remote Write.
+func (t *QueueManager) AppendWatcherMetadata(ctx context.Context, metadata []scrape.MetricMetadata) {
+	// no op for any newer proto format, which will cache metadata sent to it from the WAL watcher.
+	if t.protoMsg != config.RemoteWriteProtoMsgV1 {
+		return
+	}
+
+	// 1.X will still get metadata in batches.
 	mm := make([]prompb.MetricMetadata, 0, len(metadata))
 	for _, entry := range metadata {
 		mm = append(mm, prompb.MetricMetadata{
 			MetricFamilyName: entry.Metric,
 			Help:             entry.Help,
-			Type:             metricTypeToMetricTypeProto(entry.Type),
+			Type:             prompb.FromMetadataType(entry.Type),
 			Unit:             entry.Unit,
 		})
 	}
@@ -542,8 +573,8 @@ func (t *QueueManager) AppendMetadata(ctx context.Context, metadata []scrape.Met
 }
 
 func (t *QueueManager) sendMetadataWithBackoff(ctx context.Context, metadata []prompb.MetricMetadata, pBuf *proto.Buffer) error {
-	// Build the WriteRequest with no samples.
-	req, _, _, err := buildWriteRequest(t.logger, nil, metadata, pBuf, nil, nil)
+	// Build the WriteRequest with no samples (v1 flow).
+	req, _, _, err := buildWriteRequest(t.logger, nil, metadata, pBuf, nil, nil, t.enc)
 	if err != nil {
 		return err
 	}
@@ -629,6 +660,36 @@ func isTimeSeriesOldFilter(metrics *queueManagerMetrics, baseTime time.Time, sam
 	}
 }
 
+func isV2TimeSeriesOldFilter(metrics *queueManagerMetrics, baseTime time.Time, sampleAgeLimit time.Duration) func(ts writev2.TimeSeries) bool {
+	return func(ts writev2.TimeSeries) bool {
+		if sampleAgeLimit == 0 {
+			// If sampleAgeLimit is unset, then we never skip samples due to their age.
+			return false
+		}
+		switch {
+		// Only the first element should be set in the series, therefore we only check the first element.
+		case len(ts.Samples) > 0:
+			if isSampleOld(baseTime, sampleAgeLimit, ts.Samples[0].Timestamp) {
+				metrics.droppedSamplesTotal.WithLabelValues(reasonTooOld).Inc()
+				return true
+			}
+		case len(ts.Histograms) > 0:
+			if isSampleOld(baseTime, sampleAgeLimit, ts.Histograms[0].Timestamp) {
+				metrics.droppedHistogramsTotal.WithLabelValues(reasonTooOld).Inc()
+				return true
+			}
+		case len(ts.Exemplars) > 0:
+			if isSampleOld(baseTime, sampleAgeLimit, ts.Exemplars[0].Timestamp) {
+				metrics.droppedExemplarsTotal.WithLabelValues(reasonTooOld).Inc()
+				return true
+			}
+		default:
+			return false
+		}
+		return false
+	}
+}
+
 // Append queues a sample to be sent to the remote storage. Blocks until all samples are
 // enqueued on their shards or a shutdown signal is received.
 func (t *QueueManager) Append(samples []record.RefSample) bool {
@@ -652,6 +713,9 @@ outer:
 			t.seriesMtx.Unlock()
 			continue
 		}
+		// TODO(cstyan): Handle or at least log an error if no metadata is found.
+		// See https://github.com/prometheus/prometheus/issues/14405
+		meta := t.seriesMetadata[s.Ref]
 		t.seriesMtx.Unlock()
 		// Start with a very small backoff. This should not be t.cfg.MinBackoff
 		// as it can happen without errors, and we want to pickup work after
@@ -666,6 +730,7 @@ outer:
 			}
 			if t.shards.enqueue(s.Ref, timeSeries{
 				seriesLabels: lbls,
+				metadata:     meta,
 				timestamp:    s.T,
 				value:        s.V,
 				sType:        tSample,
@@ -711,6 +776,7 @@ outer:
 			t.seriesMtx.Unlock()
 			continue
 		}
+		meta := t.seriesMetadata[e.Ref]
 		t.seriesMtx.Unlock()
 		// This will only loop if the queues are being resharded.
 		backoff := t.cfg.MinBackoff
@@ -722,6 +788,7 @@ outer:
 			}
 			if t.shards.enqueue(e.Ref, timeSeries{
 				seriesLabels:   lbls,
+				metadata:       meta,
 				timestamp:      e.T,
 				value:          e.V,
 				exemplarLabels: e.Labels,
@@ -765,6 +832,7 @@ outer:
 			t.seriesMtx.Unlock()
 			continue
 		}
+		meta := t.seriesMetadata[h.Ref]
 		t.seriesMtx.Unlock()
 
 		backoff := model.Duration(5 * time.Millisecond)
@@ -776,6 +844,7 @@ outer:
 			}
 			if t.shards.enqueue(h.Ref, timeSeries{
 				seriesLabels: lbls,
+				metadata:     meta,
 				timestamp:    h.T,
 				histogram:    h.H,
 				sType:        tHistogram,
@@ -818,6 +887,7 @@ outer:
 			t.seriesMtx.Unlock()
 			continue
 		}
+		meta := t.seriesMetadata[h.Ref]
 		t.seriesMtx.Unlock()
 
 		backoff := model.Duration(5 * time.Millisecond)
@@ -829,6 +899,7 @@ outer:
 			}
 			if t.shards.enqueue(h.Ref, timeSeries{
 				seriesLabels:   lbls,
+				metadata:       meta,
 				timestamp:      h.T,
 				floatHistogram: h.FH,
 				sType:          tFloatHistogram,
@@ -925,6 +996,23 @@ func (t *QueueManager) StoreSeries(series []record.RefSeries, index int) {
 	}
 }
 
+// StoreMetadata keeps track of known series' metadata for lookups when sending samples to remote.
+func (t *QueueManager) StoreMetadata(meta []record.RefMetadata) {
+	if t.protoMsg == config.RemoteWriteProtoMsgV1 {
+		return
+	}
+
+	t.seriesMtx.Lock()
+	defer t.seriesMtx.Unlock()
+	for _, m := range meta {
+		t.seriesMetadata[m.Ref] = &metadata.Metadata{
+			Type: record.ToMetricType(m.Type),
+			Unit: m.Unit,
+			Help: m.Help,
+		}
+	}
+}
+
 // UpdateSeriesSegment updates the segment number held against the series,
 // so we can trim older ones in SeriesReset.
 func (t *QueueManager) UpdateSeriesSegment(series []record.RefSeries, index int) {
@@ -950,6 +1038,7 @@ func (t *QueueManager) SeriesReset(index int) {
 			delete(t.seriesSegmentIndexes, k)
 			t.releaseLabels(t.seriesLabels[k])
 			delete(t.seriesLabels, k)
+			delete(t.seriesMetadata, k)
 			delete(t.droppedSeries, k)
 		}
 	}
@@ -1165,6 +1254,7 @@ type shards struct {
 	samplesDroppedOnHardShutdown    atomic.Uint32
 	exemplarsDroppedOnHardShutdown  atomic.Uint32
 	histogramsDroppedOnHardShutdown atomic.Uint32
+	metadataDroppedOnHardShutdown   atomic.Uint32
 }
 
 // start the shards; must be called before any call to enqueue.
@@ -1193,6 +1283,7 @@ func (s *shards) start(n int) {
 	s.samplesDroppedOnHardShutdown.Store(0)
 	s.exemplarsDroppedOnHardShutdown.Store(0)
 	s.histogramsDroppedOnHardShutdown.Store(0)
+	s.metadataDroppedOnHardShutdown.Store(0)
 	for i := 0; i < n; i++ {
 		go s.runShard(hardShutdownCtx, i, newQueues[i])
 	}
@@ -1245,7 +1336,6 @@ func (s *shards) stop() {
 func (s *shards) enqueue(ref chunks.HeadSeriesRef, data timeSeries) bool {
 	s.mtx.RLock()
 	defer s.mtx.RUnlock()
-
 	shard := uint64(ref) % uint64(len(s.queues))
 	select {
 	case <-s.softShutdown:
@@ -1288,6 +1378,7 @@ type timeSeries struct {
 	value          float64
 	histogram      *histogram.Histogram
 	floatHistogram *histogram.FloatHistogram
+	metadata       *metadata.Metadata
 	timestamp      int64
 	exemplarLabels labels.Labels
 	// The type of series: sample, exemplar, or histogram.
@@ -1301,6 +1392,7 @@ const (
 	tExemplar
 	tHistogram
 	tFloatHistogram
+	tMetadata
 )
 
 func newQueue(batchSize, capacity int) *queue {
@@ -1324,6 +1416,10 @@ func newQueue(batchSize, capacity int) *queue {
 func (q *queue) Append(datum timeSeries) bool {
 	q.batchMtx.Lock()
 	defer q.batchMtx.Unlock()
+	// TODO(cstyan): Check if metadata now means we've reduced the total # of samples
+	// we can batch together here, and if so find a way to not include metadata
+	// in the batch size calculation.
+	// See https://github.com/prometheus/prometheus/issues/14405
 	q.batch = append(q.batch, datum)
 	if len(q.batch) == cap(q.batch) {
 		select {
@@ -1347,7 +1443,6 @@ func (q *queue) Chan() <-chan []timeSeries {
 func (q *queue) Batch() []timeSeries {
 	q.batchMtx.Lock()
 	defer q.batchMtx.Unlock()
-
 	select {
 	case batch := <-q.batchQueue:
 		return batch
@@ -1373,6 +1468,8 @@ func (q *queue) FlushAndShutdown(done <-chan struct{}) {
 	for q.tryEnqueueingBatch(done) {
 		time.Sleep(time.Second)
 	}
+	q.batchMtx.Lock()
+	defer q.batchMtx.Unlock()
 	q.batch = nil
 	close(q.batchQueue)
 }
@@ -1419,19 +1516,23 @@ func (s *shards) runShard(ctx context.Context, shardID int, queue *queue) {
 	}()
 
 	shardNum := strconv.Itoa(shardID)
+	symbolTable := writev2.NewSymbolTable()
 
 	// Send batches of at most MaxSamplesPerSend samples to the remote storage.
 	// If we have fewer samples than that, flush them out after a deadline anyways.
 	var (
 		max = s.qm.cfg.MaxSamplesPerSend
 
-		pBuf = proto.NewBuffer(nil)
-		buf  []byte
+		pBuf    = proto.NewBuffer(nil)
+		pBufRaw []byte
+		buf     []byte
 	)
+	// TODO(@tpaschalis) Should we also raise the max if we have WAL metadata?
 	if s.qm.sendExemplars {
 		max += int(float64(max) * 0.1)
 	}
 
+	// TODO: Dry all of this, we should make an interface/generic for the timeseries type.
 	batchQueue := queue.Chan()
 	pendingData := make([]prompb.TimeSeries, max)
 	for i := range pendingData {
@@ -1440,6 +1541,10 @@ func (s *shards) runShard(ctx context.Context, shardID int, queue *queue) {
 			pendingData[i].Exemplars = []prompb.Exemplar{{}}
 		}
 	}
+	pendingDataV2 := make([]writev2.TimeSeries, max)
+	for i := range pendingDataV2 {
+		pendingDataV2[i].Samples = []writev2.Sample{{}}
+	}
 
 	timer := time.NewTimer(time.Duration(s.qm.cfg.BatchSendDeadline))
 	stop := func() {
@@ -1452,6 +1557,24 @@ func (s *shards) runShard(ctx context.Context, shardID int, queue *queue) {
 	}
 	defer stop()
 
+	sendBatch := func(batch []timeSeries, protoMsg config.RemoteWriteProtoMsg, enc Compression, timer bool) {
+		switch protoMsg {
+		case config.RemoteWriteProtoMsgV1:
+			nPendingSamples, nPendingExemplars, nPendingHistograms := populateTimeSeries(batch, pendingData, s.qm.sendExemplars, s.qm.sendNativeHistograms)
+			n := nPendingSamples + nPendingExemplars + nPendingHistograms
+			if timer {
+				level.Debug(s.qm.logger).Log("msg", "runShard timer ticked, sending buffered data", "samples", nPendingSamples,
+					"exemplars", nPendingExemplars, "shard", shardNum, "histograms", nPendingHistograms)
+			}
+			_ = s.sendSamples(ctx, pendingData[:n], nPendingSamples, nPendingExemplars, nPendingHistograms, pBuf, &buf, enc)
+		case config.RemoteWriteProtoMsgV2:
+			nPendingSamples, nPendingExemplars, nPendingHistograms, nPendingMetadata := populateV2TimeSeries(&symbolTable, batch, pendingDataV2, s.qm.sendExemplars, s.qm.sendNativeHistograms)
+			n := nPendingSamples + nPendingExemplars + nPendingHistograms
+			_ = s.sendV2Samples(ctx, pendingDataV2[:n], symbolTable.Symbols(), nPendingSamples, nPendingExemplars, nPendingHistograms, nPendingMetadata, &pBufRaw, &buf, enc)
+			symbolTable.Reset()
+		}
+	}
+
 	for {
 		select {
 		case <-ctx.Done():
@@ -1475,10 +1598,11 @@ func (s *shards) runShard(ctx context.Context, shardID int, queue *queue) {
 			if !ok {
 				return
 			}
-			nPendingSamples, nPendingExemplars, nPendingHistograms := s.populateTimeSeries(batch, pendingData)
+
+			sendBatch(batch, s.qm.protoMsg, s.qm.enc, false)
+			// TODO(bwplotka): Previously the return was between popular and send.
+			// Consider this when DRY-ing https://github.com/prometheus/prometheus/issues/14409
 			queue.ReturnForReuse(batch)
-			n := nPendingSamples + nPendingExemplars + nPendingHistograms
-			s.sendSamples(ctx, pendingData[:n], nPendingSamples, nPendingExemplars, nPendingHistograms, pBuf, &buf)
 
 			stop()
 			timer.Reset(time.Duration(s.qm.cfg.BatchSendDeadline))
@@ -1486,11 +1610,7 @@ func (s *shards) runShard(ctx context.Context, shardID int, queue *queue) {
 		case <-timer.C:
 			batch := queue.Batch()
 			if len(batch) > 0 {
-				nPendingSamples, nPendingExemplars, nPendingHistograms := s.populateTimeSeries(batch, pendingData)
-				n := nPendingSamples + nPendingExemplars + nPendingHistograms
-				level.Debug(s.qm.logger).Log("msg", "runShard timer ticked, sending buffered data", "samples", nPendingSamples,
-					"exemplars", nPendingExemplars, "shard", shardNum, "histograms", nPendingHistograms)
-				s.sendSamples(ctx, pendingData[:n], nPendingSamples, nPendingExemplars, nPendingHistograms, pBuf, &buf)
+				sendBatch(batch, s.qm.protoMsg, s.qm.enc, true)
 			}
 			queue.ReturnForReuse(batch)
 			timer.Reset(time.Duration(s.qm.cfg.BatchSendDeadline))
@@ -1498,21 +1618,22 @@ func (s *shards) runShard(ctx context.Context, shardID int, queue *queue) {
 	}
 }
 
-func (s *shards) populateTimeSeries(batch []timeSeries, pendingData []prompb.TimeSeries) (int, int, int) {
+func populateTimeSeries(batch []timeSeries, pendingData []prompb.TimeSeries, sendExemplars, sendNativeHistograms bool) (int, int, int) {
 	var nPendingSamples, nPendingExemplars, nPendingHistograms int
 	for nPending, d := range batch {
 		pendingData[nPending].Samples = pendingData[nPending].Samples[:0]
-		if s.qm.sendExemplars {
+		if sendExemplars {
 			pendingData[nPending].Exemplars = pendingData[nPending].Exemplars[:0]
 		}
-		if s.qm.sendNativeHistograms {
+		if sendNativeHistograms {
 			pendingData[nPending].Histograms = pendingData[nPending].Histograms[:0]
 		}
 
 		// Number of pending samples is limited by the fact that sendSamples (via sendSamplesWithBackoff)
 		// retries endlessly, so once we reach max samples, if we can never send to the endpoint we'll
 		// stop reading from the queue. This makes it safe to reference pendingSamples by index.
-		pendingData[nPending].Labels = LabelsToLabelsProto(d.seriesLabels, pendingData[nPending].Labels)
+		pendingData[nPending].Labels = prompb.FromLabels(d.seriesLabels, pendingData[nPending].Labels)
+
 		switch d.sType {
 		case tSample:
 			pendingData[nPending].Samples = append(pendingData[nPending].Samples, prompb.Sample{
@@ -1522,25 +1643,39 @@ func (s *shards) populateTimeSeries(batch []timeSeries, pendingData []prompb.Tim
 			nPendingSamples++
 		case tExemplar:
 			pendingData[nPending].Exemplars = append(pendingData[nPending].Exemplars, prompb.Exemplar{
-				Labels:    LabelsToLabelsProto(d.exemplarLabels, nil),
+				Labels:    prompb.FromLabels(d.exemplarLabels, nil),
 				Value:     d.value,
 				Timestamp: d.timestamp,
 			})
 			nPendingExemplars++
 		case tHistogram:
-			pendingData[nPending].Histograms = append(pendingData[nPending].Histograms, HistogramToHistogramProto(d.timestamp, d.histogram))
+			pendingData[nPending].Histograms = append(pendingData[nPending].Histograms, prompb.FromIntHistogram(d.timestamp, d.histogram))
 			nPendingHistograms++
 		case tFloatHistogram:
-			pendingData[nPending].Histograms = append(pendingData[nPending].Histograms, FloatHistogramToHistogramProto(d.timestamp, d.floatHistogram))
+			pendingData[nPending].Histograms = append(pendingData[nPending].Histograms, prompb.FromFloatHistogram(d.timestamp, d.floatHistogram))
 			nPendingHistograms++
 		}
 	}
 	return nPendingSamples, nPendingExemplars, nPendingHistograms
 }
 
-func (s *shards) sendSamples(ctx context.Context, samples []prompb.TimeSeries, sampleCount, exemplarCount, histogramCount int, pBuf *proto.Buffer, buf *[]byte) {
+func (s *shards) sendSamples(ctx context.Context, samples []prompb.TimeSeries, sampleCount, exemplarCount, histogramCount int, pBuf *proto.Buffer, buf *[]byte, enc Compression) error {
 	begin := time.Now()
-	err := s.sendSamplesWithBackoff(ctx, samples, sampleCount, exemplarCount, histogramCount, pBuf, buf)
+	err := s.sendSamplesWithBackoff(ctx, samples, sampleCount, exemplarCount, histogramCount, 0, pBuf, buf, enc)
+	s.updateMetrics(ctx, err, sampleCount, exemplarCount, histogramCount, 0, time.Since(begin))
+	return err
+}
+
+// TODO(bwplotka): DRY this (have one logic for both v1 and v2).
+// See https://github.com/prometheus/prometheus/issues/14409
+func (s *shards) sendV2Samples(ctx context.Context, samples []writev2.TimeSeries, labels []string, sampleCount, exemplarCount, histogramCount, metadataCount int, pBuf, buf *[]byte, enc Compression) error {
+	begin := time.Now()
+	err := s.sendV2SamplesWithBackoff(ctx, samples, labels, sampleCount, exemplarCount, histogramCount, metadataCount, pBuf, buf, enc)
+	s.updateMetrics(ctx, err, sampleCount, exemplarCount, histogramCount, metadataCount, time.Since(begin))
+	return err
+}
+
+func (s *shards) updateMetrics(_ context.Context, err error, sampleCount, exemplarCount, histogramCount, metadataCount int, duration time.Duration) {
 	if err != nil {
 		level.Error(s.qm.logger).Log("msg", "non-recoverable error", "count", sampleCount, "exemplarCount", exemplarCount, "histogramCount", histogramCount, "err", err)
 		s.qm.metrics.failedSamplesTotal.Add(float64(sampleCount))
@@ -1550,8 +1685,8 @@ func (s *shards) sendSamples(ctx context.Context, samples []prompb.TimeSeries, s
 
 	// These counters are used to calculate the dynamic sharding, and as such
 	// should be maintained irrespective of success or failure.
-	s.qm.dataOut.incr(int64(len(samples)))
-	s.qm.dataOutDuration.incr(int64(time.Since(begin)))
+	s.qm.dataOut.incr(int64(sampleCount + exemplarCount + histogramCount + metadataCount))
+	s.qm.dataOutDuration.incr(int64(duration))
 	s.qm.lastSendTimestamp.Store(time.Now().Unix())
 	// Pending samples/exemplars/histograms also should be subtracted, as an error means
 	// they will not be retried.
@@ -1564,9 +1699,9 @@ func (s *shards) sendSamples(ctx context.Context, samples []prompb.TimeSeries, s
 }
 
 // sendSamples to the remote storage with backoff for recoverable errors.
-func (s *shards) sendSamplesWithBackoff(ctx context.Context, samples []prompb.TimeSeries, sampleCount, exemplarCount, histogramCount int, pBuf *proto.Buffer, buf *[]byte) error {
+func (s *shards) sendSamplesWithBackoff(ctx context.Context, samples []prompb.TimeSeries, sampleCount, exemplarCount, histogramCount, metadataCount int, pBuf *proto.Buffer, buf *[]byte, enc Compression) error {
 	// Build the WriteRequest with no metadata.
-	req, highest, lowest, err := buildWriteRequest(s.qm.logger, samples, nil, pBuf, *buf, nil)
+	req, highest, lowest, err := buildWriteRequest(s.qm.logger, samples, nil, pBuf, buf, nil, enc)
 	s.qm.buildRequestLimitTimestamp.Store(lowest)
 	if err != nil {
 		// Failing to build the write request is non-recoverable, since it will
@@ -1590,8 +1725,9 @@ func (s *shards) sendSamplesWithBackoff(ctx context.Context, samples []prompb.Ti
 				samples,
 				nil,
 				pBuf,
-				*buf,
+				buf,
 				isTimeSeriesOldFilter(s.qm.metrics, currentTime, time.Duration(s.qm.cfg.SampleAgeLimit)),
+				enc,
 			)
 			s.qm.buildRequestLimitTimestamp.Store(lowest)
 			if err != nil {
@@ -1622,6 +1758,7 @@ func (s *shards) sendSamplesWithBackoff(ctx context.Context, samples []prompb.Ti
 		s.qm.metrics.samplesTotal.Add(float64(sampleCount))
 		s.qm.metrics.exemplarsTotal.Add(float64(exemplarCount))
 		s.qm.metrics.histogramsTotal.Add(float64(histogramCount))
+		s.qm.metrics.metadataTotal.Add(float64(metadataCount))
 		err := s.qm.client().Store(ctx, *buf, try)
 		s.qm.metrics.sentBatchDuration.Observe(time.Since(begin).Seconds())
 
@@ -1652,6 +1789,148 @@ func (s *shards) sendSamplesWithBackoff(ctx context.Context, samples []prompb.Ti
 	return err
 }
 
+// sendV2Samples to the remote storage with backoff for recoverable errors.
+func (s *shards) sendV2SamplesWithBackoff(ctx context.Context, samples []writev2.TimeSeries, labels []string, sampleCount, exemplarCount, histogramCount, metadataCount int, pBuf, buf *[]byte, enc Compression) error {
+	// Build the WriteRequest with no metadata.
+	req, highest, lowest, err := buildV2WriteRequest(s.qm.logger, samples, labels, pBuf, buf, nil, enc)
+	s.qm.buildRequestLimitTimestamp.Store(lowest)
+	if err != nil {
+		// Failing to build the write request is non-recoverable, since it will
+		// only error if marshaling the proto to bytes fails.
+		return err
+	}
+
+	reqSize := len(req)
+	*buf = req
+
+	// An anonymous function allows us to defer the completion of our per-try spans
+	// without causing a memory leak, and it has the nice effect of not propagating any
+	// parameters for sendSamplesWithBackoff/3.
+	attemptStore := func(try int) error {
+		currentTime := time.Now()
+		lowest := s.qm.buildRequestLimitTimestamp.Load()
+		if isSampleOld(currentTime, time.Duration(s.qm.cfg.SampleAgeLimit), lowest) {
+			// This will filter out old samples during retries.
+			req, _, lowest, err := buildV2WriteRequest(
+				s.qm.logger,
+				samples,
+				labels,
+				pBuf,
+				buf,
+				isV2TimeSeriesOldFilter(s.qm.metrics, currentTime, time.Duration(s.qm.cfg.SampleAgeLimit)),
+				enc,
+			)
+			s.qm.buildRequestLimitTimestamp.Store(lowest)
+			if err != nil {
+				return err
+			}
+			*buf = req
+		}
+
+		ctx, span := otel.Tracer("").Start(ctx, "Remote Send Batch")
+		defer span.End()
+
+		span.SetAttributes(
+			attribute.Int("request_size", reqSize),
+			attribute.Int("samples", sampleCount),
+			attribute.Int("try", try),
+			attribute.String("remote_name", s.qm.storeClient.Name()),
+			attribute.String("remote_url", s.qm.storeClient.Endpoint()),
+		)
+
+		if exemplarCount > 0 {
+			span.SetAttributes(attribute.Int("exemplars", exemplarCount))
+		}
+		if histogramCount > 0 {
+			span.SetAttributes(attribute.Int("histograms", histogramCount))
+		}
+
+		begin := time.Now()
+		s.qm.metrics.samplesTotal.Add(float64(sampleCount))
+		s.qm.metrics.exemplarsTotal.Add(float64(exemplarCount))
+		s.qm.metrics.histogramsTotal.Add(float64(histogramCount))
+		s.qm.metrics.metadataTotal.Add(float64(metadataCount))
+		err := s.qm.client().Store(ctx, *buf, try)
+		s.qm.metrics.sentBatchDuration.Observe(time.Since(begin).Seconds())
+
+		if err != nil {
+			span.RecordError(err)
+			return err
+		}
+
+		return nil
+	}
+
+	onRetry := func() {
+		s.qm.metrics.retriedSamplesTotal.Add(float64(sampleCount))
+		s.qm.metrics.retriedExemplarsTotal.Add(float64(exemplarCount))
+		s.qm.metrics.retriedHistogramsTotal.Add(float64(histogramCount))
+	}
+
+	err = s.qm.sendWriteRequestWithBackoff(ctx, attemptStore, onRetry)
+	if errors.Is(err, context.Canceled) {
+		// When there is resharding, we cancel the context for this queue, which means the data is not sent.
+		// So we exit early to not update the metrics.
+		return err
+	}
+
+	s.qm.metrics.sentBytesTotal.Add(float64(reqSize))
+	s.qm.metrics.highestSentTimestamp.Set(float64(highest / 1000))
+
+	return err
+}
+
+func populateV2TimeSeries(symbolTable *writev2.SymbolsTable, batch []timeSeries, pendingData []writev2.TimeSeries, sendExemplars, sendNativeHistograms bool) (int, int, int, int) {
+	var nPendingSamples, nPendingExemplars, nPendingHistograms, nPendingMetadata int
+	for nPending, d := range batch {
+		pendingData[nPending].Samples = pendingData[nPending].Samples[:0]
+		// todo: should we also safeguard against empty metadata here?
+		if d.metadata != nil {
+			pendingData[nPending].Metadata.Type = writev2.FromMetadataType(d.metadata.Type)
+			pendingData[nPending].Metadata.HelpRef = symbolTable.Symbolize(d.metadata.Help)
+			pendingData[nPending].Metadata.HelpRef = symbolTable.Symbolize(d.metadata.Unit)
+			nPendingMetadata++
+		}
+
+		if sendExemplars {
+			pendingData[nPending].Exemplars = pendingData[nPending].Exemplars[:0]
+		}
+		if sendNativeHistograms {
+			pendingData[nPending].Histograms = pendingData[nPending].Histograms[:0]
+		}
+
+		// Number of pending samples is limited by the fact that sendSamples (via sendSamplesWithBackoff)
+		// retries endlessly, so once we reach max samples, if we can never send to the endpoint we'll
+		// stop reading from the queue. This makes it safe to reference pendingSamples by index.
+		pendingData[nPending].LabelsRefs = symbolTable.SymbolizeLabels(d.seriesLabels, pendingData[nPending].LabelsRefs)
+		switch d.sType {
+		case tSample:
+			pendingData[nPending].Samples = append(pendingData[nPending].Samples, writev2.Sample{
+				Value:     d.value,
+				Timestamp: d.timestamp,
+			})
+			nPendingSamples++
+		case tExemplar:
+			pendingData[nPending].Exemplars = append(pendingData[nPending].Exemplars, writev2.Exemplar{
+				LabelsRefs: symbolTable.SymbolizeLabels(d.exemplarLabels, nil), // TODO: optimize, reuse slice
+				Value:      d.value,
+				Timestamp:  d.timestamp,
+			})
+			nPendingExemplars++
+		case tHistogram:
+			pendingData[nPending].Histograms = append(pendingData[nPending].Histograms, writev2.FromIntHistogram(d.timestamp, d.histogram))
+			nPendingHistograms++
+		case tFloatHistogram:
+			pendingData[nPending].Histograms = append(pendingData[nPending].Histograms, writev2.FromFloatHistogram(d.timestamp, d.floatHistogram))
+			nPendingHistograms++
+		case tMetadata:
+			// TODO: log or return an error?
+			// we shouldn't receive metadata type data here, it should already be inserted into the timeSeries
+		}
+	}
+	return nPendingSamples, nPendingExemplars, nPendingHistograms, nPendingMetadata
+}
+
 func (t *QueueManager) sendWriteRequestWithBackoff(ctx context.Context, attempt func(int) error, onRetry func()) error {
 	backoff := t.cfg.MinBackoff
 	sleepDuration := model.Duration(0)
@@ -1795,7 +2074,21 @@ func buildTimeSeries(timeSeries []prompb.TimeSeries, filter func(prompb.TimeSeri
 	return highest, lowest, timeSeries, droppedSamples, droppedExemplars, droppedHistograms
 }
 
-func buildWriteRequest(logger log.Logger, timeSeries []prompb.TimeSeries, metadata []prompb.MetricMetadata, pBuf *proto.Buffer, buf []byte, filter func(prompb.TimeSeries) bool) ([]byte, int64, int64, error) {
+func compressPayload(tmpbuf *[]byte, inp []byte, enc Compression) (compressed []byte, _ error) {
+	switch enc {
+	case SnappyBlockCompression:
+		compressed = snappy.Encode(*tmpbuf, inp)
+		if n := snappy.MaxEncodedLen(len(inp)); n > len(*tmpbuf) {
+			// grow the buffer for the next time
+			*tmpbuf = make([]byte, n)
+		}
+		return compressed, nil
+	default:
+		return compressed, fmt.Errorf("Unknown compression scheme [%v]", enc)
+	}
+}
+
+func buildWriteRequest(logger log.Logger, timeSeries []prompb.TimeSeries, metadata []prompb.MetricMetadata, pBuf *proto.Buffer, buf *[]byte, filter func(prompb.TimeSeries) bool, enc Compression) (compressed []byte, highest, lowest int64, _ error) {
 	highest, lowest, timeSeries,
 		droppedSamples, droppedExemplars, droppedHistograms := buildTimeSeries(timeSeries, filter)
 
@@ -1821,8 +2114,105 @@ func buildWriteRequest(logger log.Logger, timeSeries []prompb.TimeSeries, metada
 	// snappy uses len() to see if it needs to allocate a new slice. Make the
 	// buffer as long as possible.
 	if buf != nil {
-		buf = buf[0:cap(buf)]
+		*buf = (*buf)[0:cap(*buf)]
+	} else {
+		buf = &[]byte{}
+	}
+
+	compressed, err = compressPayload(buf, pBuf.Bytes(), enc)
+	if err != nil {
+		return nil, highest, lowest, err
 	}
-	compressed := snappy.Encode(buf, pBuf.Bytes())
 	return compressed, highest, lowest, nil
 }
+
+func buildV2WriteRequest(logger log.Logger, samples []writev2.TimeSeries, labels []string, pBuf, buf *[]byte, filter func(writev2.TimeSeries) bool, enc Compression) (compressed []byte, highest, lowest int64, _ error) {
+	highest, lowest, timeSeries, droppedSamples, droppedExemplars, droppedHistograms := buildV2TimeSeries(samples, filter)
+
+	if droppedSamples > 0 || droppedExemplars > 0 || droppedHistograms > 0 {
+		level.Debug(logger).Log("msg", "dropped data due to their age", "droppedSamples", droppedSamples, "droppedExemplars", droppedExemplars, "droppedHistograms", droppedHistograms)
+	}
+
+	req := &writev2.Request{
+		Symbols:    labels,
+		Timeseries: timeSeries,
+	}
+
+	if pBuf == nil {
+		pBuf = &[]byte{} // For convenience in tests. Not efficient.
+	}
+
+	data, err := req.OptimizedMarshal(*pBuf)
+	if err != nil {
+		return nil, highest, lowest, err
+	}
+	*pBuf = data
+
+	// snappy uses len() to see if it needs to allocate a new slice. Make the
+	// buffer as long as possible.
+	if buf != nil {
+		*buf = (*buf)[0:cap(*buf)]
+	} else {
+		buf = &[]byte{}
+	}
+
+	compressed, err = compressPayload(buf, data, enc)
+	if err != nil {
+		return nil, highest, lowest, err
+	}
+	return compressed, highest, lowest, nil
+}
+
+func buildV2TimeSeries(timeSeries []writev2.TimeSeries, filter func(writev2.TimeSeries) bool) (int64, int64, []writev2.TimeSeries, int, int, int) {
+	var highest int64
+	var lowest int64
+	var droppedSamples, droppedExemplars, droppedHistograms int
+
+	keepIdx := 0
+	lowest = math.MaxInt64
+	for i, ts := range timeSeries {
+		if filter != nil && filter(ts) {
+			if len(ts.Samples) > 0 {
+				droppedSamples++
+			}
+			if len(ts.Exemplars) > 0 {
+				droppedExemplars++
+			}
+			if len(ts.Histograms) > 0 {
+				droppedHistograms++
+			}
+			continue
+		}
+
+		// At the moment we only ever append a TimeSeries with a single sample or exemplar in it.
+		if len(ts.Samples) > 0 && ts.Samples[0].Timestamp > highest {
+			highest = ts.Samples[0].Timestamp
+		}
+		if len(ts.Exemplars) > 0 && ts.Exemplars[0].Timestamp > highest {
+			highest = ts.Exemplars[0].Timestamp
+		}
+		if len(ts.Histograms) > 0 && ts.Histograms[0].Timestamp > highest {
+			highest = ts.Histograms[0].Timestamp
+		}
+
+		// Get the lowest timestamp.
+		if len(ts.Samples) > 0 && ts.Samples[0].Timestamp < lowest {
+			lowest = ts.Samples[0].Timestamp
+		}
+		if len(ts.Exemplars) > 0 && ts.Exemplars[0].Timestamp < lowest {
+			lowest = ts.Exemplars[0].Timestamp
+		}
+		if len(ts.Histograms) > 0 && ts.Histograms[0].Timestamp < lowest {
+			lowest = ts.Histograms[0].Timestamp
+		}
+		if i != keepIdx {
+			// We have to swap the kept timeseries with the one which should be dropped.
+			// Copying any elements within timeSeries could cause data corruptions when reusing the slice in a next batch (shards.populateTimeSeries).
+			timeSeries[keepIdx], timeSeries[i] = timeSeries[i], timeSeries[keepIdx]
+		}
+		keepIdx++
+	}
+
+	timeSeries = timeSeries[:keepIdx]
+	return highest, lowest, timeSeries, droppedSamples, droppedExemplars, droppedHistograms
+}

storage/remote/read.go

@@ -210,13 +210,13 @@ func (q querier) addExternalLabels(ms []*labels.Matcher) ([]*labels.Matcher, []s
 }
 
 // LabelValues implements storage.Querier and is a noop.
-func (q *querier) LabelValues(context.Context, string, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+func (q *querier) LabelValues(context.Context, string, *storage.LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
 	// TODO: Implement: https://github.com/prometheus/prometheus/issues/3351
 	return nil, nil, errors.New("not implemented")
 }
 
 // LabelNames implements storage.Querier and is a noop.
-func (q *querier) LabelNames(context.Context, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+func (q *querier) LabelNames(context.Context, *storage.LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
 	// TODO: Implement: https://github.com/prometheus/prometheus/issues/3351
 	return nil, nil, errors.New("not implemented")
 }

storage/remote/read_handler_test.go

@@ -124,7 +124,7 @@ func TestSampledReadEndpoint(t *testing.T) {
 					{Name: "d", Value: "e"},
 				},
 				Histograms: []prompb.Histogram{
-					FloatHistogramToHistogramProto(0, tsdbutil.GenerateTestFloatHistogram(0)),
+					prompb.FromFloatHistogram(0, tsdbutil.GenerateTestFloatHistogram(0)),
 				},
 			},
 		},

storage/remote/read_test.go

@@ -92,7 +92,7 @@ func TestNoDuplicateReadConfigs(t *testing.T) {
 
 	for _, tc := range cases {
 		t.Run("", func(t *testing.T) {
-			s := NewStorage(nil, nil, nil, dir, defaultFlushDeadline, nil)
+			s := NewStorage(nil, nil, nil, dir, defaultFlushDeadline, nil, false)
 			conf := &config.Config{
 				GlobalConfig:      config.DefaultGlobalConfig,
 				RemoteReadConfigs: tc.cfgs,
@@ -172,12 +172,12 @@ func TestSeriesSetFilter(t *testing.T) {
 			toRemove: []string{"foo"},
 			in: &prompb.QueryResult{
 				Timeseries: []*prompb.TimeSeries{
-					{Labels: LabelsToLabelsProto(labels.FromStrings("foo", "bar", "a", "b"), nil)},
+					{Labels: prompb.FromLabels(labels.FromStrings("foo", "bar", "a", "b"), nil)},
 				},
 			},
 			expected: &prompb.QueryResult{
 				Timeseries: []*prompb.TimeSeries{
-					{Labels: LabelsToLabelsProto(labels.FromStrings("a", "b"), nil)},
+					{Labels: prompb.FromLabels(labels.FromStrings("a", "b"), nil)},
 				},
 			},
 		},
@@ -211,7 +211,7 @@ func (c *mockedRemoteClient) Read(_ context.Context, query *prompb.Query) (*prom
 
 	q := &prompb.QueryResult{}
 	for _, s := range c.store {
-		l := LabelProtosToLabels(&c.b, s.Labels)
+		l := s.ToLabels(&c.b, nil)
 		var notMatch bool
 
 		for _, m := range matchers {

storage/remote/storage.go

@@ -62,7 +62,7 @@ type Storage struct {
 }
 
 // NewStorage returns a remote.Storage.
-func NewStorage(l log.Logger, reg prometheus.Registerer, stCallback startTimeCallback, walDir string, flushDeadline time.Duration, sm ReadyScrapeManager) *Storage {
+func NewStorage(l log.Logger, reg prometheus.Registerer, stCallback startTimeCallback, walDir string, flushDeadline time.Duration, sm ReadyScrapeManager, metadataInWAL bool) *Storage {
 	if l == nil {
 		l = log.NewNopLogger()
 	}
@@ -72,7 +72,7 @@ func NewStorage(l log.Logger, reg prometheus.Registerer, stCallback startTimeCal
 		logger:                 logger,
 		localStartTimeCallback: stCallback,
 	}
-	s.rws = NewWriteStorage(s.logger, reg, walDir, flushDeadline, sm)
+	s.rws = NewWriteStorage(s.logger, reg, walDir, flushDeadline, sm, metadataInWAL)
 	return s
 }
 

storage/remote/storage_test.go

@@ -29,7 +29,7 @@ import (
 func TestStorageLifecycle(t *testing.T) {
 	dir := t.TempDir()
 
-	s := NewStorage(nil, nil, nil, dir, defaultFlushDeadline, nil)
+	s := NewStorage(nil, nil, nil, dir, defaultFlushDeadline, nil, false)
 	conf := &config.Config{
 		GlobalConfig: config.DefaultGlobalConfig,
 		RemoteWriteConfigs: []*config.RemoteWriteConfig{
@@ -56,7 +56,7 @@ func TestStorageLifecycle(t *testing.T) {
 func TestUpdateRemoteReadConfigs(t *testing.T) {
 	dir := t.TempDir()
 
-	s := NewStorage(nil, nil, nil, dir, defaultFlushDeadline, nil)
+	s := NewStorage(nil, nil, nil, dir, defaultFlushDeadline, nil, false)
 
 	conf := &config.Config{
 		GlobalConfig: config.GlobalConfig{},
@@ -77,7 +77,7 @@ func TestUpdateRemoteReadConfigs(t *testing.T) {
 func TestFilterExternalLabels(t *testing.T) {
 	dir := t.TempDir()
 
-	s := NewStorage(nil, nil, nil, dir, defaultFlushDeadline, nil)
+	s := NewStorage(nil, nil, nil, dir, defaultFlushDeadline, nil, false)
 
 	conf := &config.Config{
 		GlobalConfig: config.GlobalConfig{
@@ -102,7 +102,7 @@ func TestFilterExternalLabels(t *testing.T) {
 func TestIgnoreExternalLabels(t *testing.T) {
 	dir := t.TempDir()
 
-	s := NewStorage(nil, nil, nil, dir, defaultFlushDeadline, nil)
+	s := NewStorage(nil, nil, nil, dir, defaultFlushDeadline, nil, false)
 
 	conf := &config.Config{
 		GlobalConfig: config.GlobalConfig{
@@ -154,7 +154,7 @@ func baseRemoteReadConfig(host string) *config.RemoteReadConfig {
 // ApplyConfig runs concurrently with Notify
 // See https://github.com/prometheus/prometheus/issues/12747
 func TestWriteStorageApplyConfigsDuringCommit(t *testing.T) {
-	s := NewStorage(nil, nil, nil, t.TempDir(), defaultFlushDeadline, nil)
+	s := NewStorage(nil, nil, nil, t.TempDir(), defaultFlushDeadline, nil, false)
 
 	var wg sync.WaitGroup
 	wg.Add(2000)

storage/remote/write.go

@@ -15,6 +15,7 @@ package remote
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"math"
 	"sync"
@@ -65,6 +66,7 @@ type WriteStorage struct {
 	externalLabels    labels.Labels
 	dir               string
 	queues            map[string]*QueueManager
+	metadataInWAL     bool
 	samplesIn         *ewmaRate
 	flushDeadline     time.Duration
 	interner          *pool
@@ -76,7 +78,7 @@ type WriteStorage struct {
 }
 
 // NewWriteStorage creates and runs a WriteStorage.
-func NewWriteStorage(logger log.Logger, reg prometheus.Registerer, dir string, flushDeadline time.Duration, sm ReadyScrapeManager) *WriteStorage {
+func NewWriteStorage(logger log.Logger, reg prometheus.Registerer, dir string, flushDeadline time.Duration, sm ReadyScrapeManager, metadataInWal bool) *WriteStorage {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -92,12 +94,13 @@ func NewWriteStorage(logger log.Logger, reg prometheus.Registerer, dir string, f
 		interner:          newPool(),
 		scraper:           sm,
 		quit:              make(chan struct{}),
+		metadataInWAL:     metadataInWal,
 		highestTimestamp: &maxTimestamp{
 			Gauge: prometheus.NewGauge(prometheus.GaugeOpts{
 				Namespace: namespace,
 				Subsystem: subsystem,
 				Name:      "highest_timestamp_in_seconds",
-				Help:      "Highest timestamp that has come into the remote storage via the Appender interface, in seconds since epoch.",
+				Help:      "Highest timestamp that has come into the remote storage via the Appender interface, in seconds since epoch. Initialized to 0 when no data has been received yet.",
 			}),
 		},
 	}
@@ -145,6 +148,9 @@ func (rws *WriteStorage) ApplyConfig(conf *config.Config) error {
 	newQueues := make(map[string]*QueueManager)
 	newHashes := []string{}
 	for _, rwConf := range conf.RemoteWriteConfigs {
+		if rwConf.ProtobufMessage == config.RemoteWriteProtoMsgV2 && !rws.metadataInWAL {
+			return errors.New("invalid remote write configuration, if you are using remote write version 2.0 the `--enable-feature=metadata-wal-records` feature flag must be enabled")
+		}
 		hash, err := toHash(rwConf)
 		if err != nil {
 			return err
@@ -165,6 +171,7 @@ func (rws *WriteStorage) ApplyConfig(conf *config.Config) error {
 
 		c, err := NewWriteClient(name, &ClientConfig{
 			URL:              rwConf.URL,
+			WriteProtoMsg:    rwConf.ProtobufMessage,
 			Timeout:          rwConf.RemoteTimeout,
 			HTTPClientConfig: rwConf.HTTPClientConfig,
 			SigV4Config:      rwConf.SigV4Config,
@@ -207,6 +214,7 @@ func (rws *WriteStorage) ApplyConfig(conf *config.Config) error {
 			rws.scraper,
 			rwConf.SendExemplars,
 			rwConf.SendNativeHistograms,
+			rwConf.ProtobufMessage,
 		)
 		// Keep track of which queues are new so we know which to start.
 		newHashes = append(newHashes, hash)

storage/remote/write_handler.go

@@ -17,19 +17,26 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
-
+	"github.com/gogo/protobuf/proto"
+	"github.com/golang/snappy"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 
+	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/model/exemplar"
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/timestamp"
 	"github.com/prometheus/prometheus/prompb"
+	writev2 "github.com/prometheus/prometheus/prompb/io/prometheus/write/v2"
 	"github.com/prometheus/prometheus/storage"
 	otlptranslator "github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite"
 )
@@ -38,148 +45,229 @@ type writeHandler struct {
 	logger     log.Logger
 	appendable storage.Appendable
 
-	samplesWithInvalidLabelsTotal prometheus.Counter
+	samplesWithInvalidLabelsTotal  prometheus.Counter
+	samplesAppendedWithoutMetadata prometheus.Counter
+
+	acceptedProtoMsgs map[config.RemoteWriteProtoMsg]struct{}
 }
 
 const maxAheadTime = 10 * time.Minute
 
-// NewWriteHandler creates a http.Handler that accepts remote write requests and
-// writes them to the provided appendable.
-func NewWriteHandler(logger log.Logger, reg prometheus.Registerer, appendable storage.Appendable) http.Handler {
+// NewWriteHandler creates a http.Handler that accepts remote write requests with
+// the given message in acceptedProtoMsgs and writes them to the provided appendable.
+//
+// NOTE(bwplotka): When accepting v2 proto and spec, partial writes are possible
+// as per https://prometheus.io/docs/specs/remote_write_spec_2_0/#partial-write.
+func NewWriteHandler(logger log.Logger, reg prometheus.Registerer, appendable storage.Appendable, acceptedProtoMsgs []config.RemoteWriteProtoMsg) http.Handler {
+	protoMsgs := map[config.RemoteWriteProtoMsg]struct{}{}
+	for _, acc := range acceptedProtoMsgs {
+		protoMsgs[acc] = struct{}{}
+	}
 	h := &writeHandler{
-		logger:     logger,
-		appendable: appendable,
-
-		samplesWithInvalidLabelsTotal: prometheus.NewCounter(prometheus.CounterOpts{
+		logger:            logger,
+		appendable:        appendable,
+		acceptedProtoMsgs: protoMsgs,
+		samplesWithInvalidLabelsTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
 			Namespace: "prometheus",
 			Subsystem: "api",
 			Name:      "remote_write_invalid_labels_samples_total",
-			Help:      "The total number of remote write samples which contains invalid labels.",
+			Help:      "The total number of received remote write samples and histogram samples which were rejected due to invalid labels.",
+		}),
+		samplesAppendedWithoutMetadata: promauto.With(reg).NewCounter(prometheus.CounterOpts{
+			Namespace: "prometheus",
+			Subsystem: "api",
+			Name:      "remote_write_without_metadata_appended_samples_total",
+			Help:      "The total number of received remote write samples (and histogram samples) which were ingested without corresponding metadata.",
 		}),
-	}
-	if reg != nil {
-		reg.MustRegister(h.samplesWithInvalidLabelsTotal)
 	}
 	return h
 }
 
+func (h *writeHandler) parseProtoMsg(contentType string) (config.RemoteWriteProtoMsg, error) {
+	contentType = strings.TrimSpace(contentType)
+
+	parts := strings.Split(contentType, ";")
+	if parts[0] != appProtoContentType {
+		return "", fmt.Errorf("expected %v as the first (media) part, got %v content-type", appProtoContentType, contentType)
+	}
+	// Parse potential https://www.rfc-editor.org/rfc/rfc9110#parameter
+	for _, p := range parts[1:] {
+		pair := strings.Split(p, "=")
+		if len(pair) != 2 {
+			return "", fmt.Errorf("as per https://www.rfc-editor.org/rfc/rfc9110#parameter expected parameters to be key-values, got %v in %v content-type", p, contentType)
+		}
+		if pair[0] == "proto" {
+			ret := config.RemoteWriteProtoMsg(pair[1])
+			if err := ret.Validate(); err != nil {
+				return "", fmt.Errorf("got %v content type; %w", contentType, err)
+			}
+			return ret, nil
+		}
+	}
+	// No "proto=" parameter, assuming v1.
+	return config.RemoteWriteProtoMsgV1, nil
+}
+
 func (h *writeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	req, err := DecodeWriteRequest(r.Body)
+	contentType := r.Header.Get("Content-Type")
+	if contentType == "" {
+		// Don't break yolo 1.0 clients if not needed. This is similar to what we did
+		// before 2.0: https://github.com/prometheus/prometheus/blob/d78253319daa62c8f28ed47e40bafcad2dd8b586/storage/remote/write_handler.go#L62
+		// We could give http.StatusUnsupportedMediaType, but let's assume 1.0 message by default.
+		contentType = appProtoContentType
+	}
+
+	msgType, err := h.parseProtoMsg(contentType)
+	if err != nil {
+		level.Error(h.logger).Log("msg", "Error decoding remote write request", "err", err)
+		http.Error(w, err.Error(), http.StatusUnsupportedMediaType)
+		return
+	}
+
+	if _, ok := h.acceptedProtoMsgs[msgType]; !ok {
+		err := fmt.Errorf("%v protobuf message is not accepted by this server; accepted %v", msgType, func() (ret []string) {
+			for k := range h.acceptedProtoMsgs {
+				ret = append(ret, string(k))
+			}
+			return ret
+		}())
+		level.Error(h.logger).Log("msg", "Error decoding remote write request", "err", err)
+		http.Error(w, err.Error(), http.StatusUnsupportedMediaType)
+	}
+
+	enc := r.Header.Get("Content-Encoding")
+	if enc == "" {
+		// Don't break yolo 1.0 clients if not needed. This is similar to what we did
+		// before 2.0: https://github.com/prometheus/prometheus/blob/d78253319daa62c8f28ed47e40bafcad2dd8b586/storage/remote/write_handler.go#L62
+		// We could give http.StatusUnsupportedMediaType, but let's assume snappy by default.
+	} else if enc != string(SnappyBlockCompression) {
+		err := fmt.Errorf("%v encoding (compression) is not accepted by this server; only %v is acceptable", enc, SnappyBlockCompression)
+		level.Error(h.logger).Log("msg", "Error decoding remote write request", "err", err)
+		http.Error(w, err.Error(), http.StatusUnsupportedMediaType)
+	}
+
+	// Read the request body.
+	body, err := io.ReadAll(r.Body)
 	if err != nil {
 		level.Error(h.logger).Log("msg", "Error decoding remote write request", "err", err.Error())
 		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
 
-	err = h.write(r.Context(), req)
-	switch {
-	case err == nil:
-	case errors.Is(err, storage.ErrOutOfOrderSample), errors.Is(err, storage.ErrOutOfBounds), errors.Is(err, storage.ErrDuplicateSampleForTimestamp), errors.Is(err, storage.ErrTooOldSample):
-		// Indicated an out of order sample is a bad request to prevent retries.
+	decompressed, err := snappy.Decode(nil, body)
+	if err != nil {
+		// TODO(bwplotka): Add more context to responded error?
+		level.Error(h.logger).Log("msg", "Error decompressing remote write request", "err", err.Error())
 		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
-	default:
-		level.Error(h.logger).Log("msg", "Error appending remote write", "err", err.Error())
-		http.Error(w, err.Error(), http.StatusInternalServerError)
+	}
+
+	// Now we have a decompressed buffer we can unmarshal it.
+
+	if msgType == config.RemoteWriteProtoMsgV1 {
+		// PRW 1.0 flow has different proto message and no partial write handling.
+		var req prompb.WriteRequest
+		if err := proto.Unmarshal(decompressed, &req); err != nil {
+			// TODO(bwplotka): Add more context to responded error?
+			level.Error(h.logger).Log("msg", "Error decoding v1 remote write request", "protobuf_message", msgType, "err", err.Error())
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		if err = h.write(r.Context(), &req); err != nil {
+			switch {
+			case errors.Is(err, storage.ErrOutOfOrderSample), errors.Is(err, storage.ErrOutOfBounds), errors.Is(err, storage.ErrDuplicateSampleForTimestamp), errors.Is(err, storage.ErrTooOldSample):
+				// Indicated an out-of-order sample is a bad request to prevent retries.
+				http.Error(w, err.Error(), http.StatusBadRequest)
+				return
+			default:
+				level.Error(h.logger).Log("msg", "Error while remote writing the v1 request", "err", err.Error())
+				http.Error(w, err.Error(), http.StatusInternalServerError)
+				return
+			}
+		}
+		w.WriteHeader(http.StatusNoContent)
 		return
 	}
 
-	w.WriteHeader(http.StatusNoContent)
-}
+	// Remote Write 2.x proto message handling.
+	var req writev2.Request
+	if err := proto.Unmarshal(decompressed, &req); err != nil {
+		// TODO(bwplotka): Add more context to responded error?
+		level.Error(h.logger).Log("msg", "Error decoding v2 remote write request", "protobuf_message", msgType, "err", err.Error())
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
 
-// checkAppendExemplarError modifies the AppendExemplar's returned error based on the error cause.
-func (h *writeHandler) checkAppendExemplarError(err error, e exemplar.Exemplar, outOfOrderErrs *int) error {
-	unwrappedErr := errors.Unwrap(err)
-	if unwrappedErr == nil {
-		unwrappedErr = err
-	}
-	switch {
-	case errors.Is(unwrappedErr, storage.ErrNotFound):
-		return storage.ErrNotFound
-	case errors.Is(unwrappedErr, storage.ErrOutOfOrderExemplar):
-		*outOfOrderErrs++
-		level.Debug(h.logger).Log("msg", "Out of order exemplar", "exemplar", fmt.Sprintf("%+v", e))
-		return nil
-	default:
-		return err
+	respStats, errHTTPCode, err := h.writeV2(r.Context(), &req)
+
+	// Set required X-Prometheus-Remote-Write-Written-* response headers, in all cases.
+	respStats.SetResponseHeaders(w.Header())
+
+	if err != nil {
+		if errHTTPCode/5 == 100 { // 5xx
+			level.Error(h.logger).Log("msg", "Error while remote writing the v2 request", "err", err.Error())
+		}
+		http.Error(w, err.Error(), errHTTPCode)
+		return
 	}
+	w.WriteHeader(http.StatusNoContent)
 }
 
 func (h *writeHandler) write(ctx context.Context, req *prompb.WriteRequest) (err error) {
 	outOfOrderExemplarErrs := 0
 	samplesWithInvalidLabels := 0
+	samplesAppended := 0
 
-	timeLimitApp := &timeLimitAppender{
+	app := &timeLimitAppender{
 		Appender: h.appendable.Appender(ctx),
 		maxTime:  timestamp.FromTime(time.Now().Add(maxAheadTime)),
 	}
 
 	defer func() {
 		if err != nil {
-			_ = timeLimitApp.Rollback()
+			_ = app.Rollback()
 			return
 		}
-		err = timeLimitApp.Commit()
+		err = app.Commit()
+		if err != nil {
+			h.samplesAppendedWithoutMetadata.Add(float64(samplesAppended))
+		}
 	}()
 
 	b := labels.NewScratchBuilder(0)
-	var exemplarErr error
-
 	for _, ts := range req.Timeseries {
-		labels := LabelProtosToLabels(&b, ts.Labels)
-		if !labels.IsValid() {
-			level.Warn(h.logger).Log("msg", "Invalid metric names or labels", "got", labels.String())
+		ls := ts.ToLabels(&b, nil)
+		if !ls.Has(labels.MetricName) || !ls.IsValid() {
+			level.Warn(h.logger).Log("msg", "Invalid metric names or labels", "got", ls.String())
 			samplesWithInvalidLabels++
+			// TODO(bwplotka): Even as per 1.0 spec, this should be a 400 error, while other samples are
+			// potentially written. Perhaps unify with fixed writeV2 implementation a bit.
 			continue
 		}
-		var ref storage.SeriesRef
-		for _, s := range ts.Samples {
-			ref, err = timeLimitApp.Append(ref, labels, s.Timestamp, s.Value)
-			if err != nil {
-				unwrappedErr := errors.Unwrap(err)
-				if unwrappedErr == nil {
-					unwrappedErr = err
-				}
-				if errors.Is(err, storage.ErrOutOfOrderSample) || errors.Is(unwrappedErr, storage.ErrOutOfBounds) || errors.Is(unwrappedErr, storage.ErrDuplicateSampleForTimestamp) {
-					level.Error(h.logger).Log("msg", "Out of order sample from remote write", "err", err.Error(), "series", labels.String(), "timestamp", s.Timestamp)
-				}
-				return err
-			}
+
+		if err := h.appendV1Samples(app, ts.Samples, ls); err != nil {
+			return err
 		}
+		samplesAppended += len(ts.Samples)
 
 		for _, ep := range ts.Exemplars {
-			e := exemplarProtoToExemplar(&b, ep)
-
-			_, exemplarErr = timeLimitApp.AppendExemplar(0, labels, e)
-			exemplarErr = h.checkAppendExemplarError(exemplarErr, e, &outOfOrderExemplarErrs)
-			if exemplarErr != nil {
-				// Since exemplar storage is still experimental, we don't fail the request on ingestion errors.
-				level.Debug(h.logger).Log("msg", "Error while adding exemplar in AddExemplar", "exemplar", fmt.Sprintf("%+v", e), "err", exemplarErr)
+			e := ep.ToExemplar(&b, nil)
+			if _, err := app.AppendExemplar(0, ls, e); err != nil {
+				switch {
+				case errors.Is(err, storage.ErrOutOfOrderExemplar):
+					outOfOrderExemplarErrs++
+					level.Debug(h.logger).Log("msg", "Out of order exemplar", "series", ls.String(), "exemplar", fmt.Sprintf("%+v", e))
+				default:
+					// Since exemplar storage is still experimental, we don't fail the request on ingestion errors
+					level.Debug(h.logger).Log("msg", "Error while adding exemplar in AppendExemplar", "series", ls.String(), "exemplar", fmt.Sprintf("%+v", e), "err", err)
+				}
 			}
 		}
 
-		for _, hp := range ts.Histograms {
-			if hp.IsFloatHistogram() {
-				fhs := FloatHistogramProtoToFloatHistogram(hp)
-				_, err = timeLimitApp.AppendHistogram(0, labels, hp.Timestamp, nil, fhs)
-			} else {
-				hs := HistogramProtoToHistogram(hp)
-				_, err = timeLimitApp.AppendHistogram(0, labels, hp.Timestamp, hs, nil)
-			}
-
-			if err != nil {
-				unwrappedErr := errors.Unwrap(err)
-				if unwrappedErr == nil {
-					unwrappedErr = err
-				}
-				// Although AppendHistogram does not currently return ErrDuplicateSampleForTimestamp there is
-				// a note indicating its inclusion in the future.
-				if errors.Is(unwrappedErr, storage.ErrOutOfOrderSample) || errors.Is(unwrappedErr, storage.ErrOutOfBounds) || errors.Is(unwrappedErr, storage.ErrDuplicateSampleForTimestamp) {
-					level.Error(h.logger).Log("msg", "Out of order histogram from remote write", "err", err.Error(), "series", labels.String(), "timestamp", hp.Timestamp)
-				}
-				return err
-			}
+		if err = h.appendV1Histograms(app, ts.Histograms, ls); err != nil {
+			return err
 		}
+		samplesAppended += len(ts.Histograms)
 	}
 
 	if outOfOrderExemplarErrs > 0 {
@@ -188,10 +276,218 @@ func (h *writeHandler) write(ctx context.Context, req *prompb.WriteRequest) (err
 	if samplesWithInvalidLabels > 0 {
 		h.samplesWithInvalidLabelsTotal.Add(float64(samplesWithInvalidLabels))
 	}
-
 	return nil
 }
 
+func (h *writeHandler) appendV1Samples(app storage.Appender, ss []prompb.Sample, labels labels.Labels) error {
+	var ref storage.SeriesRef
+	var err error
+	for _, s := range ss {
+		ref, err = app.Append(ref, labels, s.GetTimestamp(), s.GetValue())
+		if err != nil {
+			if errors.Is(err, storage.ErrOutOfOrderSample) ||
+				errors.Is(err, storage.ErrOutOfBounds) ||
+				errors.Is(err, storage.ErrDuplicateSampleForTimestamp) {
+				level.Error(h.logger).Log("msg", "Out of order sample from remote write", "err", err.Error(), "series", labels.String(), "timestamp", s.Timestamp)
+			}
+			return err
+		}
+	}
+	return nil
+}
+
+func (h *writeHandler) appendV1Histograms(app storage.Appender, hh []prompb.Histogram, labels labels.Labels) error {
+	var err error
+	for _, hp := range hh {
+		if hp.IsFloatHistogram() {
+			_, err = app.AppendHistogram(0, labels, hp.Timestamp, nil, hp.ToFloatHistogram())
+		} else {
+			_, err = app.AppendHistogram(0, labels, hp.Timestamp, hp.ToIntHistogram(), nil)
+		}
+		if err != nil {
+			// Although AppendHistogram does not currently return ErrDuplicateSampleForTimestamp there is
+			// a note indicating its inclusion in the future.
+			if errors.Is(err, storage.ErrOutOfOrderSample) ||
+				errors.Is(err, storage.ErrOutOfBounds) ||
+				errors.Is(err, storage.ErrDuplicateSampleForTimestamp) {
+				level.Error(h.logger).Log("msg", "Out of order histogram from remote write", "err", err.Error(), "series", labels.String(), "timestamp", hp.Timestamp)
+			}
+			return err
+		}
+	}
+	return nil
+}
+
+const (
+	prw20WrittenSamplesHeader   = "X-Prometheus-Remote-Write-Written-Samples"
+	rw20WrittenHistogramsHeader = "X-Prometheus-Remote-Write-Written-Histograms"
+	rw20WrittenExemplarsHeader  = "X-Prometheus-Remote-Write-Written-Exemplars"
+)
+
+type responseStats struct {
+	samples    int
+	histograms int
+	exemplars  int
+}
+
+func (s responseStats) SetResponseHeaders(h http.Header) {
+	h.Set(prw20WrittenSamplesHeader, strconv.Itoa(s.samples))
+	h.Set(rw20WrittenHistogramsHeader, strconv.Itoa(s.histograms))
+	h.Set(rw20WrittenExemplarsHeader, strconv.Itoa(s.exemplars))
+}
+
+// writeV2 is similar to write, but it works with v2 proto message,
+// allows partial 4xx writes and gathers statistics.
+//
+// writeV2 returns the statistics.
+// In error cases, writeV2, also returns statistics, but also the error that
+// should be propagated to the remote write sender and httpCode to use for status.
+//
+// NOTE(bwplotka): TSDB storage is NOT idempotent, so we don't allow "partial retry-able" errors.
+// Once we have 5xx type of error, we immediately stop and rollback all appends.
+func (h *writeHandler) writeV2(ctx context.Context, req *writev2.Request) (_ responseStats, errHTTPCode int, _ error) {
+	app := &timeLimitAppender{
+		Appender: h.appendable.Appender(ctx),
+		maxTime:  timestamp.FromTime(time.Now().Add(maxAheadTime)),
+	}
+
+	rs := responseStats{}
+	samplesWithoutMetadata, errHTTPCode, err := h.appendV2(app, req, &rs)
+	if err != nil {
+		if errHTTPCode/5 == 100 {
+			// On 5xx, we always rollback, because we expect
+			// sender to retry and TSDB is not idempotent.
+			if rerr := app.Rollback(); rerr != nil {
+				level.Error(h.logger).Log("msg", "writev2 rollback failed on retry-able error", "err", rerr)
+			}
+			return responseStats{}, errHTTPCode, err
+		}
+
+		// Non-retriable (e.g. bad request error case). Can be partially written.
+		commitErr := app.Commit()
+		if commitErr != nil {
+			// Bad requests does not matter as we have internal error (retryable).
+			return responseStats{}, http.StatusInternalServerError, commitErr
+		}
+		// Bad request error happened, but rest of data (if any) was written.
+		h.samplesAppendedWithoutMetadata.Add(float64(samplesWithoutMetadata))
+		return rs, errHTTPCode, err
+	}
+
+	// All good just commit.
+	if err := app.Commit(); err != nil {
+		return responseStats{}, http.StatusInternalServerError, err
+	}
+	h.samplesAppendedWithoutMetadata.Add(float64(samplesWithoutMetadata))
+	return rs, 0, nil
+}
+
+func (h *writeHandler) appendV2(app storage.Appender, req *writev2.Request, rs *responseStats) (samplesWithoutMetadata, errHTTPCode int, err error) {
+	var (
+		badRequestErrs                                   []error
+		outOfOrderExemplarErrs, samplesWithInvalidLabels int
+
+		b = labels.NewScratchBuilder(0)
+	)
+	for _, ts := range req.Timeseries {
+		ls := ts.ToLabels(&b, req.Symbols)
+		// Validate series labels early.
+		// NOTE(bwplotka): While spec allows UTF-8, Prometheus Receiver may impose
+		// specific limits and follow https://prometheus.io/docs/specs/remote_write_spec_2_0/#invalid-samples case.
+		if !ls.Has(labels.MetricName) || !ls.IsValid() {
+			badRequestErrs = append(badRequestErrs, fmt.Errorf("invalid metric name or labels, got %v", ls.String()))
+			samplesWithInvalidLabels += len(ts.Samples) + len(ts.Histograms)
+			continue
+		}
+
+		allSamplesSoFar := rs.samples + rs.histograms
+		var ref storage.SeriesRef
+
+		// Samples.
+		for _, s := range ts.Samples {
+			ref, err = app.Append(ref, ls, s.GetTimestamp(), s.GetValue())
+			if err == nil {
+				rs.samples++
+				continue
+			}
+			// Handle append error.
+			if errors.Is(err, storage.ErrOutOfOrderSample) ||
+				errors.Is(err, storage.ErrOutOfBounds) ||
+				errors.Is(err, storage.ErrDuplicateSampleForTimestamp) ||
+				errors.Is(err, storage.ErrTooOldSample) {
+				// TODO(bwplotka): Not too spammy log?
+				level.Error(h.logger).Log("msg", "Out of order sample from remote write", "err", err.Error(), "series", ls.String(), "timestamp", s.Timestamp)
+				badRequestErrs = append(badRequestErrs, fmt.Errorf("%w for series %v", err, ls.String()))
+				continue
+			}
+			return 0, http.StatusInternalServerError, err
+		}
+
+		// Native Histograms.
+		for _, hp := range ts.Histograms {
+			if hp.IsFloatHistogram() {
+				ref, err = app.AppendHistogram(ref, ls, hp.Timestamp, nil, hp.ToFloatHistogram())
+			} else {
+				ref, err = app.AppendHistogram(ref, ls, hp.Timestamp, hp.ToIntHistogram(), nil)
+			}
+			if err == nil {
+				rs.histograms++
+				continue
+			}
+			// Handle append error.
+			// Although AppendHistogram does not currently return ErrDuplicateSampleForTimestamp there is
+			// a note indicating its inclusion in the future.
+			if errors.Is(err, storage.ErrOutOfOrderSample) ||
+				errors.Is(err, storage.ErrOutOfBounds) ||
+				errors.Is(err, storage.ErrDuplicateSampleForTimestamp) {
+				// TODO(bwplotka): Not too spammy log?
+				level.Error(h.logger).Log("msg", "Out of order histogram from remote write", "err", err.Error(), "series", ls.String(), "timestamp", hp.Timestamp)
+				badRequestErrs = append(badRequestErrs, fmt.Errorf("%w for series %v", err, ls.String()))
+				continue
+			}
+			return 0, http.StatusInternalServerError, err
+		}
+
+		// Exemplars.
+		for _, ep := range ts.Exemplars {
+			e := ep.ToExemplar(&b, req.Symbols)
+			ref, err = app.AppendExemplar(ref, ls, e)
+			if err == nil {
+				rs.exemplars++
+				continue
+			}
+			// Handle append error.
+			// TODO(bwplotka): I left the logic as in v1, but we might want to make it consistent with samples and histograms.
+			// Since exemplar storage is still experimental, we don't fail in anyway, the request on ingestion errors.
+			if errors.Is(err, storage.ErrOutOfOrderExemplar) {
+				outOfOrderExemplarErrs++
+				level.Debug(h.logger).Log("msg", "Out of order exemplar", "err", err.Error(), "series", ls.String(), "exemplar", fmt.Sprintf("%+v", e))
+				continue
+			}
+			level.Debug(h.logger).Log("msg", "Error while adding exemplar in AppendExemplar", "series", ls.String(), "exemplar", fmt.Sprintf("%+v", e), "err", err)
+		}
+
+		m := ts.ToMetadata(req.Symbols)
+		if _, err = app.UpdateMetadata(ref, ls, m); err != nil {
+			level.Debug(h.logger).Log("msg", "error while updating metadata from remote write", "err", err)
+			// Metadata is attached to each series, so since Prometheus does not reject sample without metadata information,
+			// we don't report remote write error either. We increment metric instead.
+			samplesWithoutMetadata += (rs.samples + rs.histograms) - allSamplesSoFar
+		}
+	}
+
+	if outOfOrderExemplarErrs > 0 {
+		level.Warn(h.logger).Log("msg", "Error on ingesting out-of-order exemplars", "num_dropped", outOfOrderExemplarErrs)
+	}
+	h.samplesWithInvalidLabelsTotal.Add(float64(samplesWithInvalidLabels))
+
+	if len(badRequestErrs) == 0 {
+		return samplesWithoutMetadata, 0, nil
+	}
+	// TODO(bwplotka): Better concat formatting? Perhaps add size limit?
+	return samplesWithoutMetadata, http.StatusBadRequest, errors.Join(badRequestErrs...)
+}
+
 // NewOTLPWriteHandler creates a http.Handler that accepts OTLP write requests and
 // writes them to the provided appendable.
 func NewOTLPWriteHandler(logger log.Logger, appendable storage.Appendable) http.Handler {

storage/remote/write_handler_test.go

@@ -16,6 +16,7 @@ package remote
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"math"
@@ -27,28 +28,234 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/gogo/protobuf/proto"
 	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/require"
 
+	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/model/exemplar"
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/metadata"
 	"github.com/prometheus/prometheus/prompb"
+	writev2 "github.com/prometheus/prometheus/prompb/io/prometheus/write/v2"
 	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/tsdb"
 	"github.com/prometheus/prometheus/util/testutil"
 )
 
-func TestRemoteWriteHandler(t *testing.T) {
-	buf, _, _, err := buildWriteRequest(nil, writeRequestFixture.Timeseries, nil, nil, nil, nil)
+func TestRemoteWriteHandlerHeadersHandling_V1Message(t *testing.T) {
+	payload, _, _, err := buildWriteRequest(nil, writeRequestFixture.Timeseries, nil, nil, nil, nil, "snappy")
 	require.NoError(t, err)
 
-	req, err := http.NewRequest("", "", bytes.NewReader(buf))
+	for _, tc := range []struct {
+		name         string
+		reqHeaders   map[string]string
+		expectedCode int
+	}{
+		// Generally Prometheus 1.0 Receiver never checked for existence of the headers, so
+		// we keep things permissive.
+		{
+			name: "correct PRW 1.0 headers",
+			reqHeaders: map[string]string{
+				"Content-Type":           remoteWriteContentTypeHeaders[config.RemoteWriteProtoMsgV1],
+				"Content-Encoding":       string(SnappyBlockCompression),
+				RemoteWriteVersionHeader: RemoteWriteVersion20HeaderValue,
+			},
+			expectedCode: http.StatusNoContent,
+		},
+		{
+			name: "missing remote write version",
+			reqHeaders: map[string]string{
+				"Content-Type":     remoteWriteContentTypeHeaders[config.RemoteWriteProtoMsgV1],
+				"Content-Encoding": string(SnappyBlockCompression),
+			},
+			expectedCode: http.StatusNoContent,
+		},
+		{
+			name:         "no headers",
+			reqHeaders:   map[string]string{},
+			expectedCode: http.StatusNoContent,
+		},
+		{
+			name: "missing content-type",
+			reqHeaders: map[string]string{
+				"Content-Encoding":       string(SnappyBlockCompression),
+				RemoteWriteVersionHeader: RemoteWriteVersion20HeaderValue,
+			},
+			expectedCode: http.StatusNoContent,
+		},
+		{
+			name: "missing content-encoding",
+			reqHeaders: map[string]string{
+				"Content-Type":           remoteWriteContentTypeHeaders[config.RemoteWriteProtoMsgV1],
+				RemoteWriteVersionHeader: RemoteWriteVersion20HeaderValue,
+			},
+			expectedCode: http.StatusNoContent,
+		},
+		{
+			name: "wrong content-type",
+			reqHeaders: map[string]string{
+				"Content-Type":           "yolo",
+				"Content-Encoding":       string(SnappyBlockCompression),
+				RemoteWriteVersionHeader: RemoteWriteVersion20HeaderValue,
+			},
+			expectedCode: http.StatusUnsupportedMediaType,
+		},
+		{
+			name: "wrong content-type2",
+			reqHeaders: map[string]string{
+				"Content-Type":           appProtoContentType + ";proto=yolo",
+				"Content-Encoding":       string(SnappyBlockCompression),
+				RemoteWriteVersionHeader: RemoteWriteVersion20HeaderValue,
+			},
+			expectedCode: http.StatusUnsupportedMediaType,
+		},
+		{
+			name: "not supported content-encoding",
+			reqHeaders: map[string]string{
+				"Content-Type":           remoteWriteContentTypeHeaders[config.RemoteWriteProtoMsgV1],
+				"Content-Encoding":       "zstd",
+				RemoteWriteVersionHeader: RemoteWriteVersion20HeaderValue,
+			},
+			expectedCode: http.StatusUnsupportedMediaType,
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			req, err := http.NewRequest("", "", bytes.NewReader(payload))
+			require.NoError(t, err)
+			for k, v := range tc.reqHeaders {
+				req.Header.Set(k, v)
+			}
+
+			appendable := &mockAppendable{}
+			handler := NewWriteHandler(log.NewNopLogger(), nil, appendable, []config.RemoteWriteProtoMsg{config.RemoteWriteProtoMsgV1})
+
+			recorder := httptest.NewRecorder()
+			handler.ServeHTTP(recorder, req)
+
+			resp := recorder.Result()
+			out, err := io.ReadAll(resp.Body)
+			require.NoError(t, err)
+			_ = resp.Body.Close()
+			require.Equal(t, tc.expectedCode, resp.StatusCode, string(out))
+		})
+	}
+}
+
+func TestRemoteWriteHandlerHeadersHandling_V2Message(t *testing.T) {
+	payload, _, _, err := buildV2WriteRequest(log.NewNopLogger(), writeV2RequestFixture.Timeseries, writeV2RequestFixture.Symbols, nil, nil, nil, "snappy")
 	require.NoError(t, err)
 
+	for _, tc := range []struct {
+		name         string
+		reqHeaders   map[string]string
+		expectedCode int
+	}{
+		{
+			name: "correct PRW 2.0 headers",
+			reqHeaders: map[string]string{
+				"Content-Type":           remoteWriteContentTypeHeaders[config.RemoteWriteProtoMsgV2],
+				"Content-Encoding":       string(SnappyBlockCompression),
+				RemoteWriteVersionHeader: RemoteWriteVersion20HeaderValue,
+			},
+			expectedCode: http.StatusNoContent,
+		},
+		{
+			name: "missing remote write version",
+			reqHeaders: map[string]string{
+				"Content-Type":     remoteWriteContentTypeHeaders[config.RemoteWriteProtoMsgV2],
+				"Content-Encoding": string(SnappyBlockCompression),
+			},
+			expectedCode: http.StatusNoContent, // We don't check for now.
+		},
+		{
+			name:         "no headers",
+			reqHeaders:   map[string]string{},
+			expectedCode: http.StatusUnsupportedMediaType,
+		},
+		{
+			name: "missing content-type",
+			reqHeaders: map[string]string{
+				"Content-Encoding":       string(SnappyBlockCompression),
+				RemoteWriteVersionHeader: RemoteWriteVersion20HeaderValue,
+			},
+			// This only gives 415, because we explicitly only support 2.0. If we supported both
+			// (default) it would be empty message parsed and ok response.
+			// This is perhaps better, than 415 for previously working 1.0 flow with
+			// no content-type.
+			expectedCode: http.StatusUnsupportedMediaType,
+		},
+		{
+			name: "missing content-encoding",
+			reqHeaders: map[string]string{
+				"Content-Type":           remoteWriteContentTypeHeaders[config.RemoteWriteProtoMsgV2],
+				RemoteWriteVersionHeader: RemoteWriteVersion20HeaderValue,
+			},
+			expectedCode: http.StatusNoContent, // Similar to 1.0 impl, we default to Snappy, so it works.
+		},
+		{
+			name: "wrong content-type",
+			reqHeaders: map[string]string{
+				"Content-Type":           "yolo",
+				"Content-Encoding":       string(SnappyBlockCompression),
+				RemoteWriteVersionHeader: RemoteWriteVersion20HeaderValue,
+			},
+			expectedCode: http.StatusUnsupportedMediaType,
+		},
+		{
+			name: "wrong content-type2",
+			reqHeaders: map[string]string{
+				"Content-Type":           appProtoContentType + ";proto=yolo",
+				"Content-Encoding":       string(SnappyBlockCompression),
+				RemoteWriteVersionHeader: RemoteWriteVersion20HeaderValue,
+			},
+			expectedCode: http.StatusUnsupportedMediaType,
+		},
+		{
+			name: "not supported content-encoding",
+			reqHeaders: map[string]string{
+				"Content-Type":           remoteWriteContentTypeHeaders[config.RemoteWriteProtoMsgV2],
+				"Content-Encoding":       "zstd",
+				RemoteWriteVersionHeader: RemoteWriteVersion20HeaderValue,
+			},
+			expectedCode: http.StatusUnsupportedMediaType,
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			req, err := http.NewRequest("", "", bytes.NewReader(payload))
+			require.NoError(t, err)
+			for k, v := range tc.reqHeaders {
+				req.Header.Set(k, v)
+			}
+
+			appendable := &mockAppendable{}
+			handler := NewWriteHandler(log.NewNopLogger(), nil, appendable, []config.RemoteWriteProtoMsg{config.RemoteWriteProtoMsgV2})
+
+			recorder := httptest.NewRecorder()
+			handler.ServeHTTP(recorder, req)
+
+			resp := recorder.Result()
+			out, err := io.ReadAll(resp.Body)
+			require.NoError(t, err)
+			_ = resp.Body.Close()
+			require.Equal(t, tc.expectedCode, resp.StatusCode, string(out))
+		})
+	}
+}
+
+func TestRemoteWriteHandler_V1Message(t *testing.T) {
+	payload, _, _, err := buildWriteRequest(nil, writeRequestFixture.Timeseries, nil, nil, nil, nil, "snappy")
+	require.NoError(t, err)
+
+	req, err := http.NewRequest("", "", bytes.NewReader(payload))
+	require.NoError(t, err)
+
+	// NOTE: Strictly speaking, even for 1.0 we require headers, but we never verified those
+	// in Prometheus, so keeping like this to not break existing 1.0 clients.
+
 	appendable := &mockAppendable{}
-	handler := NewWriteHandler(nil, nil, appendable)
+	handler := NewWriteHandler(log.NewNopLogger(), nil, appendable, []config.RemoteWriteProtoMsg{config.RemoteWriteProtoMsgV1})
 
 	recorder := httptest.NewRecorder()
 	handler.ServeHTTP(recorder, req)
@@ -61,24 +268,22 @@ func TestRemoteWriteHandler(t *testing.T) {
 	j := 0
 	k := 0
 	for _, ts := range writeRequestFixture.Timeseries {
-		labels := LabelProtosToLabels(&b, ts.Labels)
+		labels := ts.ToLabels(&b, nil)
 		for _, s := range ts.Samples {
 			requireEqual(t, mockSample{labels, s.Timestamp, s.Value}, appendable.samples[i])
 			i++
 		}
-
 		for _, e := range ts.Exemplars {
-			exemplarLabels := LabelProtosToLabels(&b, e.Labels)
+			exemplarLabels := e.ToExemplar(&b, nil).Labels
 			requireEqual(t, mockExemplar{labels, exemplarLabels, e.Timestamp, e.Value}, appendable.exemplars[j])
 			j++
 		}
-
 		for _, hp := range ts.Histograms {
 			if hp.IsFloatHistogram() {
-				fh := FloatHistogramProtoToFloatHistogram(hp)
+				fh := hp.ToFloatHistogram()
 				requireEqual(t, mockHistogram{labels, hp.Timestamp, nil, fh}, appendable.histograms[k])
 			} else {
-				h := HistogramProtoToHistogram(hp)
+				h := hp.ToIntHistogram()
 				requireEqual(t, mockHistogram{labels, hp.Timestamp, h, nil}, appendable.histograms[k])
 			}
 
@@ -87,8 +292,226 @@ func TestRemoteWriteHandler(t *testing.T) {
 	}
 }
 
-func TestOutOfOrderSample(t *testing.T) {
-	tests := []struct {
+func expectHeaderValue(t testing.TB, expected int, got string) {
+	t.Helper()
+
+	require.NotEmpty(t, got)
+	i, err := strconv.Atoi(got)
+	require.NoError(t, err)
+	require.Equal(t, expected, i)
+}
+
+func TestRemoteWriteHandler_V2Message(t *testing.T) {
+	// V2 supports partial writes for non-retriable errors, so test them.
+	for _, tc := range []struct {
+		desc             string
+		input            []writev2.TimeSeries
+		expectedCode     int
+		expectedRespBody string
+
+		commitErr          error
+		appendSampleErr    error
+		appendHistogramErr error
+		appendExemplarErr  error
+		updateMetadataErr  error
+	}{
+		{
+			desc:         "All timeseries accepted",
+			input:        writeV2RequestFixture.Timeseries,
+			expectedCode: http.StatusNoContent,
+		},
+		{
+			desc: "Partial write; first series with invalid labels (no metric name)",
+			input: append(
+				// Series with test_metric1="test_metric1" labels.
+				[]writev2.TimeSeries{{LabelsRefs: []uint32{2, 2}, Samples: []writev2.Sample{{Value: 1, Timestamp: 1}}}},
+				writeV2RequestFixture.Timeseries...),
+			expectedCode:     http.StatusBadRequest,
+			expectedRespBody: "invalid metric name or labels, got {test_metric1=\"test_metric1\"}\n",
+		},
+		{
+			desc: "Partial write; first series with invalid labels (empty metric name)",
+			input: append(
+				// Series with __name__="" labels.
+				[]writev2.TimeSeries{{LabelsRefs: []uint32{1, 0}, Samples: []writev2.Sample{{Value: 1, Timestamp: 1}}}},
+				writeV2RequestFixture.Timeseries...),
+			expectedCode:     http.StatusBadRequest,
+			expectedRespBody: "invalid metric name or labels, got {__name__=\"\"}\n",
+		},
+		{
+			desc: "Partial write; first series with one OOO sample",
+			input: func() []writev2.TimeSeries {
+				f := proto.Clone(writeV2RequestFixture).(*writev2.Request)
+				f.Timeseries[0].Samples = append(f.Timeseries[0].Samples, writev2.Sample{Value: 2, Timestamp: 0})
+				return f.Timeseries
+			}(),
+			expectedCode:     http.StatusBadRequest,
+			expectedRespBody: "out of order sample for series {__name__=\"test_metric1\", b=\"c\", baz=\"qux\", d=\"e\", foo=\"bar\"}\n",
+		},
+		{
+			desc: "Partial write; first series with one dup sample",
+			input: func() []writev2.TimeSeries {
+				f := proto.Clone(writeV2RequestFixture).(*writev2.Request)
+				f.Timeseries[0].Samples = append(f.Timeseries[0].Samples, f.Timeseries[0].Samples[0])
+				return f.Timeseries
+			}(),
+			expectedCode:     http.StatusBadRequest,
+			expectedRespBody: "duplicate sample for timestamp for series {__name__=\"test_metric1\", b=\"c\", baz=\"qux\", d=\"e\", foo=\"bar\"}\n",
+		},
+		{
+			desc: "Partial write; first series with one OOO histogram sample",
+			input: func() []writev2.TimeSeries {
+				f := proto.Clone(writeV2RequestFixture).(*writev2.Request)
+				f.Timeseries[0].Histograms = append(f.Timeseries[0].Histograms, writev2.FromFloatHistogram(1, testHistogram.ToFloat(nil)))
+				return f.Timeseries
+			}(),
+			expectedCode:     http.StatusBadRequest,
+			expectedRespBody: "out of order sample for series {__name__=\"test_metric1\", b=\"c\", baz=\"qux\", d=\"e\", foo=\"bar\"}\n",
+		},
+		{
+			desc: "Partial write; first series with one dup histogram sample",
+			input: func() []writev2.TimeSeries {
+				f := proto.Clone(writeV2RequestFixture).(*writev2.Request)
+				f.Timeseries[0].Histograms = append(f.Timeseries[0].Histograms, f.Timeseries[0].Histograms[1])
+				return f.Timeseries
+			}(),
+			expectedCode:     http.StatusBadRequest,
+			expectedRespBody: "duplicate sample for timestamp for series {__name__=\"test_metric1\", b=\"c\", baz=\"qux\", d=\"e\", foo=\"bar\"}\n",
+		},
+		// Non retriable errors from various parts.
+		{
+			desc:            "Internal sample append error; rollback triggered",
+			input:           writeV2RequestFixture.Timeseries,
+			appendSampleErr: errors.New("some sample internal append error"),
+
+			expectedCode:     http.StatusInternalServerError,
+			expectedRespBody: "some sample internal append error\n",
+		},
+		{
+			desc:               "Internal histogram sample append error; rollback triggered",
+			input:              writeV2RequestFixture.Timeseries,
+			appendHistogramErr: errors.New("some histogram sample internal append error"),
+
+			expectedCode:     http.StatusInternalServerError,
+			expectedRespBody: "some histogram sample internal append error\n",
+		},
+		{
+			desc:              "Partial write; skipped exemplar; exemplar storage errs are noop",
+			input:             writeV2RequestFixture.Timeseries,
+			appendExemplarErr: errors.New("some exemplar append error"),
+
+			expectedCode: http.StatusNoContent,
+		},
+		{
+			desc:              "Partial write; skipped metadata; metadata storage errs are noop",
+			input:             writeV2RequestFixture.Timeseries,
+			updateMetadataErr: errors.New("some metadata update error"),
+
+			expectedCode: http.StatusNoContent,
+		},
+		{
+			desc:      "Internal commit error; rollback triggered",
+			input:     writeV2RequestFixture.Timeseries,
+			commitErr: errors.New("storage error"),
+
+			expectedCode:     http.StatusInternalServerError,
+			expectedRespBody: "storage error\n",
+		},
+	} {
+		t.Run(tc.desc, func(t *testing.T) {
+			payload, _, _, err := buildV2WriteRequest(log.NewNopLogger(), tc.input, writeV2RequestFixture.Symbols, nil, nil, nil, "snappy")
+			require.NoError(t, err)
+
+			req, err := http.NewRequest("", "", bytes.NewReader(payload))
+			require.NoError(t, err)
+
+			req.Header.Set("Content-Type", remoteWriteContentTypeHeaders[config.RemoteWriteProtoMsgV2])
+			req.Header.Set("Content-Encoding", string(SnappyBlockCompression))
+			req.Header.Set(RemoteWriteVersionHeader, RemoteWriteVersion20HeaderValue)
+
+			appendable := &mockAppendable{
+				commitErr:          tc.commitErr,
+				appendSampleErr:    tc.appendSampleErr,
+				appendHistogramErr: tc.appendHistogramErr,
+				appendExemplarErr:  tc.appendExemplarErr,
+				updateMetadataErr:  tc.updateMetadataErr,
+			}
+			handler := NewWriteHandler(log.NewNopLogger(), nil, appendable, []config.RemoteWriteProtoMsg{config.RemoteWriteProtoMsgV2})
+
+			recorder := httptest.NewRecorder()
+			handler.ServeHTTP(recorder, req)
+
+			resp := recorder.Result()
+			require.Equal(t, tc.expectedCode, resp.StatusCode)
+			respBody, err := io.ReadAll(resp.Body)
+			require.NoError(t, err)
+			require.Equal(t, tc.expectedRespBody, string(respBody))
+
+			if tc.expectedCode == http.StatusInternalServerError {
+				// We don't expect writes for partial writes with retry-able code.
+				expectHeaderValue(t, 0, resp.Header.Get("X-Prometheus-Remote-Write-Written-Samples"))
+				expectHeaderValue(t, 0, resp.Header.Get("X-Prometheus-Remote-Write-Written-Histograms"))
+				expectHeaderValue(t, 0, resp.Header.Get("X-Prometheus-Remote-Write-Written-Exemplars"))
+
+				require.Empty(t, len(appendable.samples))
+				require.Empty(t, len(appendable.histograms))
+				require.Empty(t, len(appendable.exemplars))
+				require.Empty(t, len(appendable.metadata))
+				return
+			}
+
+			// Double check mandatory 2.0 stats.
+			// writeV2RequestFixture has 2 series with 1 sample, 2 histograms, 1 exemplar each.
+			expectHeaderValue(t, 2, resp.Header.Get("X-Prometheus-Remote-Write-Written-Samples"))
+			expectHeaderValue(t, 4, resp.Header.Get("X-Prometheus-Remote-Write-Written-Histograms"))
+			if tc.appendExemplarErr != nil {
+				expectHeaderValue(t, 0, resp.Header.Get("X-Prometheus-Remote-Write-Written-Exemplars"))
+			} else {
+				expectHeaderValue(t, 2, resp.Header.Get("X-Prometheus-Remote-Write-Written-Exemplars"))
+			}
+
+			// Double check what was actually appended.
+			var (
+				b          = labels.NewScratchBuilder(0)
+				i, j, k, m int
+			)
+			for _, ts := range writeV2RequestFixture.Timeseries {
+				ls := ts.ToLabels(&b, writeV2RequestFixture.Symbols)
+
+				for _, s := range ts.Samples {
+					requireEqual(t, mockSample{ls, s.Timestamp, s.Value}, appendable.samples[i])
+					i++
+				}
+				for _, hp := range ts.Histograms {
+					if hp.IsFloatHistogram() {
+						fh := hp.ToFloatHistogram()
+						requireEqual(t, mockHistogram{ls, hp.Timestamp, nil, fh}, appendable.histograms[k])
+					} else {
+						h := hp.ToIntHistogram()
+						requireEqual(t, mockHistogram{ls, hp.Timestamp, h, nil}, appendable.histograms[k])
+					}
+					k++
+				}
+				if tc.appendExemplarErr == nil {
+					for _, e := range ts.Exemplars {
+						exemplarLabels := e.ToExemplar(&b, writeV2RequestFixture.Symbols).Labels
+						requireEqual(t, mockExemplar{ls, exemplarLabels, e.Timestamp, e.Value}, appendable.exemplars[j])
+						j++
+					}
+				}
+				if tc.updateMetadataErr == nil {
+					expectedMeta := ts.ToMetadata(writeV2RequestFixture.Symbols)
+					requireEqual(t, mockMetadata{ls, expectedMeta}, appendable.metadata[m])
+					m++
+				}
+			}
+		})
+	}
+}
+
+// NOTE: V2 Message is tested in TestRemoteWriteHandler_V2Message.
+func TestOutOfOrderSample_V1Message(t *testing.T) {
+	for _, tc := range []struct {
 		Name      string
 		Timestamp int64
 	}{
@@ -100,23 +523,19 @@ func TestOutOfOrderSample(t *testing.T) {
 			Name:      "future",
 			Timestamp: math.MaxInt64,
 		},
-	}
-
-	for _, tc := range tests {
+	} {
 		t.Run(tc.Name, func(t *testing.T) {
-			buf, _, _, err := buildWriteRequest(nil, []prompb.TimeSeries{{
+			payload, _, _, err := buildWriteRequest(nil, []prompb.TimeSeries{{
 				Labels:  []prompb.Label{{Name: "__name__", Value: "test_metric"}},
 				Samples: []prompb.Sample{{Value: 1, Timestamp: tc.Timestamp}},
-			}}, nil, nil, nil, nil)
+			}}, nil, nil, nil, nil, "snappy")
 			require.NoError(t, err)
 
-			req, err := http.NewRequest("", "", bytes.NewReader(buf))
+			req, err := http.NewRequest("", "", bytes.NewReader(payload))
 			require.NoError(t, err)
 
-			appendable := &mockAppendable{
-				latestSample: 100,
-			}
-			handler := NewWriteHandler(log.NewNopLogger(), nil, appendable)
+			appendable := &mockAppendable{latestSample: map[uint64]int64{labels.FromStrings("__name__", "test_metric").Hash(): 100}}
+			handler := NewWriteHandler(log.NewNopLogger(), nil, appendable, []config.RemoteWriteProtoMsg{config.RemoteWriteProtoMsgV1})
 
 			recorder := httptest.NewRecorder()
 			handler.ServeHTTP(recorder, req)
@@ -128,9 +547,10 @@ func TestOutOfOrderSample(t *testing.T) {
 }
 
 // This test case currently aims to verify that the WriteHandler endpoint
-// don't fail on ingestion errors since the exemplar storage is
+// don't fail on exemplar ingestion errors since the exemplar storage is
 // still experimental.
-func TestOutOfOrderExemplar(t *testing.T) {
+// NOTE: V2 Message is tested in TestRemoteWriteHandler_V2Message.
+func TestOutOfOrderExemplar_V1Message(t *testing.T) {
 	tests := []struct {
 		Name      string
 		Timestamp int64
@@ -147,19 +567,17 @@ func TestOutOfOrderExemplar(t *testing.T) {
 
 	for _, tc := range tests {
 		t.Run(tc.Name, func(t *testing.T) {
-			buf, _, _, err := buildWriteRequest(nil, []prompb.TimeSeries{{
+			payload, _, _, err := buildWriteRequest(nil, []prompb.TimeSeries{{
 				Labels:    []prompb.Label{{Name: "__name__", Value: "test_metric"}},
 				Exemplars: []prompb.Exemplar{{Labels: []prompb.Label{{Name: "foo", Value: "bar"}}, Value: 1, Timestamp: tc.Timestamp}},
-			}}, nil, nil, nil, nil)
+			}}, nil, nil, nil, nil, "snappy")
 			require.NoError(t, err)
 
-			req, err := http.NewRequest("", "", bytes.NewReader(buf))
+			req, err := http.NewRequest("", "", bytes.NewReader(payload))
 			require.NoError(t, err)
 
-			appendable := &mockAppendable{
-				latestExemplar: 100,
-			}
-			handler := NewWriteHandler(log.NewNopLogger(), nil, appendable)
+			appendable := &mockAppendable{latestSample: map[uint64]int64{labels.FromStrings("__name__", "test_metric").Hash(): 100}}
+			handler := NewWriteHandler(log.NewNopLogger(), nil, appendable, []config.RemoteWriteProtoMsg{config.RemoteWriteProtoMsgV1})
 
 			recorder := httptest.NewRecorder()
 			handler.ServeHTTP(recorder, req)
@@ -171,8 +589,9 @@ func TestOutOfOrderExemplar(t *testing.T) {
 	}
 }
 
-func TestOutOfOrderHistogram(t *testing.T) {
-	tests := []struct {
+// NOTE: V2 Message is tested in TestRemoteWriteHandler_V2Message.
+func TestOutOfOrderHistogram_V1Message(t *testing.T) {
+	for _, tc := range []struct {
 		Name      string
 		Timestamp int64
 	}{
@@ -184,23 +603,19 @@ func TestOutOfOrderHistogram(t *testing.T) {
 			Name:      "future",
 			Timestamp: math.MaxInt64,
 		},
-	}
-
-	for _, tc := range tests {
+	} {
 		t.Run(tc.Name, func(t *testing.T) {
-			buf, _, _, err := buildWriteRequest(nil, []prompb.TimeSeries{{
+			payload, _, _, err := buildWriteRequest(nil, []prompb.TimeSeries{{
 				Labels:     []prompb.Label{{Name: "__name__", Value: "test_metric"}},
-				Histograms: []prompb.Histogram{HistogramToHistogramProto(tc.Timestamp, &testHistogram), FloatHistogramToHistogramProto(1, testHistogram.ToFloat(nil))},
-			}}, nil, nil, nil, nil)
+				Histograms: []prompb.Histogram{prompb.FromIntHistogram(tc.Timestamp, &testHistogram), prompb.FromFloatHistogram(1, testHistogram.ToFloat(nil))},
+			}}, nil, nil, nil, nil, "snappy")
 			require.NoError(t, err)
 
-			req, err := http.NewRequest("", "", bytes.NewReader(buf))
+			req, err := http.NewRequest("", "", bytes.NewReader(payload))
 			require.NoError(t, err)
 
-			appendable := &mockAppendable{
-				latestHistogram: 100,
-			}
-			handler := NewWriteHandler(log.NewNopLogger(), nil, appendable)
+			appendable := &mockAppendable{latestSample: map[uint64]int64{labels.FromStrings("__name__", "test_metric").Hash(): 100}}
+			handler := NewWriteHandler(log.NewNopLogger(), nil, appendable, []config.RemoteWriteProtoMsg{config.RemoteWriteProtoMsgV1})
 
 			recorder := httptest.NewRecorder()
 			handler.ServeHTTP(recorder, req)
@@ -211,9 +626,9 @@ func TestOutOfOrderHistogram(t *testing.T) {
 	}
 }
 
-func BenchmarkRemoteWritehandler(b *testing.B) {
+func BenchmarkRemoteWriteHandler(b *testing.B) {
 	const labelValue = "abcdefg'hijlmn234!@#$%^&*()_+~`\"{}[],./<>?hello0123hiOlá你好Dzieńdobry9Zd8ra765v4stvuyte"
-	reqs := []*http.Request{}
+	var reqs []*http.Request
 	for i := 0; i < b.N; i++ {
 		num := strings.Repeat(strconv.Itoa(i), 16)
 		buf, _, _, err := buildWriteRequest(nil, []prompb.TimeSeries{{
@@ -221,8 +636,8 @@ func BenchmarkRemoteWritehandler(b *testing.B) {
 				{Name: "__name__", Value: "test_metric"},
 				{Name: "test_label_name_" + num, Value: labelValue + num},
 			},
-			Histograms: []prompb.Histogram{HistogramToHistogramProto(0, &testHistogram)},
-		}}, nil, nil, nil, nil)
+			Histograms: []prompb.Histogram{prompb.FromIntHistogram(0, &testHistogram)},
+		}}, nil, nil, nil, nil, "snappy")
 		require.NoError(b, err)
 		req, err := http.NewRequest("", "", bytes.NewReader(buf))
 		require.NoError(b, err)
@@ -230,7 +645,8 @@ func BenchmarkRemoteWritehandler(b *testing.B) {
 	}
 
 	appendable := &mockAppendable{}
-	handler := NewWriteHandler(log.NewNopLogger(), nil, appendable)
+	// TODO: test with other proto format(s)
+	handler := NewWriteHandler(log.NewNopLogger(), nil, appendable, []config.RemoteWriteProtoMsg{config.RemoteWriteProtoMsgV1})
 	recorder := httptest.NewRecorder()
 
 	b.ResetTimer()
@@ -239,17 +655,39 @@ func BenchmarkRemoteWritehandler(b *testing.B) {
 	}
 }
 
-func TestCommitErr(t *testing.T) {
-	buf, _, _, err := buildWriteRequest(nil, writeRequestFixture.Timeseries, nil, nil, nil, nil)
+func TestCommitErr_V1Message(t *testing.T) {
+	payload, _, _, err := buildWriteRequest(nil, writeRequestFixture.Timeseries, nil, nil, nil, nil, "snappy")
 	require.NoError(t, err)
 
-	req, err := http.NewRequest("", "", bytes.NewReader(buf))
+	req, err := http.NewRequest("", "", bytes.NewReader(payload))
 	require.NoError(t, err)
 
-	appendable := &mockAppendable{
-		commitErr: fmt.Errorf("commit error"),
-	}
-	handler := NewWriteHandler(log.NewNopLogger(), nil, appendable)
+	appendable := &mockAppendable{commitErr: fmt.Errorf("commit error")}
+	handler := NewWriteHandler(log.NewNopLogger(), nil, appendable, []config.RemoteWriteProtoMsg{config.RemoteWriteProtoMsgV1})
+
+	recorder := httptest.NewRecorder()
+	handler.ServeHTTP(recorder, req)
+
+	resp := recorder.Result()
+	body, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusInternalServerError, resp.StatusCode)
+	require.Equal(t, "commit error\n", string(body))
+}
+
+func TestCommitErr_V2Message(t *testing.T) {
+	payload, _, _, err := buildV2WriteRequest(log.NewNopLogger(), writeV2RequestFixture.Timeseries, writeV2RequestFixture.Symbols, nil, nil, nil, "snappy")
+	require.NoError(t, err)
+
+	req, err := http.NewRequest("", "", bytes.NewReader(payload))
+	require.NoError(t, err)
+
+	req.Header.Set("Content-Type", remoteWriteContentTypeHeaders[config.RemoteWriteProtoMsgV2])
+	req.Header.Set("Content-Encoding", string(SnappyBlockCompression))
+	req.Header.Set(RemoteWriteVersionHeader, RemoteWriteVersion20HeaderValue)
+
+	appendable := &mockAppendable{commitErr: fmt.Errorf("commit error")}
+	handler := NewWriteHandler(log.NewNopLogger(), nil, appendable, []config.RemoteWriteProtoMsg{config.RemoteWriteProtoMsgV2})
 
 	recorder := httptest.NewRecorder()
 	handler.ServeHTTP(recorder, req)
@@ -275,10 +713,10 @@ func BenchmarkRemoteWriteOOOSamples(b *testing.B) {
 	b.Cleanup(func() {
 		require.NoError(b, db.Close())
 	})
+	// TODO: test with other proto format(s)
+	handler := NewWriteHandler(log.NewNopLogger(), nil, db.Head(), []config.RemoteWriteProtoMsg{config.RemoteWriteProtoMsgV1})
 
-	handler := NewWriteHandler(log.NewNopLogger(), nil, db.Head())
-
-	buf, _, _, err := buildWriteRequest(nil, genSeriesWithSample(1000, 200*time.Minute.Milliseconds()), nil, nil, nil, nil)
+	buf, _, _, err := buildWriteRequest(nil, genSeriesWithSample(1000, 200*time.Minute.Milliseconds()), nil, nil, nil, nil, "snappy")
 	require.NoError(b, err)
 
 	req, err := http.NewRequest("", "", bytes.NewReader(buf))
@@ -291,7 +729,7 @@ func BenchmarkRemoteWriteOOOSamples(b *testing.B) {
 
 	var bufRequests [][]byte
 	for i := 0; i < 100; i++ {
-		buf, _, _, err = buildWriteRequest(nil, genSeriesWithSample(1000, int64(80+i)*time.Minute.Milliseconds()), nil, nil, nil, nil)
+		buf, _, _, err = buildWriteRequest(nil, genSeriesWithSample(1000, int64(80+i)*time.Minute.Milliseconds()), nil, nil, nil, nil, "snappy")
 		require.NoError(b, err)
 		bufRequests = append(bufRequests, buf)
 	}
@@ -322,13 +760,20 @@ func genSeriesWithSample(numSeries int, ts int64) []prompb.TimeSeries {
 }
 
 type mockAppendable struct {
-	latestSample    int64
+	latestSample    map[uint64]int64
 	samples         []mockSample
-	latestExemplar  int64
+	latestExemplar  map[uint64]int64
 	exemplars       []mockExemplar
-	latestHistogram int64
+	latestHistogram map[uint64]int64
 	histograms      []mockHistogram
-	commitErr       error
+	metadata        []mockMetadata
+
+	// optional errors to inject.
+	commitErr          error
+	appendSampleErr    error
+	appendHistogramErr error
+	appendExemplarErr  error
+	updateMetadataErr  error
 }
 
 type mockSample struct {
@@ -351,62 +796,114 @@ type mockHistogram struct {
 	fh *histogram.FloatHistogram
 }
 
+type mockMetadata struct {
+	l labels.Labels
+	m metadata.Metadata
+}
+
 // Wrapper to instruct go-cmp package to compare a list of structs with unexported fields.
 func requireEqual(t *testing.T, expected, actual interface{}, msgAndArgs ...interface{}) {
+	t.Helper()
+
 	testutil.RequireEqualWithOptions(t, expected, actual,
-		[]cmp.Option{cmp.AllowUnexported(mockSample{}), cmp.AllowUnexported(mockExemplar{}), cmp.AllowUnexported(mockHistogram{})},
+		[]cmp.Option{cmp.AllowUnexported(mockSample{}), cmp.AllowUnexported(mockExemplar{}), cmp.AllowUnexported(mockHistogram{}), cmp.AllowUnexported(mockMetadata{})},
 		msgAndArgs...)
 }
 
 func (m *mockAppendable) Appender(_ context.Context) storage.Appender {
+	if m.latestSample == nil {
+		m.latestSample = map[uint64]int64{}
+	}
+	if m.latestHistogram == nil {
+		m.latestHistogram = map[uint64]int64{}
+	}
+	if m.latestExemplar == nil {
+		m.latestExemplar = map[uint64]int64{}
+	}
 	return m
 }
 
 func (m *mockAppendable) Append(_ storage.SeriesRef, l labels.Labels, t int64, v float64) (storage.SeriesRef, error) {
-	if t < m.latestSample {
-		return 0, storage.ErrOutOfOrderSample
+	if m.appendSampleErr != nil {
+		return 0, m.appendSampleErr
 	}
 
-	m.latestSample = t
+	latestTs := m.latestSample[l.Hash()]
+	if t < latestTs {
+		return 0, storage.ErrOutOfOrderSample
+	}
+	if t == latestTs {
+		return 0, storage.ErrDuplicateSampleForTimestamp
+	}
+
+	m.latestSample[l.Hash()] = t
 	m.samples = append(m.samples, mockSample{l, t, v})
 	return 0, nil
 }
 
 func (m *mockAppendable) Commit() error {
+	if m.commitErr != nil {
+		_ = m.Rollback() // As per Commit method contract.
+	}
 	return m.commitErr
 }
 
-func (*mockAppendable) Rollback() error {
-	return fmt.Errorf("not implemented")
+func (m *mockAppendable) Rollback() error {
+	m.samples = m.samples[:0]
+	m.exemplars = m.exemplars[:0]
+	m.histograms = m.histograms[:0]
+	m.metadata = m.metadata[:0]
+	return nil
 }
 
 func (m *mockAppendable) AppendExemplar(_ storage.SeriesRef, l labels.Labels, e exemplar.Exemplar) (storage.SeriesRef, error) {
-	if e.Ts < m.latestExemplar {
-		return 0, storage.ErrOutOfOrderExemplar
+	if m.appendExemplarErr != nil {
+		return 0, m.appendExemplarErr
 	}
 
-	m.latestExemplar = e.Ts
+	latestTs := m.latestExemplar[l.Hash()]
+	if e.Ts < latestTs {
+		return 0, storage.ErrOutOfOrderExemplar
+	}
+	if e.Ts == latestTs {
+		return 0, storage.ErrDuplicateExemplar
+	}
+
+	m.latestExemplar[l.Hash()] = e.Ts
 	m.exemplars = append(m.exemplars, mockExemplar{l, e.Labels, e.Ts, e.Value})
 	return 0, nil
 }
 
 func (m *mockAppendable) AppendHistogram(_ storage.SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) {
-	if t < m.latestHistogram {
-		return 0, storage.ErrOutOfOrderSample
+	if m.appendHistogramErr != nil {
+		return 0, m.appendHistogramErr
 	}
 
-	m.latestHistogram = t
+	latestTs := m.latestHistogram[l.Hash()]
+	if t < latestTs {
+		return 0, storage.ErrOutOfOrderSample
+	}
+	if t == latestTs {
+		return 0, storage.ErrDuplicateSampleForTimestamp
+	}
+
+	m.latestHistogram[l.Hash()] = t
 	m.histograms = append(m.histograms, mockHistogram{l, t, h, fh})
 	return 0, nil
 }
 
-func (m *mockAppendable) UpdateMetadata(_ storage.SeriesRef, _ labels.Labels, _ metadata.Metadata) (storage.SeriesRef, error) {
-	// TODO: Wire metadata in a mockAppendable field when we get around to handling metadata in remote_write.
-	// UpdateMetadata is no-op for remote write (where mockAppendable is being used to test) for now.
+func (m *mockAppendable) UpdateMetadata(_ storage.SeriesRef, l labels.Labels, mp metadata.Metadata) (storage.SeriesRef, error) {
+	if m.updateMetadataErr != nil {
+		return 0, m.updateMetadataErr
+	}
+
+	m.metadata = append(m.metadata, mockMetadata{l: l, m: mp})
 	return 0, nil
 }
 
 func (m *mockAppendable) AppendCTZeroSample(_ storage.SeriesRef, _ labels.Labels, _, _ int64) (storage.SeriesRef, error) {
 	// AppendCTZeroSample is no-op for remote-write for now.
+	// TODO(bwplotka): Add support for PRW 2.0 for CT zero feature (but also we might
+	// replace this with in-metadata CT storage, see https://github.com/prometheus/prometheus/issues/14218).
 	return 0, nil
 }

storage/remote/write_test.go

@@ -15,6 +15,7 @@ package remote
 
 import (
 	"bytes"
+	"errors"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -43,11 +44,12 @@ func testRemoteWriteConfig() *config.RemoteWriteConfig {
 				Host:   "localhost",
 			},
 		},
-		QueueConfig: config.DefaultQueueConfig,
+		QueueConfig:     config.DefaultQueueConfig,
+		ProtobufMessage: config.RemoteWriteProtoMsgV1,
 	}
 }
 
-func TestNoDuplicateWriteConfigs(t *testing.T) {
+func TestWriteStorageApplyConfig_NoDuplicateWriteConfigs(t *testing.T) {
 	dir := t.TempDir()
 
 	cfg1 := config.RemoteWriteConfig{
@@ -58,7 +60,8 @@ func TestNoDuplicateWriteConfigs(t *testing.T) {
 				Host:   "localhost",
 			},
 		},
-		QueueConfig: config.DefaultQueueConfig,
+		QueueConfig:     config.DefaultQueueConfig,
+		ProtobufMessage: config.RemoteWriteProtoMsgV1,
 	}
 	cfg2 := config.RemoteWriteConfig{
 		Name: "write-2",
@@ -68,7 +71,8 @@ func TestNoDuplicateWriteConfigs(t *testing.T) {
 				Host:   "localhost",
 			},
 		},
-		QueueConfig: config.DefaultQueueConfig,
+		QueueConfig:     config.DefaultQueueConfig,
+		ProtobufMessage: config.RemoteWriteProtoMsgV1,
 	}
 	cfg3 := config.RemoteWriteConfig{
 		URL: &common_config.URL{
@@ -77,61 +81,49 @@ func TestNoDuplicateWriteConfigs(t *testing.T) {
 				Host:   "localhost",
 			},
 		},
-		QueueConfig: config.DefaultQueueConfig,
+		QueueConfig:     config.DefaultQueueConfig,
+		ProtobufMessage: config.RemoteWriteProtoMsgV1,
 	}
 
-	type testcase struct {
-		cfgs []*config.RemoteWriteConfig
-		err  bool
-	}
-
-	cases := []testcase{
+	for _, tc := range []struct {
+		cfgs        []*config.RemoteWriteConfig
+		expectedErr error
+	}{
 		{ // Two duplicates, we should get an error.
-			cfgs: []*config.RemoteWriteConfig{
-				&cfg1,
-				&cfg1,
-			},
-			err: true,
+			cfgs:        []*config.RemoteWriteConfig{&cfg1, &cfg1},
+			expectedErr: errors.New("duplicate remote write configs are not allowed, found duplicate for URL: http://localhost"),
 		},
 		{ // Duplicates but with different names, we should not get an error.
-			cfgs: []*config.RemoteWriteConfig{
-				&cfg1,
-				&cfg2,
-			},
-			err: false,
+			cfgs: []*config.RemoteWriteConfig{&cfg1, &cfg2},
 		},
 		{ // Duplicates but one with no name, we should not get an error.
-			cfgs: []*config.RemoteWriteConfig{
-				&cfg1,
-				&cfg3,
-			},
-			err: false,
+			cfgs: []*config.RemoteWriteConfig{&cfg1, &cfg3},
 		},
 		{ // Duplicates both with no name, we should get an error.
-			cfgs: []*config.RemoteWriteConfig{
-				&cfg3,
-				&cfg3,
-			},
-			err: true,
+			cfgs:        []*config.RemoteWriteConfig{&cfg3, &cfg3},
+			expectedErr: errors.New("duplicate remote write configs are not allowed, found duplicate for URL: http://localhost"),
 		},
-	}
+	} {
+		t.Run("", func(t *testing.T) {
+			s := NewWriteStorage(nil, nil, dir, time.Millisecond, nil, false)
+			conf := &config.Config{
+				GlobalConfig:       config.DefaultGlobalConfig,
+				RemoteWriteConfigs: tc.cfgs,
+			}
+			err := s.ApplyConfig(conf)
+			if tc.expectedErr == nil {
+				require.NoError(t, err)
+			} else {
+				require.Error(t, err)
+				require.Equal(t, tc.expectedErr, err)
+			}
 
-	for _, tc := range cases {
-		s := NewWriteStorage(nil, nil, dir, time.Millisecond, nil)
-		conf := &config.Config{
-			GlobalConfig:       config.DefaultGlobalConfig,
-			RemoteWriteConfigs: tc.cfgs,
-		}
-		err := s.ApplyConfig(conf)
-		gotError := err != nil
-		require.Equal(t, tc.err, gotError)
-
-		err = s.Close()
-		require.NoError(t, err)
+			require.NoError(t, s.Close())
+		})
 	}
 }
 
-func TestRestartOnNameChange(t *testing.T) {
+func TestWriteStorageApplyConfig_RestartOnNameChange(t *testing.T) {
 	dir := t.TempDir()
 
 	cfg := testRemoteWriteConfig()
@@ -139,13 +131,11 @@ func TestRestartOnNameChange(t *testing.T) {
 	hash, err := toHash(cfg)
 	require.NoError(t, err)
 
-	s := NewWriteStorage(nil, nil, dir, time.Millisecond, nil)
+	s := NewWriteStorage(nil, nil, dir, time.Millisecond, nil, false)
 
 	conf := &config.Config{
-		GlobalConfig: config.DefaultGlobalConfig,
-		RemoteWriteConfigs: []*config.RemoteWriteConfig{
-			cfg,
-		},
+		GlobalConfig:       config.DefaultGlobalConfig,
+		RemoteWriteConfigs: []*config.RemoteWriteConfig{cfg},
 	}
 	require.NoError(t, s.ApplyConfig(conf))
 	require.Equal(t, s.queues[hash].client().Name(), cfg.Name)
@@ -157,14 +147,13 @@ func TestRestartOnNameChange(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, s.queues[hash].client().Name(), conf.RemoteWriteConfigs[0].Name)
 
-	err = s.Close()
-	require.NoError(t, err)
+	require.NoError(t, s.Close())
 }
 
-func TestUpdateWithRegisterer(t *testing.T) {
+func TestWriteStorageApplyConfig_UpdateWithRegisterer(t *testing.T) {
 	dir := t.TempDir()
 
-	s := NewWriteStorage(nil, prometheus.NewRegistry(), dir, time.Millisecond, nil)
+	s := NewWriteStorage(nil, prometheus.NewRegistry(), dir, time.Millisecond, nil, false)
 	c1 := &config.RemoteWriteConfig{
 		Name: "named",
 		URL: &common_config.URL{
@@ -173,7 +162,8 @@ func TestUpdateWithRegisterer(t *testing.T) {
 				Host:   "localhost",
 			},
 		},
-		QueueConfig: config.DefaultQueueConfig,
+		QueueConfig:     config.DefaultQueueConfig,
+		ProtobufMessage: config.RemoteWriteProtoMsgV1,
 	}
 	c2 := &config.RemoteWriteConfig{
 		URL: &common_config.URL{
@@ -182,7 +172,8 @@ func TestUpdateWithRegisterer(t *testing.T) {
 				Host:   "localhost",
 			},
 		},
-		QueueConfig: config.DefaultQueueConfig,
+		QueueConfig:     config.DefaultQueueConfig,
+		ProtobufMessage: config.RemoteWriteProtoMsgV1,
 	}
 	conf := &config.Config{
 		GlobalConfig:       config.DefaultGlobalConfig,
@@ -197,14 +188,13 @@ func TestUpdateWithRegisterer(t *testing.T) {
 		require.Equal(t, 10, queue.cfg.MaxShards)
 	}
 
-	err := s.Close()
-	require.NoError(t, err)
+	require.NoError(t, s.Close())
 }
 
-func TestWriteStorageLifecycle(t *testing.T) {
+func TestWriteStorageApplyConfig_Lifecycle(t *testing.T) {
 	dir := t.TempDir()
 
-	s := NewWriteStorage(nil, nil, dir, defaultFlushDeadline, nil)
+	s := NewWriteStorage(nil, nil, dir, defaultFlushDeadline, nil, false)
 	conf := &config.Config{
 		GlobalConfig: config.DefaultGlobalConfig,
 		RemoteWriteConfigs: []*config.RemoteWriteConfig{
@@ -214,14 +204,13 @@ func TestWriteStorageLifecycle(t *testing.T) {
 	require.NoError(t, s.ApplyConfig(conf))
 	require.Len(t, s.queues, 1)
 
-	err := s.Close()
-	require.NoError(t, err)
+	require.NoError(t, s.Close())
 }
 
-func TestUpdateExternalLabels(t *testing.T) {
+func TestWriteStorageApplyConfig_UpdateExternalLabels(t *testing.T) {
 	dir := t.TempDir()
 
-	s := NewWriteStorage(nil, prometheus.NewRegistry(), dir, time.Second, nil)
+	s := NewWriteStorage(nil, prometheus.NewRegistry(), dir, time.Second, nil, false)
 
 	externalLabels := labels.FromStrings("external", "true")
 	conf := &config.Config{
@@ -243,15 +232,13 @@ func TestUpdateExternalLabels(t *testing.T) {
 	require.Len(t, s.queues, 1)
 	require.Equal(t, []labels.Label{{Name: "external", Value: "true"}}, s.queues[hash].externalLabels)
 
-	err = s.Close()
-	require.NoError(t, err)
+	require.NoError(t, s.Close())
 }
 
-func TestWriteStorageApplyConfigsIdempotent(t *testing.T) {
+func TestWriteStorageApplyConfig_Idempotent(t *testing.T) {
 	dir := t.TempDir()
 
-	s := NewWriteStorage(nil, nil, dir, defaultFlushDeadline, nil)
-
+	s := NewWriteStorage(nil, nil, dir, defaultFlushDeadline, nil, false)
 	conf := &config.Config{
 		GlobalConfig: config.GlobalConfig{},
 		RemoteWriteConfigs: []*config.RemoteWriteConfig{
@@ -269,14 +256,13 @@ func TestWriteStorageApplyConfigsIdempotent(t *testing.T) {
 	_, hashExists := s.queues[hash]
 	require.True(t, hashExists, "Queue pointer should have remained the same")
 
-	err = s.Close()
-	require.NoError(t, err)
+	require.NoError(t, s.Close())
 }
 
-func TestWriteStorageApplyConfigsPartialUpdate(t *testing.T) {
+func TestWriteStorageApplyConfig_PartialUpdate(t *testing.T) {
 	dir := t.TempDir()
 
-	s := NewWriteStorage(nil, nil, dir, defaultFlushDeadline, nil)
+	s := NewWriteStorage(nil, nil, dir, defaultFlushDeadline, nil, false)
 
 	c0 := &config.RemoteWriteConfig{
 		RemoteTimeout: model.Duration(10 * time.Second),
@@ -286,6 +272,7 @@ func TestWriteStorageApplyConfigsPartialUpdate(t *testing.T) {
 				Regex: relabel.MustNewRegexp(".+"),
 			},
 		},
+		ProtobufMessage: config.RemoteWriteProtoMsgV1,
 	}
 	c1 := &config.RemoteWriteConfig{
 		RemoteTimeout: model.Duration(20 * time.Second),
@@ -293,10 +280,12 @@ func TestWriteStorageApplyConfigsPartialUpdate(t *testing.T) {
 		HTTPClientConfig: common_config.HTTPClientConfig{
 			BearerToken: "foo",
 		},
+		ProtobufMessage: config.RemoteWriteProtoMsgV1,
 	}
 	c2 := &config.RemoteWriteConfig{
-		RemoteTimeout: model.Duration(30 * time.Second),
-		QueueConfig:   config.DefaultQueueConfig,
+		RemoteTimeout:   model.Duration(30 * time.Second),
+		QueueConfig:     config.DefaultQueueConfig,
+		ProtobufMessage: config.RemoteWriteProtoMsgV1,
 	}
 
 	conf := &config.Config{
@@ -376,12 +365,11 @@ func TestWriteStorageApplyConfigsPartialUpdate(t *testing.T) {
 	_, hashExists = s.queues[hashes[2]]
 	require.True(t, hashExists, "Pointer of unchanged queue should have remained the same")
 
-	err = s.Close()
-	require.NoError(t, err)
+	require.NoError(t, s.Close())
 }
 
 func TestOTLPWriteHandler(t *testing.T) {
-	exportRequest := generateOTLPWriteRequest(t)
+	exportRequest := generateOTLPWriteRequest()
 
 	buf, err := exportRequest.MarshalProto()
 	require.NoError(t, err)
@@ -404,7 +392,7 @@ func TestOTLPWriteHandler(t *testing.T) {
 	require.Len(t, appendable.exemplars, 1)  // 1 (exemplar)
 }
 
-func generateOTLPWriteRequest(t *testing.T) pmetricotlp.ExportRequest {
+func generateOTLPWriteRequest() pmetricotlp.ExportRequest {
 	d := pmetric.NewMetrics()
 
 	// Generate One Counter, One Gauge, One Histogram, One Exponential-Histogram
@@ -434,6 +422,7 @@ func generateOTLPWriteRequest(t *testing.T) pmetricotlp.ExportRequest {
 	counterDataPoint.Attributes().PutStr("foo.bar", "baz")
 
 	counterExemplar := counterDataPoint.Exemplars().AppendEmpty()
+
 	counterExemplar.SetTimestamp(pcommon.NewTimestampFromTime(timestamp))
 	counterExemplar.SetDoubleValue(10.0)
 	counterExemplar.SetSpanID(pcommon.SpanID{0, 1, 2, 3, 4, 5, 6, 7})

storage/secondary.go

@@ -49,16 +49,16 @@ func newSecondaryQuerierFromChunk(cq ChunkQuerier) genericQuerier {
 	return &secondaryQuerier{genericQuerier: newGenericQuerierFromChunk(cq)}
 }
 
-func (s *secondaryQuerier) LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
-	vals, w, err := s.genericQuerier.LabelValues(ctx, name, matchers...)
+func (s *secondaryQuerier) LabelValues(ctx context.Context, name string, hints *LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+	vals, w, err := s.genericQuerier.LabelValues(ctx, name, hints, matchers...)
 	if err != nil {
 		return nil, w.Add(err), nil
 	}
 	return vals, w, nil
 }
 
-func (s *secondaryQuerier) LabelNames(ctx context.Context, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
-	names, w, err := s.genericQuerier.LabelNames(ctx, matchers...)
+func (s *secondaryQuerier) LabelNames(ctx context.Context, hints *LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+	names, w, err := s.genericQuerier.LabelNames(ctx, hints, matchers...)
 	if err != nil {
 		return nil, w.Add(err), nil
 	}

tsdb/agent/db_test.go

@@ -89,7 +89,7 @@ func createTestAgentDB(t testing.TB, reg prometheus.Registerer, opts *Options) *
 	t.Helper()
 
 	dbDir := t.TempDir()
-	rs := remote.NewStorage(log.NewNopLogger(), reg, startTime, dbDir, time.Second*30, nil)
+	rs := remote.NewStorage(log.NewNopLogger(), reg, startTime, dbDir, time.Second*30, nil, false)
 	t.Cleanup(func() {
 		require.NoError(t, rs.Close())
 	})
@@ -585,7 +585,7 @@ func TestLockfile(t *testing.T) {
 	tsdbutil.TestDirLockerUsage(t, func(t *testing.T, data string, createLock bool) (*tsdbutil.DirLocker, testutil.Closer) {
 		logger := log.NewNopLogger()
 		reg := prometheus.NewRegistry()
-		rs := remote.NewStorage(logger, reg, startTime, data, time.Second*30, nil)
+		rs := remote.NewStorage(logger, reg, startTime, data, time.Second*30, nil, false)
 		t.Cleanup(func() {
 			require.NoError(t, rs.Close())
 		})
@@ -605,7 +605,7 @@ func TestLockfile(t *testing.T) {
 
 func Test_ExistingWAL_NextRef(t *testing.T) {
 	dbDir := t.TempDir()
-	rs := remote.NewStorage(log.NewNopLogger(), nil, startTime, dbDir, time.Second*30, nil)
+	rs := remote.NewStorage(log.NewNopLogger(), nil, startTime, dbDir, time.Second*30, nil, false)
 	defer func() {
 		require.NoError(t, rs.Close())
 	}()

tsdb/chunkenc/xor.go

@@ -60,7 +60,7 @@ type XORChunk struct {
 	b bstream
 }
 
-// NewXORChunk returns a new chunk with XOR encoding of the given size.
+// NewXORChunk returns a new chunk with XOR encoding.
 func NewXORChunk() *XORChunk {
 	b := make([]byte, 2, 128)
 	return &XORChunk{b: bstream{stream: b, count: 0}}

tsdb/chunks/chunks.go

@@ -133,15 +133,6 @@ type Meta struct {
 	// Time range the data covers.
 	// When MaxTime == math.MaxInt64 the chunk is still open and being appended to.
 	MinTime, MaxTime int64
-
-	// OOOLastRef, OOOLastMinTime and OOOLastMaxTime are kept as markers for
-	// overlapping chunks.
-	// These fields point to the last created out of order Chunk (the head) that existed
-	// when Series() was called and was overlapping.
-	// Series() and Chunk() method responses should be consistent for the same
-	// query even if new data is added in between the calls.
-	OOOLastRef                     ChunkRef
-	OOOLastMinTime, OOOLastMaxTime int64
 }
 
 // ChunkFromSamples requires all samples to have the same type.

tsdb/db.go

@@ -1407,6 +1407,9 @@ func (db *DB) compactHead(head *RangeHead) error {
 	if err = db.head.truncateMemory(head.BlockMaxTime()); err != nil {
 		return fmt.Errorf("head memory truncate: %w", err)
 	}
+
+	db.head.RebuildSymbolTable(db.logger)
+
 	return nil
 }
 

tsdb/db_test.go

@@ -1001,7 +1001,7 @@ func TestWALFlushedOnDBClose(t *testing.T) {
 	q, err := db.Querier(0, 1)
 	require.NoError(t, err)
 
-	values, ws, err := q.LabelValues(ctx, "labelname")
+	values, ws, err := q.LabelValues(ctx, "labelname", nil)
 	require.NoError(t, err)
 	require.Empty(t, ws)
 	require.Equal(t, []string{"labelvalue"}, values)
@@ -1976,7 +1976,7 @@ func TestQuerierWithBoundaryChunks(t *testing.T) {
 	defer q.Close()
 
 	// The requested interval covers 2 blocks, so the querier's label values for blockID should give us 2 values, one from each block.
-	b, ws, err := q.LabelValues(ctx, "blockID")
+	b, ws, err := q.LabelValues(ctx, "blockID", nil)
 	require.NoError(t, err)
 	var nilAnnotations annotations.Annotations
 	require.Equal(t, nilAnnotations, ws)
@@ -2288,7 +2288,7 @@ func TestDB_LabelNames(t *testing.T) {
 		q, err := db.Querier(math.MinInt64, math.MaxInt64)
 		require.NoError(t, err)
 		var ws annotations.Annotations
-		labelNames, ws, err = q.LabelNames(ctx)
+		labelNames, ws, err = q.LabelNames(ctx, nil)
 		require.NoError(t, err)
 		require.Empty(t, ws)
 		require.NoError(t, q.Close())
@@ -4495,6 +4495,14 @@ func TestMetadataAssertInMemoryData(t *testing.T) {
 //
 //	are not included in this compaction.
 func TestOOOCompaction(t *testing.T) {
+	for name, scenario := range sampleTypeScenarios {
+		t.Run(name, func(t *testing.T) {
+			testOOOCompaction(t, scenario)
+		})
+	}
+}
+
+func testOOOCompaction(t *testing.T, scenario sampleTypeScenario) {
 	dir := t.TempDir()
 	ctx := context.Background()
 
@@ -4516,9 +4524,9 @@ func TestOOOCompaction(t *testing.T) {
 		app := db.Appender(context.Background())
 		for min := fromMins; min <= toMins; min++ {
 			ts := min * time.Minute.Milliseconds()
-			_, err := app.Append(0, series1, ts, float64(ts))
+			_, _, err := scenario.appendFunc(app, series1, ts, ts)
 			require.NoError(t, err)
-			_, err = app.Append(0, series2, ts, float64(2*ts))
+			_, _, err = scenario.appendFunc(app, series2, ts, 2*ts)
 			require.NoError(t, err)
 		}
 		require.NoError(t, app.Commit())
@@ -4551,8 +4559,8 @@ func TestOOOCompaction(t *testing.T) {
 			fromMins, toMins := r[0], r[1]
 			for min := fromMins; min <= toMins; min++ {
 				ts := min * time.Minute.Milliseconds()
-				series1Samples = append(series1Samples, sample{ts, float64(ts), nil, nil})
-				series2Samples = append(series2Samples, sample{ts, float64(2 * ts), nil, nil})
+				series1Samples = append(series1Samples, scenario.sampleFunc(ts, ts))
+				series2Samples = append(series2Samples, scenario.sampleFunc(ts, 2*ts))
 			}
 		}
 		expRes := map[string][]chunks.Sample{
@@ -4564,7 +4572,7 @@ func TestOOOCompaction(t *testing.T) {
 		require.NoError(t, err)
 
 		actRes := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar.*"))
-		require.Equal(t, expRes, actRes)
+		requireEqualSeries(t, expRes, actRes, true)
 	}
 
 	verifyDBSamples() // Before any compaction.
@@ -4619,8 +4627,8 @@ func TestOOOCompaction(t *testing.T) {
 		series2Samples := make([]chunks.Sample, 0, toMins-fromMins+1)
 		for min := fromMins; min <= toMins; min++ {
 			ts := min * time.Minute.Milliseconds()
-			series1Samples = append(series1Samples, sample{ts, float64(ts), nil, nil})
-			series2Samples = append(series2Samples, sample{ts, float64(2 * ts), nil, nil})
+			series1Samples = append(series1Samples, scenario.sampleFunc(ts, ts))
+			series2Samples = append(series2Samples, scenario.sampleFunc(ts, 2*ts))
 		}
 		expRes := map[string][]chunks.Sample{
 			series1.String(): series1Samples,
@@ -4631,7 +4639,7 @@ func TestOOOCompaction(t *testing.T) {
 		require.NoError(t, err)
 
 		actRes := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar.*"))
-		require.Equal(t, expRes, actRes)
+		requireEqualSeries(t, expRes, actRes, true)
 	}
 
 	// Checking for expected data in the blocks.
@@ -4675,6 +4683,14 @@ func TestOOOCompaction(t *testing.T) {
 // TestOOOCompactionWithNormalCompaction tests if OOO compaction is performed
 // when the normal head's compaction is done.
 func TestOOOCompactionWithNormalCompaction(t *testing.T) {
+	for name, scenario := range sampleTypeScenarios {
+		t.Run(name, func(t *testing.T) {
+			testOOOCompactionWithNormalCompaction(t, scenario)
+		})
+	}
+}
+
+func testOOOCompactionWithNormalCompaction(t *testing.T, scenario sampleTypeScenario) {
 	dir := t.TempDir()
 	ctx := context.Background()
 
@@ -4696,9 +4712,9 @@ func TestOOOCompactionWithNormalCompaction(t *testing.T) {
 		app := db.Appender(context.Background())
 		for min := fromMins; min <= toMins; min++ {
 			ts := min * time.Minute.Milliseconds()
-			_, err := app.Append(0, series1, ts, float64(ts))
+			_, _, err := scenario.appendFunc(app, series1, ts, ts)
 			require.NoError(t, err)
-			_, err = app.Append(0, series2, ts, float64(2*ts))
+			_, _, err = scenario.appendFunc(app, series2, ts, 2*ts)
 			require.NoError(t, err)
 		}
 		require.NoError(t, app.Commit())
@@ -4751,8 +4767,8 @@ func TestOOOCompactionWithNormalCompaction(t *testing.T) {
 		series2Samples := make([]chunks.Sample, 0, toMins-fromMins+1)
 		for min := fromMins; min <= toMins; min++ {
 			ts := min * time.Minute.Milliseconds()
-			series1Samples = append(series1Samples, sample{ts, float64(ts), nil, nil})
-			series2Samples = append(series2Samples, sample{ts, float64(2 * ts), nil, nil})
+			series1Samples = append(series1Samples, scenario.sampleFunc(ts, ts))
+			series2Samples = append(series2Samples, scenario.sampleFunc(ts, 2*ts))
 		}
 		expRes := map[string][]chunks.Sample{
 			series1.String(): series1Samples,
@@ -4763,7 +4779,7 @@ func TestOOOCompactionWithNormalCompaction(t *testing.T) {
 		require.NoError(t, err)
 
 		actRes := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar.*"))
-		require.Equal(t, expRes, actRes)
+		requireEqualSeries(t, expRes, actRes, true)
 	}
 
 	// Checking for expected data in the blocks.
@@ -4775,6 +4791,14 @@ func TestOOOCompactionWithNormalCompaction(t *testing.T) {
 // configured to not have wal and wbl but its able to compact both the in-order
 // and out-of-order head.
 func TestOOOCompactionWithDisabledWriteLog(t *testing.T) {
+	for name, scenario := range sampleTypeScenarios {
+		t.Run(name, func(t *testing.T) {
+			testOOOCompactionWithDisabledWriteLog(t, scenario)
+		})
+	}
+}
+
+func testOOOCompactionWithDisabledWriteLog(t *testing.T, scenario sampleTypeScenario) {
 	dir := t.TempDir()
 	ctx := context.Background()
 
@@ -4797,9 +4821,9 @@ func TestOOOCompactionWithDisabledWriteLog(t *testing.T) {
 		app := db.Appender(context.Background())
 		for min := fromMins; min <= toMins; min++ {
 			ts := min * time.Minute.Milliseconds()
-			_, err := app.Append(0, series1, ts, float64(ts))
+			_, _, err := scenario.appendFunc(app, series1, ts, ts)
 			require.NoError(t, err)
-			_, err = app.Append(0, series2, ts, float64(2*ts))
+			_, _, err = scenario.appendFunc(app, series2, ts, 2*ts)
 			require.NoError(t, err)
 		}
 		require.NoError(t, app.Commit())
@@ -4852,8 +4876,8 @@ func TestOOOCompactionWithDisabledWriteLog(t *testing.T) {
 		series2Samples := make([]chunks.Sample, 0, toMins-fromMins+1)
 		for min := fromMins; min <= toMins; min++ {
 			ts := min * time.Minute.Milliseconds()
-			series1Samples = append(series1Samples, sample{ts, float64(ts), nil, nil})
-			series2Samples = append(series2Samples, sample{ts, float64(2 * ts), nil, nil})
+			series1Samples = append(series1Samples, scenario.sampleFunc(ts, ts))
+			series2Samples = append(series2Samples, scenario.sampleFunc(ts, 2*ts))
 		}
 		expRes := map[string][]chunks.Sample{
 			series1.String(): series1Samples,
@@ -4864,7 +4888,7 @@ func TestOOOCompactionWithDisabledWriteLog(t *testing.T) {
 		require.NoError(t, err)
 
 		actRes := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar.*"))
-		require.Equal(t, expRes, actRes)
+		requireEqualSeries(t, expRes, actRes, true)
 	}
 
 	// Checking for expected data in the blocks.
@@ -4876,6 +4900,14 @@ func TestOOOCompactionWithDisabledWriteLog(t *testing.T) {
 // missing after a restart while snapshot was enabled, but the query still returns the right
 // data from the mmap chunks.
 func TestOOOQueryAfterRestartWithSnapshotAndRemovedWBL(t *testing.T) {
+	for name, scenario := range sampleTypeScenarios {
+		t.Run(name, func(t *testing.T) {
+			testOOOQueryAfterRestartWithSnapshotAndRemovedWBL(t, scenario)
+		})
+	}
+}
+
+func testOOOQueryAfterRestartWithSnapshotAndRemovedWBL(t *testing.T, scenario sampleTypeScenario) {
 	dir := t.TempDir()
 	ctx := context.Background()
 
@@ -4898,9 +4930,9 @@ func TestOOOQueryAfterRestartWithSnapshotAndRemovedWBL(t *testing.T) {
 		app := db.Appender(context.Background())
 		for min := fromMins; min <= toMins; min++ {
 			ts := min * time.Minute.Milliseconds()
-			_, err := app.Append(0, series1, ts, float64(ts))
+			_, _, err := scenario.appendFunc(app, series1, ts, ts)
 			require.NoError(t, err)
-			_, err = app.Append(0, series2, ts, float64(2*ts))
+			_, _, err = scenario.appendFunc(app, series2, ts, 2*ts)
 			require.NoError(t, err)
 		}
 		require.NoError(t, app.Commit())
@@ -4946,8 +4978,8 @@ func TestOOOQueryAfterRestartWithSnapshotAndRemovedWBL(t *testing.T) {
 		series2Samples := make([]chunks.Sample, 0, toMins-fromMins+1)
 		for min := fromMins; min <= toMins; min++ {
 			ts := min * time.Minute.Milliseconds()
-			series1Samples = append(series1Samples, sample{ts, float64(ts), nil, nil})
-			series2Samples = append(series2Samples, sample{ts, float64(2 * ts), nil, nil})
+			series1Samples = append(series1Samples, scenario.sampleFunc(ts, ts))
+			series2Samples = append(series2Samples, scenario.sampleFunc(ts, ts*2))
 		}
 		expRes := map[string][]chunks.Sample{
 			series1.String(): series1Samples,
@@ -4958,7 +4990,7 @@ func TestOOOQueryAfterRestartWithSnapshotAndRemovedWBL(t *testing.T) {
 		require.NoError(t, err)
 
 		actRes := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar.*"))
-		require.Equal(t, expRes, actRes)
+		requireEqualSeries(t, expRes, actRes, true)
 	}
 
 	// Checking for expected ooo data from mmap chunks.
@@ -5159,6 +5191,14 @@ func Test_ChunkQuerier_OOOQuery(t *testing.T) {
 }
 
 func TestOOOAppendAndQuery(t *testing.T) {
+	for name, scenario := range sampleTypeScenarios {
+		t.Run(name, func(t *testing.T) {
+			testOOOAppendAndQuery(t, scenario)
+		})
+	}
+}
+
+func testOOOAppendAndQuery(t *testing.T, scenario sampleTypeScenario) {
 	opts := DefaultOptions()
 	opts.OutOfOrderCapMax = 30
 	opts.OutOfOrderTimeWindow = 4 * time.Hour.Milliseconds()
@@ -5180,13 +5220,13 @@ func TestOOOAppendAndQuery(t *testing.T) {
 		key := lbls.String()
 		from, to := minutes(fromMins), minutes(toMins)
 		for min := from; min <= to; min += time.Minute.Milliseconds() {
-			val := rand.Float64()
-			_, err := app.Append(0, lbls, min, val)
+			val := rand.Intn(1000)
+			_, s, err := scenario.appendFunc(app, lbls, min, int64(val))
 			if faceError {
 				require.Error(t, err)
 			} else {
 				require.NoError(t, err)
-				appendedSamples[key] = append(appendedSamples[key], sample{t: min, f: val})
+				appendedSamples[key] = append(appendedSamples[key], s)
 				totalSamples++
 			}
 		}
@@ -5222,7 +5262,7 @@ func TestOOOAppendAndQuery(t *testing.T) {
 				expSamples[k] = append(expSamples[k], s)
 			}
 		}
-		require.Equal(t, expSamples, seriesSet)
+		requireEqualSeries(t, expSamples, seriesSet, true)
 		requireEqualOOOSamples(t, totalSamples-2, db)
 	}
 
@@ -5284,6 +5324,14 @@ func TestOOOAppendAndQuery(t *testing.T) {
 }
 
 func TestOOODisabled(t *testing.T) {
+	for name, scenario := range sampleTypeScenarios {
+		t.Run(name, func(t *testing.T) {
+			testOOODisabled(t, scenario)
+		})
+	}
+}
+
+func testOOODisabled(t *testing.T, scenario sampleTypeScenario) {
 	opts := DefaultOptions()
 	opts.OutOfOrderTimeWindow = 0
 	db := openTestDB(t, opts, nil)
@@ -5297,19 +5345,19 @@ func TestOOODisabled(t *testing.T) {
 	expSamples := make(map[string][]chunks.Sample)
 	totalSamples := 0
 	failedSamples := 0
-	addSample := func(lbls labels.Labels, fromMins, toMins int64, faceError bool) {
+
+	addSample := func(db *DB, lbls labels.Labels, fromMins, toMins int64, faceError bool) {
 		app := db.Appender(context.Background())
 		key := lbls.String()
 		from, to := minutes(fromMins), minutes(toMins)
 		for min := from; min <= to; min += time.Minute.Milliseconds() {
-			val := rand.Float64()
-			_, err := app.Append(0, lbls, min, val)
+			_, _, err := scenario.appendFunc(app, lbls, min, min)
 			if faceError {
 				require.Error(t, err)
 				failedSamples++
 			} else {
 				require.NoError(t, err)
-				expSamples[key] = append(expSamples[key], sample{t: min, f: val})
+				expSamples[key] = append(expSamples[key], scenario.sampleFunc(min, min))
 				totalSamples++
 			}
 		}
@@ -5320,21 +5368,21 @@ func TestOOODisabled(t *testing.T) {
 		}
 	}
 
-	addSample(s1, 300, 300, false) // In-order samples.
-	addSample(s1, 250, 260, true)  // Some ooo samples.
-	addSample(s1, 59, 59, true)    // Out of time window.
-	addSample(s1, 60, 65, true)    // At the edge of time window, also it would be "out of bound" without the ooo support.
-	addSample(s1, 59, 59, true)    // Out of time window again.
-	addSample(s1, 301, 310, false) // More in-order samples.
+	addSample(db, s1, 300, 300, false) // In-order samples.
+	addSample(db, s1, 250, 260, true)  // Some ooo samples.
+	addSample(db, s1, 59, 59, true)    // Out of time window.
+	addSample(db, s1, 60, 65, true)    // At the edge of time window, also it would be "out of bound" without the ooo support.
+	addSample(db, s1, 59, 59, true)    // Out of time window again.
+	addSample(db, s1, 301, 310, false) // More in-order samples.
 
 	querier, err := db.Querier(math.MinInt64, math.MaxInt64)
 	require.NoError(t, err)
 
 	seriesSet := query(t, querier, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar."))
-	require.Equal(t, expSamples, seriesSet)
+	requireEqualSeries(t, expSamples, seriesSet, true)
 	requireEqualOOOSamples(t, 0, db)
 	require.Equal(t, float64(failedSamples),
-		prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat))+prom_testutil.ToFloat64(db.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeFloat)),
+		prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType))+prom_testutil.ToFloat64(db.head.metrics.outOfBoundSamples.WithLabelValues(scenario.sampleType)),
 		"number of ooo/oob samples mismatch")
 
 	// Verifying that no OOO artifacts were generated.
@@ -5349,6 +5397,14 @@ func TestOOODisabled(t *testing.T) {
 }
 
 func TestWBLAndMmapReplay(t *testing.T) {
+	for name, scenario := range sampleTypeScenarios {
+		t.Run(name, func(t *testing.T) {
+			testWBLAndMmapReplay(t, scenario)
+		})
+	}
+}
+
+func testWBLAndMmapReplay(t *testing.T, scenario sampleTypeScenario) {
 	opts := DefaultOptions()
 	opts.OutOfOrderCapMax = 30
 	opts.OutOfOrderTimeWindow = 4 * time.Hour.Milliseconds()
@@ -5369,10 +5425,10 @@ func TestWBLAndMmapReplay(t *testing.T) {
 		key := lbls.String()
 		from, to := minutes(fromMins), minutes(toMins)
 		for min := from; min <= to; min += time.Minute.Milliseconds() {
-			val := rand.Float64()
-			_, err := app.Append(0, lbls, min, val)
+			val := rand.Intn(1000)
+			_, s, err := scenario.appendFunc(app, lbls, min, int64(val))
 			require.NoError(t, err)
-			expSamples[key] = append(expSamples[key], sample{t: min, f: val})
+			expSamples[key] = append(expSamples[key], s)
 			totalSamples++
 		}
 		require.NoError(t, app.Commit())
@@ -5390,7 +5446,7 @@ func TestWBLAndMmapReplay(t *testing.T) {
 			})
 			exp[k] = v
 		}
-		require.Equal(t, exp, seriesSet)
+		requireEqualSeries(t, exp, seriesSet, true)
 	}
 
 	// In-order samples.
@@ -5413,10 +5469,9 @@ func TestWBLAndMmapReplay(t *testing.T) {
 		chk, err := db.head.chunkDiskMapper.Chunk(mc.ref)
 		require.NoError(t, err)
 		it := chk.Iterator(nil)
-		for it.Next() == chunkenc.ValFloat {
-			ts, val := it.At()
-			s1MmapSamples = append(s1MmapSamples, sample{t: ts, f: val})
-		}
+		smpls, err := storage.ExpandSamples(it, newSample)
+		require.NoError(t, err)
+		s1MmapSamples = append(s1MmapSamples, smpls...)
 	}
 	require.NotEmpty(t, s1MmapSamples)
 
@@ -5534,6 +5589,14 @@ func TestWBLAndMmapReplay(t *testing.T) {
 }
 
 func TestOOOCompactionFailure(t *testing.T) {
+	for name, scenario := range sampleTypeScenarios {
+		t.Run(name, func(t *testing.T) {
+			testOOOCompactionFailure(t, scenario)
+		})
+	}
+}
+
+func testOOOCompactionFailure(t *testing.T, scenario sampleTypeScenario) {
 	dir := t.TempDir()
 	ctx := context.Background()
 
@@ -5554,7 +5617,7 @@ func TestOOOCompactionFailure(t *testing.T) {
 		app := db.Appender(context.Background())
 		for min := fromMins; min <= toMins; min++ {
 			ts := min * time.Minute.Milliseconds()
-			_, err := app.Append(0, series1, ts, float64(ts))
+			_, _, err := scenario.appendFunc(app, series1, ts, ts)
 			require.NoError(t, err)
 		}
 		require.NoError(t, app.Commit())
@@ -5642,7 +5705,7 @@ func TestOOOCompactionFailure(t *testing.T) {
 		series1Samples := make([]chunks.Sample, 0, toMins-fromMins+1)
 		for min := fromMins; min <= toMins; min++ {
 			ts := min * time.Minute.Milliseconds()
-			series1Samples = append(series1Samples, sample{ts, float64(ts), nil, nil})
+			series1Samples = append(series1Samples, scenario.sampleFunc(ts, ts))
 		}
 		expRes := map[string][]chunks.Sample{
 			series1.String(): series1Samples,
@@ -5650,9 +5713,8 @@ func TestOOOCompactionFailure(t *testing.T) {
 
 		q, err := NewBlockQuerier(block, math.MinInt64, math.MaxInt64)
 		require.NoError(t, err)
-
 		actRes := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar.*"))
-		require.Equal(t, expRes, actRes)
+		requireEqualSeries(t, expRes, actRes, true)
 	}
 
 	// Checking for expected data in the blocks.
@@ -5819,6 +5881,14 @@ func TestWBLCorruption(t *testing.T) {
 }
 
 func TestOOOMmapCorruption(t *testing.T) {
+	for name, scenario := range sampleTypeScenarios {
+		t.Run(name, func(t *testing.T) {
+			testOOOMmapCorruption(t, scenario)
+		})
+	}
+}
+
+func testOOOMmapCorruption(t *testing.T, scenario sampleTypeScenario) {
 	dir := t.TempDir()
 
 	opts := DefaultOptions()
@@ -5838,11 +5908,11 @@ func TestOOOMmapCorruption(t *testing.T) {
 		app := db.Appender(context.Background())
 		for min := fromMins; min <= toMins; min++ {
 			ts := min * time.Minute.Milliseconds()
-			_, err := app.Append(0, series1, ts, float64(ts))
+			_, s, err := scenario.appendFunc(app, series1, ts, ts)
 			require.NoError(t, err)
-			allSamples = append(allSamples, sample{t: ts, f: float64(ts)})
+			allSamples = append(allSamples, s)
 			if inMmapAfterCorruption {
-				expInMmapChunks = append(expInMmapChunks, sample{t: ts, f: float64(ts)})
+				expInMmapChunks = append(expInMmapChunks, s)
 			}
 		}
 		require.NoError(t, app.Commit())
@@ -5880,7 +5950,7 @@ func TestOOOMmapCorruption(t *testing.T) {
 		require.NoError(t, err)
 
 		actRes := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar.*"))
-		require.Equal(t, expRes, actRes)
+		requireEqualSeries(t, expRes, actRes, true)
 	}
 
 	verifySamples(allSamples)
@@ -5942,6 +6012,14 @@ func TestOOOMmapCorruption(t *testing.T) {
 }
 
 func TestOutOfOrderRuntimeConfig(t *testing.T) {
+	for name, scenario := range sampleTypeScenarios {
+		t.Run(name, func(t *testing.T) {
+			testOutOfOrderRuntimeConfig(t, scenario)
+		})
+	}
+}
+
+func testOutOfOrderRuntimeConfig(t *testing.T, scenario sampleTypeScenario) {
 	ctx := context.Background()
 
 	getDB := func(oooTimeWindow int64) *DB {
@@ -5975,10 +6053,10 @@ func TestOutOfOrderRuntimeConfig(t *testing.T) {
 		app := db.Appender(context.Background())
 		for min := fromMins; min <= toMins; min++ {
 			ts := min * time.Minute.Milliseconds()
-			_, err := app.Append(0, series1, ts, float64(ts))
+			_, s, err := scenario.appendFunc(app, series1, ts, ts)
 			if success {
 				require.NoError(t, err)
-				allSamples = append(allSamples, sample{t: ts, f: float64(ts)})
+				allSamples = append(allSamples, s)
 			} else {
 				require.Error(t, err)
 			}
@@ -6000,7 +6078,7 @@ func TestOutOfOrderRuntimeConfig(t *testing.T) {
 		require.NoError(t, err)
 
 		actRes := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar.*"))
-		require.Equal(t, expRes, actRes)
+		requireEqualSeries(t, expRes, actRes, true)
 	}
 
 	doOOOCompaction := func(t *testing.T, db *DB) {
@@ -6173,12 +6251,20 @@ func TestOutOfOrderRuntimeConfig(t *testing.T) {
 }
 
 func TestNoGapAfterRestartWithOOO(t *testing.T) {
+	for name, scenario := range sampleTypeScenarios {
+		t.Run(name, func(t *testing.T) {
+			testNoGapAfterRestartWithOOO(t, scenario)
+		})
+	}
+}
+
+func testNoGapAfterRestartWithOOO(t *testing.T, scenario sampleTypeScenario) {
 	series1 := labels.FromStrings("foo", "bar1")
 	addSamples := func(t *testing.T, db *DB, fromMins, toMins int64, success bool) {
 		app := db.Appender(context.Background())
 		for min := fromMins; min <= toMins; min++ {
 			ts := min * time.Minute.Milliseconds()
-			_, err := app.Append(0, series1, ts, float64(ts))
+			_, _, err := scenario.appendFunc(app, series1, ts, ts)
 			if success {
 				require.NoError(t, err)
 			} else {
@@ -6192,7 +6278,7 @@ func TestNoGapAfterRestartWithOOO(t *testing.T) {
 		var expSamples []chunks.Sample
 		for min := fromMins; min <= toMins; min++ {
 			ts := min * time.Minute.Milliseconds()
-			expSamples = append(expSamples, sample{t: ts, f: float64(ts)})
+			expSamples = append(expSamples, scenario.sampleFunc(ts, ts))
 		}
 
 		expRes := map[string][]chunks.Sample{
@@ -6203,7 +6289,7 @@ func TestNoGapAfterRestartWithOOO(t *testing.T) {
 		require.NoError(t, err)
 
 		actRes := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar.*"))
-		require.Equal(t, expRes, actRes)
+		requireEqualSeries(t, expRes, actRes, true)
 	}
 
 	cases := []struct {
@@ -6280,6 +6366,14 @@ func TestNoGapAfterRestartWithOOO(t *testing.T) {
 }
 
 func TestWblReplayAfterOOODisableAndRestart(t *testing.T) {
+	for name, scenario := range sampleTypeScenarios {
+		t.Run(name, func(t *testing.T) {
+			testWblReplayAfterOOODisableAndRestart(t, scenario)
+		})
+	}
+}
+
+func testWblReplayAfterOOODisableAndRestart(t *testing.T, scenario sampleTypeScenario) {
 	dir := t.TempDir()
 
 	opts := DefaultOptions()
@@ -6298,9 +6392,9 @@ func TestWblReplayAfterOOODisableAndRestart(t *testing.T) {
 		app := db.Appender(context.Background())
 		for min := fromMins; min <= toMins; min++ {
 			ts := min * time.Minute.Milliseconds()
-			_, err := app.Append(0, series1, ts, float64(ts))
+			_, s, err := scenario.appendFunc(app, series1, ts, ts)
 			require.NoError(t, err)
-			allSamples = append(allSamples, sample{t: ts, f: float64(ts)})
+			allSamples = append(allSamples, s)
 		}
 		require.NoError(t, app.Commit())
 	}
@@ -6323,7 +6417,7 @@ func TestWblReplayAfterOOODisableAndRestart(t *testing.T) {
 		require.NoError(t, err)
 
 		actRes := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar.*"))
-		require.Equal(t, expRes, actRes)
+		requireEqualSeries(t, expRes, actRes, true)
 	}
 
 	verifySamples(allSamples)
@@ -6339,6 +6433,14 @@ func TestWblReplayAfterOOODisableAndRestart(t *testing.T) {
 }
 
 func TestPanicOnApplyConfig(t *testing.T) {
+	for name, scenario := range sampleTypeScenarios {
+		t.Run(name, func(t *testing.T) {
+			testPanicOnApplyConfig(t, scenario)
+		})
+	}
+}
+
+func testPanicOnApplyConfig(t *testing.T, scenario sampleTypeScenario) {
 	dir := t.TempDir()
 
 	opts := DefaultOptions()
@@ -6357,9 +6459,9 @@ func TestPanicOnApplyConfig(t *testing.T) {
 		app := db.Appender(context.Background())
 		for min := fromMins; min <= toMins; min++ {
 			ts := min * time.Minute.Milliseconds()
-			_, err := app.Append(0, series1, ts, float64(ts))
+			_, s, err := scenario.appendFunc(app, series1, ts, ts)
 			require.NoError(t, err)
-			allSamples = append(allSamples, sample{t: ts, f: float64(ts)})
+			allSamples = append(allSamples, s)
 		}
 		require.NoError(t, app.Commit())
 	}
@@ -6387,6 +6489,14 @@ func TestPanicOnApplyConfig(t *testing.T) {
 }
 
 func TestDiskFillingUpAfterDisablingOOO(t *testing.T) {
+	for name, scenario := range sampleTypeScenarios {
+		t.Run(name, func(t *testing.T) {
+			testDiskFillingUpAfterDisablingOOO(t, scenario)
+		})
+	}
+}
+
+func testDiskFillingUpAfterDisablingOOO(t *testing.T, scenario sampleTypeScenario) {
 	dir := t.TempDir()
 	ctx := context.Background()
 
@@ -6406,9 +6516,9 @@ func TestDiskFillingUpAfterDisablingOOO(t *testing.T) {
 		app := db.Appender(context.Background())
 		for min := fromMins; min <= toMins; min++ {
 			ts := min * time.Minute.Milliseconds()
-			_, err := app.Append(0, series1, ts, float64(ts))
+			_, s, err := scenario.appendFunc(app, series1, ts, ts)
 			require.NoError(t, err)
-			allSamples = append(allSamples, sample{t: ts, f: float64(ts)})
+			allSamples = append(allSamples, s)
 		}
 		require.NoError(t, app.Commit())
 	}
@@ -7060,12 +7170,6 @@ Outer:
 	require.NoError(t, writerErr)
 }
 
-func requireEqualOOOSamples(t *testing.T, expectedSamples int, db *DB) {
-	require.Equal(t, float64(expectedSamples),
-		prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamplesAppended.WithLabelValues(sampleMetricTypeFloat)),
-		"number of ooo appended samples mismatch")
-}
-
 type mockCompactorFn struct {
 	planFn    func() ([]string, error)
 	compactFn func() ([]ulid.ULID, error)

tsdb/head.go

@@ -1759,12 +1759,12 @@ type seriesHashmap struct {
 
 func (m *seriesHashmap) get(hash uint64, lset labels.Labels) *memSeries {
 	if s, found := m.unique[hash]; found {
-		if labels.Equal(s.lset, lset) {
+		if labels.Equal(s.labels(), lset) {
 			return s
 		}
 	}
 	for _, s := range m.conflicts[hash] {
-		if labels.Equal(s.lset, lset) {
+		if labels.Equal(s.labels(), lset) {
 			return s
 		}
 	}
@@ -1772,7 +1772,7 @@ func (m *seriesHashmap) get(hash uint64, lset labels.Labels) *memSeries {
 }
 
 func (m *seriesHashmap) set(hash uint64, s *memSeries) {
-	if existing, found := m.unique[hash]; !found || labels.Equal(existing.lset, s.lset) {
+	if existing, found := m.unique[hash]; !found || labels.Equal(existing.labels(), s.labels()) {
 		m.unique[hash] = s
 		return
 	}
@@ -1781,7 +1781,7 @@ func (m *seriesHashmap) set(hash uint64, s *memSeries) {
 	}
 	l := m.conflicts[hash]
 	for i, prev := range l {
-		if labels.Equal(prev.lset, s.lset) {
+		if labels.Equal(prev.labels(), s.labels()) {
 			l[i] = s
 			return
 		}
@@ -1931,7 +1931,7 @@ func (s *stripeSeries) gc(mint int64, minOOOMmapRef chunks.ChunkDiskMapperRef) (
 		series.lset.Range(func(l labels.Label) { affected[l] = struct{}{} })
 		s.hashes[hashShard].del(hash, series.ref)
 		delete(s.series[refShard], series.ref)
-		deletedForCallback[series.ref] = series.lset
+		deletedForCallback[series.ref] = series.lset // OK to access lset; series is locked at the top of this function.
 	}
 
 	s.iterForDeletion(check)
@@ -2023,7 +2023,7 @@ func (s *stripeSeries) getOrSet(hash uint64, lset labels.Labels, createSeries fu
 	}
 	// Setting the series in the s.hashes marks the creation of series
 	// as any further calls to this methods would return that series.
-	s.seriesLifecycleCallback.PostCreation(series.lset)
+	s.seriesLifecycleCallback.PostCreation(series.labels())
 
 	i = uint64(series.ref) & uint64(s.size-1)
 
@@ -2064,16 +2064,19 @@ func (s sample) Type() chunkenc.ValueType {
 // memSeries is the in-memory representation of a series. None of its methods
 // are goroutine safe and it is the caller's responsibility to lock it.
 type memSeries struct {
-	sync.Mutex
-
+	// Members up to the Mutex are not changed after construction, so can be accessed without a lock.
 	ref  chunks.HeadSeriesRef
-	lset labels.Labels
 	meta *metadata.Metadata
 
 	// Series labels hash to use for sharding purposes. The value is always 0 when sharding has not
 	// been explicitly enabled in TSDB.
 	shardHash uint64
 
+	// Everything after here should only be accessed with the lock held.
+	sync.Mutex
+
+	lset labels.Labels // Locking required with -tags dedupelabels, not otherwise.
+
 	// Immutable chunks on disk that have not yet gone into a block, in order of ascending time stamps.
 	// When compaction runs, chunks get moved into a block and all pointers are shifted like so:
 	//

tsdb/head_append.go

@@ -554,7 +554,7 @@ func (a *headAppender) AppendExemplar(ref storage.SeriesRef, lset labels.Labels,
 	// Ensure no empty labels have gotten through.
 	e.Labels = e.Labels.WithoutEmpty()
 
-	err := a.head.exemplars.ValidateExemplar(s.lset, e)
+	err := a.head.exemplars.ValidateExemplar(s.labels(), e)
 	if err != nil {
 		if errors.Is(err, storage.ErrDuplicateExemplar) || errors.Is(err, storage.ErrExemplarsDisabled) {
 			// Duplicate, don't return an error but don't accept the exemplar.
@@ -708,7 +708,7 @@ func (a *headAppender) GetRef(lset labels.Labels, hash uint64) (storage.SeriesRe
 		return 0, labels.EmptyLabels()
 	}
 	// returned labels must be suitable to pass to Append()
-	return storage.SeriesRef(s.ref), s.lset
+	return storage.SeriesRef(s.ref), s.labels()
 }
 
 // log writes all headAppender's data to the WAL.
@@ -816,7 +816,7 @@ func (a *headAppender) Commit() (err error) {
 			continue
 		}
 		// We don't instrument exemplar appends here, all is instrumented by storage.
-		if err := a.head.exemplars.AddExemplar(s.lset, e.exemplar); err != nil {
+		if err := a.head.exemplars.AddExemplar(s.labels(), e.exemplar); err != nil {
 			if errors.Is(err, storage.ErrOutOfOrderExemplar) {
 				continue
 			}

tsdb/head_dedupelabels.go

@@ -0,0 +1,95 @@
+// Copyright 2024 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build dedupelabels
+
+package tsdb
+
+import (
+	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
+
+	"github.com/prometheus/prometheus/model/labels"
+)
+
+// Helper method to access labels under lock.
+func (s *memSeries) labels() labels.Labels {
+	s.Lock()
+	defer s.Unlock()
+	return s.lset
+}
+
+// RebuildSymbolTable goes through all the series in h, build a SymbolTable with all names and values,
+// replace each series' Labels with one using that SymbolTable.
+func (h *Head) RebuildSymbolTable(logger log.Logger) *labels.SymbolTable {
+	level.Info(logger).Log("msg", "RebuildSymbolTable starting")
+	st := labels.NewSymbolTable()
+	builder := labels.NewScratchBuilderWithSymbolTable(st, 0)
+	rebuildLabels := func(lbls labels.Labels) labels.Labels {
+		builder.Reset()
+		lbls.Range(func(l labels.Label) {
+			builder.Add(l.Name, l.Value)
+		})
+		return builder.Labels()
+	}
+
+	for i := 0; i < h.series.size; i++ {
+		h.series.locks[i].Lock()
+
+		for _, s := range h.series.hashes[i].unique {
+			s.Lock()
+			s.lset = rebuildLabels(s.lset)
+			s.Unlock()
+		}
+
+		for _, all := range h.series.hashes[i].conflicts {
+			for _, s := range all {
+				s.Lock()
+				s.lset = rebuildLabels(s.lset)
+				s.Unlock()
+			}
+		}
+
+		h.series.locks[i].Unlock()
+	}
+	type withReset interface{ ResetSymbolTable(*labels.SymbolTable) }
+	if e, ok := h.exemplars.(withReset); ok {
+		e.ResetSymbolTable(st)
+	}
+	level.Info(logger).Log("msg", "RebuildSymbolTable finished", "size", st.Len())
+	return st
+}
+
+func (ce *CircularExemplarStorage) ResetSymbolTable(st *labels.SymbolTable) {
+	builder := labels.NewScratchBuilderWithSymbolTable(st, 0)
+	rebuildLabels := func(lbls labels.Labels) labels.Labels {
+		builder.Reset()
+		lbls.Range(func(l labels.Label) {
+			builder.Add(l.Name, l.Value)
+		})
+		return builder.Labels()
+	}
+
+	ce.lock.RLock()
+	defer ce.lock.RUnlock()
+
+	for _, v := range ce.index {
+		v.seriesLabels = rebuildLabels(v.seriesLabels)
+	}
+	for i := range ce.exemplars {
+		if ce.exemplars[i].ref == nil {
+			continue
+		}
+		ce.exemplars[i].exemplar.Labels = rebuildLabels(ce.exemplars[i].exemplar.Labels)
+	}
+}

tsdb/head_other.go

@@ -0,0 +1,32 @@
+// Copyright 2024 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !dedupelabels
+
+package tsdb
+
+import (
+	"github.com/go-kit/log"
+
+	"github.com/prometheus/prometheus/model/labels"
+)
+
+// Helper method to access labels; trivial when not using dedupelabels.
+func (s *memSeries) labels() labels.Labels {
+	return s.lset
+}
+
+// No-op when not using dedupelabels.
+func (h *Head) RebuildSymbolTable(logger log.Logger) *labels.SymbolTable {
+	return nil
+}

tsdb/head_read.go

@@ -142,7 +142,7 @@ func (h *headIndexReader) SortedPostings(p index.Postings) index.Postings {
 	}
 
 	slices.SortFunc(series, func(a, b *memSeries) int {
-		return labels.Compare(a.lset, b.lset)
+		return labels.Compare(a.labels(), b.labels())
 	})
 
 	// Convert back to list.
@@ -189,7 +189,7 @@ func (h *headIndexReader) Series(ref storage.SeriesRef, builder *labels.ScratchB
 		h.head.metrics.seriesNotFound.Inc()
 		return storage.ErrNotFound
 	}
-	builder.Assign(s.lset)
+	builder.Assign(s.labels())
 
 	if chks == nil {
 		return nil
@@ -259,7 +259,7 @@ func (h *headIndexReader) LabelValueFor(_ context.Context, id storage.SeriesRef,
 		return "", storage.ErrNotFound
 	}
 
-	value := memSeries.lset.Get(label)
+	value := memSeries.labels().Get(label)
 	if value == "" {
 		return "", storage.ErrNotFound
 	}
@@ -283,7 +283,7 @@ func (h *headIndexReader) LabelNamesFor(ctx context.Context, series index.Postin
 			// when series was garbage collected after the caller got the series IDs.
 			continue
 		}
-		memSeries.lset.Range(func(lbl labels.Label) {
+		memSeries.labels().Range(func(lbl labels.Label) {
 			namesMap[lbl.Name] = struct{}{}
 		})
 	}
@@ -487,55 +487,24 @@ func (s *memSeries) oooMergedChunks(meta chunks.Meta, cdm *chunks.ChunkDiskMappe
 
 	// We create a temporary slice of chunk metas to hold the information of all
 	// possible chunks that may overlap with the requested chunk.
-	tmpChks := make([]chunkMetaAndChunkDiskMapperRef, 0, len(s.ooo.oooMmappedChunks))
-
-	oooHeadRef := chunks.ChunkRef(chunks.NewHeadChunkRef(s.ref, s.oooHeadChunkID(len(s.ooo.oooMmappedChunks))))
-	if s.ooo.oooHeadChunk != nil && s.ooo.oooHeadChunk.OverlapsClosedInterval(mint, maxt) {
-		// We only want to append the head chunk if this chunk existed when
-		// Series() was called. This brings consistency in case new data
-		// is added in between Series() and Chunk() calls.
-		if oooHeadRef == meta.OOOLastRef {
-			tmpChks = append(tmpChks, chunkMetaAndChunkDiskMapperRef{
-				meta: chunks.Meta{
-					// Ignoring samples added before and after the last known min and max time for this chunk.
-					MinTime: meta.OOOLastMinTime,
-					MaxTime: meta.OOOLastMaxTime,
-					Ref:     oooHeadRef,
-				},
-			})
-		}
-	}
+	tmpChks := make([]chunkMetaAndChunkDiskMapperRef, 0, len(s.ooo.oooMmappedChunks)+1)
 
 	for i, c := range s.ooo.oooMmappedChunks {
-		chunkRef := chunks.ChunkRef(chunks.NewHeadChunkRef(s.ref, s.oooHeadChunkID(i)))
-		// We can skip chunks that came in later than the last known OOOLastRef.
-		if chunkRef > meta.OOOLastRef {
-			break
-		}
-
-		switch {
-		case chunkRef == meta.OOOLastRef:
-			tmpChks = append(tmpChks, chunkMetaAndChunkDiskMapperRef{
-				meta: chunks.Meta{
-					MinTime: meta.OOOLastMinTime,
-					MaxTime: meta.OOOLastMaxTime,
-					Ref:     chunkRef,
-				},
-				ref:      c.ref,
-				origMinT: c.minTime,
-				origMaxT: c.maxTime,
-			})
-		case c.OverlapsClosedInterval(mint, maxt):
+		if c.OverlapsClosedInterval(mint, maxt) {
 			tmpChks = append(tmpChks, chunkMetaAndChunkDiskMapperRef{
 				meta: chunks.Meta{
 					MinTime: c.minTime,
 					MaxTime: c.maxTime,
-					Ref:     chunkRef,
+					Ref:     chunks.ChunkRef(chunks.NewHeadChunkRef(s.ref, s.oooHeadChunkID(i))),
 				},
 				ref: c.ref,
 			})
 		}
 	}
+	// Add in data copied from the head OOO chunk.
+	if meta.Chunk != nil {
+		tmpChks = append(tmpChks, chunkMetaAndChunkDiskMapperRef{meta: meta})
+	}
 
 	// Next we want to sort all the collected chunks by min time so we can find
 	// those that overlap and stop when we know the rest don't.
@@ -548,22 +517,8 @@ func (s *memSeries) oooMergedChunks(meta chunks.Meta, cdm *chunks.ChunkDiskMappe
 			continue
 		}
 		var iterable chunkenc.Iterable
-		if c.meta.Ref == oooHeadRef {
-			var xor *chunkenc.XORChunk
-			var err error
-			// If head chunk min and max time match the meta OOO markers
-			// that means that the chunk has not expanded so we can append
-			// it as it is.
-			if s.ooo.oooHeadChunk.minTime == meta.OOOLastMinTime && s.ooo.oooHeadChunk.maxTime == meta.OOOLastMaxTime {
-				xor, err = s.ooo.oooHeadChunk.chunk.ToXOR() // TODO(jesus.vazquez) (This is an optimization idea that has no priority and might not be that useful) See if we could use a copy of the underlying slice. That would leave the more expensive ToXOR() function only for the usecase where Bytes() is called.
-			} else {
-				// We need to remove samples that are outside of the markers
-				xor, err = s.ooo.oooHeadChunk.chunk.ToXORBetweenTimestamps(meta.OOOLastMinTime, meta.OOOLastMaxTime)
-			}
-			if err != nil {
-				return nil, fmt.Errorf("failed to convert ooo head chunk to xor chunk: %w", err)
-			}
-			iterable = xor
+		if c.meta.Chunk != nil {
+			iterable = c.meta.Chunk
 		} else {
 			chk, err := cdm.Chunk(c.ref)
 			if err != nil {
@@ -573,16 +528,7 @@ func (s *memSeries) oooMergedChunks(meta chunks.Meta, cdm *chunks.ChunkDiskMappe
 				}
 				return nil, err
 			}
-			if c.meta.Ref == meta.OOOLastRef &&
-				(c.origMinT != meta.OOOLastMinTime || c.origMaxT != meta.OOOLastMaxTime) {
-				// The head expanded and was memory mapped so now we need to
-				// wrap the chunk within a chunk that doesnt allows us to iterate
-				// through samples out of the OOOLastMinT and OOOLastMaxT
-				// markers.
-				iterable = boundedIterable{chk, meta.OOOLastMinTime, meta.OOOLastMaxTime}
-			} else {
-				iterable = chk
-			}
+			iterable = chk
 		}
 		mc.chunkIterables = append(mc.chunkIterables, iterable)
 		if c.meta.MaxTime > absoluteMax {
@@ -593,74 +539,6 @@ func (s *memSeries) oooMergedChunks(meta chunks.Meta, cdm *chunks.ChunkDiskMappe
 	return mc, nil
 }
 
-var _ chunkenc.Iterable = &boundedIterable{}
-
-// boundedIterable is an implementation of chunkenc.Iterable that uses a
-// boundedIterator that only iterates through samples which timestamps are
-// >= minT and <= maxT.
-type boundedIterable struct {
-	chunk chunkenc.Chunk
-	minT  int64
-	maxT  int64
-}
-
-func (b boundedIterable) Iterator(iterator chunkenc.Iterator) chunkenc.Iterator {
-	it := b.chunk.Iterator(iterator)
-	if it == nil {
-		panic("iterator shouldn't be nil")
-	}
-	return boundedIterator{it, b.minT, b.maxT}
-}
-
-var _ chunkenc.Iterator = &boundedIterator{}
-
-// boundedIterator is an implementation of Iterator that only iterates through
-// samples which timestamps are >= minT and <= maxT.
-type boundedIterator struct {
-	chunkenc.Iterator
-	minT int64
-	maxT int64
-}
-
-// Next the first time its called it will advance as many positions as necessary
-// until its able to find a sample within the bounds minT and maxT.
-// If there are samples within bounds it will advance one by one amongst them.
-// If there are no samples within bounds it will return false.
-func (b boundedIterator) Next() chunkenc.ValueType {
-	for b.Iterator.Next() == chunkenc.ValFloat {
-		t, _ := b.Iterator.At()
-		switch {
-		case t < b.minT:
-			continue
-		case t > b.maxT:
-			return chunkenc.ValNone
-		default:
-			return chunkenc.ValFloat
-		}
-	}
-	return chunkenc.ValNone
-}
-
-func (b boundedIterator) Seek(t int64) chunkenc.ValueType {
-	if t < b.minT {
-		// We must seek at least up to b.minT if it is asked for something before that.
-		val := b.Iterator.Seek(b.minT)
-		if !(val == chunkenc.ValFloat) {
-			return chunkenc.ValNone
-		}
-		t, _ := b.Iterator.At()
-		if t <= b.maxT {
-			return chunkenc.ValFloat
-		}
-	}
-	if t > b.maxT {
-		// We seek anyway so that the subsequent Next() calls will also return false.
-		b.Iterator.Seek(t)
-		return chunkenc.ValNone
-	}
-	return b.Iterator.Seek(t)
-}
-
 // safeHeadChunk makes sure that the chunk can be accessed without a race condition.
 type safeHeadChunk struct {
 	chunkenc.Chunk

tsdb/head_read_test.go

@@ -15,7 +15,6 @@ package tsdb
 
 import (
 	"context"
-	"fmt"
 	"sync"
 	"testing"
 
@@ -26,150 +25,6 @@ import (
 	"github.com/prometheus/prometheus/tsdb/chunks"
 )
 
-func TestBoundedChunk(t *testing.T) {
-	tests := []struct {
-		name           string
-		inputChunk     chunkenc.Chunk
-		inputMinT      int64
-		inputMaxT      int64
-		initialSeek    int64
-		seekIsASuccess bool
-		expSamples     []sample
-	}{
-		{
-			name:       "if there are no samples it returns nothing",
-			inputChunk: newTestChunk(0),
-			expSamples: nil,
-		},
-		{
-			name:       "bounds represent a single sample",
-			inputChunk: newTestChunk(10),
-			expSamples: []sample{
-				{0, 0, nil, nil},
-			},
-		},
-		{
-			name:       "if there are bounds set only samples within them are returned",
-			inputChunk: newTestChunk(10),
-			inputMinT:  1,
-			inputMaxT:  8,
-			expSamples: []sample{
-				{1, 1, nil, nil},
-				{2, 2, nil, nil},
-				{3, 3, nil, nil},
-				{4, 4, nil, nil},
-				{5, 5, nil, nil},
-				{6, 6, nil, nil},
-				{7, 7, nil, nil},
-				{8, 8, nil, nil},
-			},
-		},
-		{
-			name:       "if bounds set and only maxt is less than actual maxt",
-			inputChunk: newTestChunk(10),
-			inputMinT:  0,
-			inputMaxT:  5,
-			expSamples: []sample{
-				{0, 0, nil, nil},
-				{1, 1, nil, nil},
-				{2, 2, nil, nil},
-				{3, 3, nil, nil},
-				{4, 4, nil, nil},
-				{5, 5, nil, nil},
-			},
-		},
-		{
-			name:       "if bounds set and only mint is more than actual mint",
-			inputChunk: newTestChunk(10),
-			inputMinT:  5,
-			inputMaxT:  9,
-			expSamples: []sample{
-				{5, 5, nil, nil},
-				{6, 6, nil, nil},
-				{7, 7, nil, nil},
-				{8, 8, nil, nil},
-				{9, 9, nil, nil},
-			},
-		},
-		{
-			name:           "if there are bounds set with seek before mint",
-			inputChunk:     newTestChunk(10),
-			inputMinT:      3,
-			inputMaxT:      7,
-			initialSeek:    1,
-			seekIsASuccess: true,
-			expSamples: []sample{
-				{3, 3, nil, nil},
-				{4, 4, nil, nil},
-				{5, 5, nil, nil},
-				{6, 6, nil, nil},
-				{7, 7, nil, nil},
-			},
-		},
-		{
-			name:           "if there are bounds set with seek between mint and maxt",
-			inputChunk:     newTestChunk(10),
-			inputMinT:      3,
-			inputMaxT:      7,
-			initialSeek:    5,
-			seekIsASuccess: true,
-			expSamples: []sample{
-				{5, 5, nil, nil},
-				{6, 6, nil, nil},
-				{7, 7, nil, nil},
-			},
-		},
-		{
-			name:           "if there are bounds set with seek after maxt",
-			inputChunk:     newTestChunk(10),
-			inputMinT:      3,
-			inputMaxT:      7,
-			initialSeek:    8,
-			seekIsASuccess: false,
-		},
-	}
-	for _, tc := range tests {
-		t.Run(fmt.Sprintf("name=%s", tc.name), func(t *testing.T) {
-			iterable := boundedIterable{tc.inputChunk, tc.inputMinT, tc.inputMaxT}
-
-			var samples []sample
-			it := iterable.Iterator(nil)
-
-			if tc.initialSeek != 0 {
-				// Testing Seek()
-				val := it.Seek(tc.initialSeek)
-				require.Equal(t, tc.seekIsASuccess, val == chunkenc.ValFloat)
-				if val == chunkenc.ValFloat {
-					t, v := it.At()
-					samples = append(samples, sample{t, v, nil, nil})
-				}
-			}
-
-			// Testing Next()
-			for it.Next() == chunkenc.ValFloat {
-				t, v := it.At()
-				samples = append(samples, sample{t, v, nil, nil})
-			}
-
-			// it.Next() should keep returning no  value.
-			for i := 0; i < 10; i++ {
-				require.Equal(t, chunkenc.ValNone, it.Next())
-			}
-
-			require.Equal(t, tc.expSamples, samples)
-		})
-	}
-}
-
-func newTestChunk(numSamples int) chunkenc.Chunk {
-	xor := chunkenc.NewXORChunk()
-	a, _ := xor.Appender()
-	for i := 0; i < numSamples; i++ {
-		a.Append(int64(i), float64(i))
-	}
-	return xor
-}
-
 // TestMemSeries_chunk runs a series of tests on memSeries.chunk() calls.
 // It will simulate various conditions to ensure all code paths in that function are covered.
 func TestMemSeries_chunk(t *testing.T) {

tsdb/head_test.go

@@ -2665,6 +2665,14 @@ func TestIsolationWithoutAdd(t *testing.T) {
 }
 
 func TestOutOfOrderSamplesMetric(t *testing.T) {
+	for name, scenario := range sampleTypeScenarios {
+		t.Run(name, func(t *testing.T) {
+			testOutOfOrderSamplesMetric(t, scenario)
+		})
+	}
+}
+
+func testOutOfOrderSamplesMetric(t *testing.T, scenario sampleTypeScenario) {
 	dir := t.TempDir()
 
 	db, err := Open(dir, nil, nil, DefaultOptions(), nil)
@@ -2674,33 +2682,38 @@ func TestOutOfOrderSamplesMetric(t *testing.T) {
 	}()
 	db.DisableCompactions()
 
+	appendSample := func(appender storage.Appender, ts int64) (storage.SeriesRef, error) {
+		ref, _, err := scenario.appendFunc(appender, labels.FromStrings("a", "b"), ts, 99)
+		return ref, err
+	}
+
 	ctx := context.Background()
 	app := db.Appender(ctx)
 	for i := 1; i <= 5; i++ {
-		_, err = app.Append(0, labels.FromStrings("a", "b"), int64(i), 99)
+		_, err = appendSample(app, int64(i))
 		require.NoError(t, err)
 	}
 	require.NoError(t, app.Commit())
 
 	// Test out of order metric.
-	require.Equal(t, 0.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat)))
+	require.Equal(t, 0.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType)))
 	app = db.Appender(ctx)
-	_, err = app.Append(0, labels.FromStrings("a", "b"), 2, 99)
+	_, err = appendSample(app, 2)
 	require.Equal(t, storage.ErrOutOfOrderSample, err)
-	require.Equal(t, 1.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat)))
+	require.Equal(t, 1.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType)))
 
-	_, err = app.Append(0, labels.FromStrings("a", "b"), 3, 99)
+	_, err = appendSample(app, 3)
 	require.Equal(t, storage.ErrOutOfOrderSample, err)
-	require.Equal(t, 2.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat)))
+	require.Equal(t, 2.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType)))
 
-	_, err = app.Append(0, labels.FromStrings("a", "b"), 4, 99)
+	_, err = appendSample(app, 4)
 	require.Equal(t, storage.ErrOutOfOrderSample, err)
-	require.Equal(t, 3.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat)))
+	require.Equal(t, 3.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType)))
 	require.NoError(t, app.Commit())
 
 	// Compact Head to test out of bound metric.
 	app = db.Appender(ctx)
-	_, err = app.Append(0, labels.FromStrings("a", "b"), DefaultBlockDuration*2, 99)
+	_, err = appendSample(app, DefaultBlockDuration*2)
 	require.NoError(t, err)
 	require.NoError(t, app.Commit())
 
@@ -2709,36 +2722,36 @@ func TestOutOfOrderSamplesMetric(t *testing.T) {
 	require.Greater(t, db.head.minValidTime.Load(), int64(0))
 
 	app = db.Appender(ctx)
-	_, err = app.Append(0, labels.FromStrings("a", "b"), db.head.minValidTime.Load()-2, 99)
+	_, err = appendSample(app, db.head.minValidTime.Load()-2)
 	require.Equal(t, storage.ErrOutOfBounds, err)
-	require.Equal(t, 1.0, prom_testutil.ToFloat64(db.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeFloat)))
+	require.Equal(t, 1.0, prom_testutil.ToFloat64(db.head.metrics.outOfBoundSamples.WithLabelValues(scenario.sampleType)))
 
-	_, err = app.Append(0, labels.FromStrings("a", "b"), db.head.minValidTime.Load()-1, 99)
+	_, err = appendSample(app, db.head.minValidTime.Load()-1)
 	require.Equal(t, storage.ErrOutOfBounds, err)
-	require.Equal(t, 2.0, prom_testutil.ToFloat64(db.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeFloat)))
+	require.Equal(t, 2.0, prom_testutil.ToFloat64(db.head.metrics.outOfBoundSamples.WithLabelValues(scenario.sampleType)))
 	require.NoError(t, app.Commit())
 
 	// Some more valid samples for out of order.
 	app = db.Appender(ctx)
 	for i := 1; i <= 5; i++ {
-		_, err = app.Append(0, labels.FromStrings("a", "b"), db.head.minValidTime.Load()+DefaultBlockDuration+int64(i), 99)
+		_, err = appendSample(app, db.head.minValidTime.Load()+DefaultBlockDuration+int64(i))
 		require.NoError(t, err)
 	}
 	require.NoError(t, app.Commit())
 
 	// Test out of order metric.
 	app = db.Appender(ctx)
-	_, err = app.Append(0, labels.FromStrings("a", "b"), db.head.minValidTime.Load()+DefaultBlockDuration+2, 99)
+	_, err = appendSample(app, db.head.minValidTime.Load()+DefaultBlockDuration+2)
 	require.Equal(t, storage.ErrOutOfOrderSample, err)
-	require.Equal(t, 4.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat)))
+	require.Equal(t, 4.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType)))
 
-	_, err = app.Append(0, labels.FromStrings("a", "b"), db.head.minValidTime.Load()+DefaultBlockDuration+3, 99)
+	_, err = appendSample(app, db.head.minValidTime.Load()+DefaultBlockDuration+3)
 	require.Equal(t, storage.ErrOutOfOrderSample, err)
-	require.Equal(t, 5.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat)))
+	require.Equal(t, 5.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType)))
 
-	_, err = app.Append(0, labels.FromStrings("a", "b"), db.head.minValidTime.Load()+DefaultBlockDuration+4, 99)
+	_, err = appendSample(app, db.head.minValidTime.Load()+DefaultBlockDuration+4)
 	require.Equal(t, storage.ErrOutOfOrderSample, err)
-	require.Equal(t, 6.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat)))
+	require.Equal(t, 6.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType)))
 	require.NoError(t, app.Commit())
 }
 
@@ -4801,6 +4814,14 @@ func TestWBLReplay(t *testing.T) {
 
 // TestOOOMmapReplay checks the replay at a low level.
 func TestOOOMmapReplay(t *testing.T) {
+	for name, scenario := range sampleTypeScenarios {
+		t.Run(name, func(t *testing.T) {
+			testOOOMmapReplay(t, scenario)
+		})
+	}
+}
+
+func testOOOMmapReplay(t *testing.T, scenario sampleTypeScenario) {
 	dir := t.TempDir()
 	wal, err := wlog.NewSize(nil, nil, filepath.Join(dir, "wal"), 32768, wlog.CompressionSnappy)
 	require.NoError(t, err)
@@ -4820,8 +4841,7 @@ func TestOOOMmapReplay(t *testing.T) {
 	l := labels.FromStrings("foo", "bar")
 	appendSample := func(mins int64) {
 		app := h.Appender(context.Background())
-		ts, v := mins*time.Minute.Milliseconds(), float64(mins)
-		_, err := app.Append(0, l, ts, v)
+		_, _, err := scenario.appendFunc(app, l, mins*time.Minute.Milliseconds(), mins)
 		require.NoError(t, err)
 		require.NoError(t, app.Commit())
 	}
@@ -5096,6 +5116,14 @@ func TestReplayAfterMmapReplayError(t *testing.T) {
 }
 
 func TestOOOAppendWithNoSeries(t *testing.T) {
+	for name, scenario := range sampleTypeScenarios {
+		t.Run(name, func(t *testing.T) {
+			testOOOAppendWithNoSeries(t, scenario.appendFunc)
+		})
+	}
+}
+
+func testOOOAppendWithNoSeries(t *testing.T, appendFunc func(appender storage.Appender, lbls labels.Labels, ts, value int64) (storage.SeriesRef, sample, error)) {
 	dir := t.TempDir()
 	wal, err := wlog.NewSize(nil, nil, filepath.Join(dir, "wal"), 32768, wlog.CompressionSnappy)
 	require.NoError(t, err)
@@ -5116,7 +5144,7 @@ func TestOOOAppendWithNoSeries(t *testing.T) {
 
 	appendSample := func(lbls labels.Labels, ts int64) {
 		app := h.Appender(context.Background())
-		_, err := app.Append(0, lbls, ts*time.Minute.Milliseconds(), float64(ts))
+		_, _, err := appendFunc(app, lbls, ts*time.Minute.Milliseconds(), ts)
 		require.NoError(t, err)
 		require.NoError(t, app.Commit())
 	}
@@ -5164,7 +5192,7 @@ func TestOOOAppendWithNoSeries(t *testing.T) {
 	// Now 179m is too old.
 	s4 := newLabels(4)
 	app := h.Appender(context.Background())
-	_, err = app.Append(0, s4, 179*time.Minute.Milliseconds(), float64(179))
+	_, _, err = appendFunc(app, s4, 179*time.Minute.Milliseconds(), 179)
 	require.Equal(t, storage.ErrTooOldSample, err)
 	require.NoError(t, app.Rollback())
 	verifyOOOSamples(s3, 1)
@@ -5177,6 +5205,14 @@ func TestOOOAppendWithNoSeries(t *testing.T) {
 }
 
 func TestHeadMinOOOTimeUpdate(t *testing.T) {
+	for name, scenario := range sampleTypeScenarios {
+		t.Run(name, func(t *testing.T) {
+			testHeadMinOOOTimeUpdate(t, scenario)
+		})
+	}
+}
+
+func testHeadMinOOOTimeUpdate(t *testing.T, scenario sampleTypeScenario) {
 	dir := t.TempDir()
 	wal, err := wlog.NewSize(nil, nil, filepath.Join(dir, "wal"), 32768, wlog.CompressionSnappy)
 	require.NoError(t, err)
@@ -5195,15 +5231,13 @@ func TestHeadMinOOOTimeUpdate(t *testing.T) {
 	require.NoError(t, h.Init(0))
 
 	appendSample := func(ts int64) {
-		lbls := labels.FromStrings("foo", "bar")
 		app := h.Appender(context.Background())
-		_, err := app.Append(0, lbls, ts*time.Minute.Milliseconds(), float64(ts))
+		_, _, err = scenario.appendFunc(app, labels.FromStrings("a", "b"), ts*time.Minute.Milliseconds(), ts)
 		require.NoError(t, err)
 		require.NoError(t, app.Commit())
 	}
 
 	appendSample(300) // In-order sample.
-
 	require.Equal(t, int64(math.MaxInt64), h.MinOOOTime())
 
 	appendSample(295) // OOO sample.

tsdb/head_wal.go

@@ -126,7 +126,7 @@ func (h *Head) loadWAL(r *wlog.Reader, syms *labels.SymbolTable, multiRef map[ch
 			}
 			// At the moment the only possible error here is out of order exemplars, which we shouldn't see when
 			// replaying the WAL, so lets just log the error if it's not that type.
-			err = h.exemplars.AddExemplar(ms.lset, exemplar.Exemplar{Ts: e.T, Value: e.V, Labels: e.Labels})
+			err = h.exemplars.AddExemplar(ms.labels(), exemplar.Exemplar{Ts: e.T, Value: e.V, Labels: e.Labels})
 			if err != nil && errors.Is(err, storage.ErrOutOfOrderExemplar) {
 				level.Warn(h.logger).Log("msg", "Unexpected error when replaying WAL on exemplar record", "err", err)
 			}
@@ -448,7 +448,7 @@ func (h *Head) resetSeriesWithMMappedChunks(mSeries *memSeries, mmc, oooMmc []*m
 			) {
 				level.Debug(h.logger).Log(
 					"msg", "M-mapped chunks overlap on a duplicate series record",
-					"series", mSeries.lset.String(),
+					"series", mSeries.labels().String(),
 					"oldref", mSeries.ref,
 					"oldmint", mSeries.mmappedChunks[0].minTime,
 					"oldmaxt", mSeries.mmappedChunks[len(mSeries.mmappedChunks)-1].maxTime,
@@ -932,7 +932,7 @@ func (s *memSeries) encodeToSnapshotRecord(b []byte) []byte {
 
 	buf.PutByte(chunkSnapshotRecordTypeSeries)
 	buf.PutBE64(uint64(s.ref))
-	record.EncodeLabels(&buf, s.lset)
+	record.EncodeLabels(&buf, s.labels())
 	buf.PutBE64int64(0) // Backwards-compatibility; was chunkRange but now unused.
 
 	s.Lock()
@@ -1485,7 +1485,7 @@ Outer:
 					continue
 				}
 
-				if err := h.exemplars.AddExemplar(ms.lset, exemplar.Exemplar{
+				if err := h.exemplars.AddExemplar(ms.labels(), exemplar.Exemplar{
 					Labels: e.Labels,
 					Value:  e.V,
 					Ts:     e.T,

tsdb/ooo_head_read.go

@@ -78,7 +78,7 @@ func (oh *OOOHeadIndexReader) series(ref storage.SeriesRef, builder *labels.Scra
 		oh.head.metrics.seriesNotFound.Inc()
 		return storage.ErrNotFound
 	}
-	builder.Assign(s.lset)
+	builder.Assign(s.labels())
 
 	if chks == nil {
 		return nil
@@ -94,48 +94,32 @@ func (oh *OOOHeadIndexReader) series(ref storage.SeriesRef, builder *labels.Scra
 
 	tmpChks := make([]chunks.Meta, 0, len(s.ooo.oooMmappedChunks))
 
-	// We define these markers to track the last chunk reference while we
-	// fill the chunk meta.
-	// These markers are useful to give consistent responses to repeated queries
-	// even if new chunks that might be overlapping or not are added afterwards.
-	// Also, lastMinT and lastMaxT are initialized to the max int as a sentinel
-	// value to know they are unset.
-	var lastChunkRef chunks.ChunkRef
-	lastMinT, lastMaxT := int64(math.MaxInt64), int64(math.MaxInt64)
-
-	addChunk := func(minT, maxT int64, ref chunks.ChunkRef) {
-		// the first time we get called is for the last included chunk.
-		// set the markers accordingly
-		if lastMinT == int64(math.MaxInt64) {
-			lastChunkRef = ref
-			lastMinT = minT
-			lastMaxT = maxT
-		}
-
+	addChunk := func(minT, maxT int64, ref chunks.ChunkRef, chunk chunkenc.Chunk) {
 		tmpChks = append(tmpChks, chunks.Meta{
-			MinTime:        minT,
-			MaxTime:        maxT,
-			Ref:            ref,
-			OOOLastRef:     lastChunkRef,
-			OOOLastMinTime: lastMinT,
-			OOOLastMaxTime: lastMaxT,
+			MinTime: minT,
+			MaxTime: maxT,
+			Ref:     ref,
+			Chunk:   chunk,
 		})
 	}
 
-	// Collect all chunks that overlap the query range, in order from most recent to most old,
-	// so we can set the correct markers.
+	// Collect all chunks that overlap the query range.
 	if s.ooo.oooHeadChunk != nil {
 		c := s.ooo.oooHeadChunk
 		if c.OverlapsClosedInterval(oh.mint, oh.maxt) && maxMmapRef == 0 {
 			ref := chunks.ChunkRef(chunks.NewHeadChunkRef(s.ref, s.oooHeadChunkID(len(s.ooo.oooMmappedChunks))))
-			addChunk(c.minTime, c.maxTime, ref)
+			var xor chunkenc.Chunk
+			if len(c.chunk.samples) > 0 { // Empty samples happens in tests, at least.
+				xor, _ = c.chunk.ToXOR() // Ignoring error because it can't fail.
+			}
+			addChunk(c.minTime, c.maxTime, ref, xor)
 		}
 	}
 	for i := len(s.ooo.oooMmappedChunks) - 1; i >= 0; i-- {
 		c := s.ooo.oooMmappedChunks[i]
 		if c.OverlapsClosedInterval(oh.mint, oh.maxt) && (maxMmapRef == 0 || maxMmapRef.GreaterThanOrEqualTo(c.ref)) && (lastGarbageCollectedMmapRef == 0 || c.ref.GreaterThan(lastGarbageCollectedMmapRef)) {
 			ref := chunks.ChunkRef(chunks.NewHeadChunkRef(s.ref, s.oooHeadChunkID(i)))
-			addChunk(c.minTime, c.maxTime, ref)
+			addChunk(c.minTime, c.maxTime, ref, nil)
 		}
 	}
 
@@ -163,6 +147,12 @@ func (oh *OOOHeadIndexReader) series(ref storage.SeriesRef, builder *labels.Scra
 		case c.MaxTime > maxTime:
 			maxTime = c.MaxTime
 			(*chks)[len(*chks)-1].MaxTime = c.MaxTime
+			fallthrough
+		default:
+			// If the head OOO chunk is part of an output chunk, copy the chunk pointer.
+			if c.Chunk != nil {
+				(*chks)[len(*chks)-1].Chunk = c.Chunk
+			}
 		}
 	}
 
@@ -185,10 +175,8 @@ func (oh *OOOHeadIndexReader) LabelValues(ctx context.Context, name string, matc
 }
 
 type chunkMetaAndChunkDiskMapperRef struct {
-	meta     chunks.Meta
-	ref      chunks.ChunkDiskMapperRef
-	origMinT int64
-	origMaxT int64
+	meta chunks.Meta
+	ref  chunks.ChunkDiskMapperRef
 }
 
 func refLessByMinTimeAndMinRef(a, b chunkMetaAndChunkDiskMapperRef) int {

tsdb/ooo_head_read_test.go

@@ -304,18 +304,6 @@ func TestOOOHeadIndexReader_Series(t *testing.T) {
 					s1, _, _ := h.getOrCreate(s1ID, s1Lset)
 					s1.ooo = &memSeriesOOOFields{}
 
-					var lastChunk chunkInterval
-					var lastChunkPos int
-
-					// the marker should be set based on whichever is the last chunk/interval that overlaps with the query range
-					for i, interv := range intervals {
-						if overlapsClosedInterval(interv.mint, interv.maxt, tc.queryMinT, tc.queryMaxT) {
-							lastChunk = interv
-							lastChunkPos = i
-						}
-					}
-					lastChunkRef := chunks.ChunkRef(chunks.NewHeadChunkRef(1, chunks.HeadChunkID(uint64(lastChunkPos))))
-
 					// define our expected chunks, by looking at the expected ChunkIntervals and setting...
 					var expChunks []chunks.Meta
 					for _, e := range tc.expChunks {
@@ -323,10 +311,6 @@ func TestOOOHeadIndexReader_Series(t *testing.T) {
 							Chunk:   chunkenc.Chunk(nil),
 							MinTime: e.mint,
 							MaxTime: e.maxt,
-							// markers based on the last chunk we found above
-							OOOLastMinTime: lastChunk.mint,
-							OOOLastMaxTime: lastChunk.maxt,
-							OOOLastRef:     lastChunkRef,
 						}
 
 						// Ref to whatever Ref the chunk has, that we refer to by ID
@@ -343,6 +327,7 @@ func TestOOOHeadIndexReader_Series(t *testing.T) {
 					if headChunk && len(intervals) > 0 {
 						// Put the last interval in the head chunk
 						s1.ooo.oooHeadChunk = &oooHeadChunk{
+							chunk:   NewOOOChunk(),
 							minTime: intervals[len(intervals)-1].mint,
 							maxTime: intervals[len(intervals)-1].maxt,
 						}
@@ -374,6 +359,15 @@ func TestOOOHeadIndexReader_Series(t *testing.T) {
 }
 
 func TestOOOHeadChunkReader_LabelValues(t *testing.T) {
+	for name, scenario := range sampleTypeScenarios {
+		t.Run(name, func(t *testing.T) {
+			testOOOHeadChunkReader_LabelValues(t, scenario)
+		})
+	}
+}
+
+//nolint:revive // unexported-return.
+func testOOOHeadChunkReader_LabelValues(t *testing.T, scenario sampleTypeScenario) {
 	chunkRange := int64(2000)
 	head, _ := newTestHead(t, chunkRange, wlog.CompressionNone, true)
 	t.Cleanup(func() { require.NoError(t, head.Close()) })
@@ -383,15 +377,15 @@ func TestOOOHeadChunkReader_LabelValues(t *testing.T) {
 	app := head.Appender(context.Background())
 
 	// Add in-order samples
-	_, err := app.Append(0, labels.FromStrings("foo", "bar1"), 100, 1)
+	_, _, err := scenario.appendFunc(app, labels.FromStrings("foo", "bar1"), 100, int64(1))
 	require.NoError(t, err)
-	_, err = app.Append(0, labels.FromStrings("foo", "bar2"), 100, 2)
+	_, _, err = scenario.appendFunc(app, labels.FromStrings("foo", "bar2"), 100, int64(2))
 	require.NoError(t, err)
 
 	// Add ooo samples for those series
-	_, err = app.Append(0, labels.FromStrings("foo", "bar1"), 90, 1)
+	_, _, err = scenario.appendFunc(app, labels.FromStrings("foo", "bar1"), 90, int64(1))
 	require.NoError(t, err)
-	_, err = app.Append(0, labels.FromStrings("foo", "bar2"), 90, 2)
+	_, _, err = scenario.appendFunc(app, labels.FromStrings("foo", "bar2"), 90, int64(2))
 	require.NoError(t, err)
 
 	require.NoError(t, app.Commit())
@@ -468,6 +462,15 @@ func TestOOOHeadChunkReader_LabelValues(t *testing.T) {
 // It does so by appending out of order samples to the db and then initializing
 // an OOOHeadChunkReader to read chunks from it.
 func TestOOOHeadChunkReader_Chunk(t *testing.T) {
+	for name, scenario := range sampleTypeScenarios {
+		t.Run(name, func(t *testing.T) {
+			testOOOHeadChunkReader_Chunk(t, scenario)
+		})
+	}
+}
+
+//nolint:revive // unexported-return.
+func testOOOHeadChunkReader_Chunk(t *testing.T, scenario sampleTypeScenario) {
 	opts := DefaultOptions()
 	opts.OutOfOrderCapMax = 5
 	opts.OutOfOrderTimeWindow = 120 * time.Minute.Milliseconds()
@@ -475,12 +478,6 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) {
 	s1 := labels.FromStrings("l", "v1")
 	minutes := func(m int64) int64 { return m * time.Minute.Milliseconds() }
 
-	appendSample := func(app storage.Appender, l labels.Labels, timestamp int64, value float64) storage.SeriesRef {
-		ref, err := app.Append(0, l, timestamp, value)
-		require.NoError(t, err)
-		return ref
-	}
-
 	t.Run("Getting a non existing chunk fails with not found error", func(t *testing.T) {
 		db := newTestDBWithOpts(t, opts)
 
@@ -499,7 +496,7 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) {
 		queryMinT            int64
 		queryMaxT            int64
 		firstInOrderSampleAt int64
-		inputSamples         chunks.SampleSlice
+		inputSamples         []testValue
 		expChunkError        bool
 		expChunksSamples     []chunks.SampleSlice
 	}{
@@ -508,9 +505,9 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) {
 			queryMinT:            minutes(0),
 			queryMaxT:            minutes(100),
 			firstInOrderSampleAt: minutes(120),
-			inputSamples: chunks.SampleSlice{
-				sample{t: minutes(30), f: float64(0)},
-				sample{t: minutes(40), f: float64(0)},
+			inputSamples: []testValue{
+				{Ts: minutes(30), V: 0},
+				{Ts: minutes(40), V: 0},
 			},
 			expChunkError: false,
 			// ts (in minutes)         0       10       20       30       40       50       60       70       80       90       100
@@ -519,8 +516,8 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) {
 			// Output Graphically                                 [--------] (With 2 samples)
 			expChunksSamples: []chunks.SampleSlice{
 				{
-					sample{t: minutes(30), f: float64(0)},
-					sample{t: minutes(40), f: float64(0)},
+					scenario.sampleFunc(minutes(30), 0),
+					scenario.sampleFunc(minutes(40), 0),
 				},
 			},
 		},
@@ -529,19 +526,8 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) {
 			queryMinT:            minutes(0),
 			queryMaxT:            minutes(100),
 			firstInOrderSampleAt: minutes(120),
-			inputSamples: chunks.SampleSlice{
-				// opts.OOOCapMax is 5 so these will be mmapped to the first mmapped chunk
-				sample{t: minutes(41), f: float64(0)},
-				sample{t: minutes(42), f: float64(0)},
-				sample{t: minutes(43), f: float64(0)},
-				sample{t: minutes(44), f: float64(0)},
-				sample{t: minutes(45), f: float64(0)},
-				// The following samples will go to the head chunk, and we want it
-				// to overlap with the previous chunk
-				sample{t: minutes(30), f: float64(1)},
-				sample{t: minutes(50), f: float64(1)},
-			},
-			expChunkError: false,
+			inputSamples:         []testValue{{Ts: minutes(41), V: 0}, {Ts: minutes(42), V: 0}, {Ts: minutes(43), V: 0}, {Ts: minutes(44), V: 0}, {Ts: minutes(45), V: 0}, {Ts: minutes(30), V: 1}, {Ts: minutes(50), V: 1}},
+			expChunkError:        false,
 			// ts (in minutes)         0       10       20       30       40       50       60       70       80       90       100
 			// Query Interval          [------------------------------------------------------------------------------------------]
 			// Chunk 0                                                     [---] (With 5 samples)
@@ -549,13 +535,13 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) {
 			// Output Graphically                                 [-----------------] (With 7 samples)
 			expChunksSamples: []chunks.SampleSlice{
 				{
-					sample{t: minutes(30), f: float64(1)},
-					sample{t: minutes(41), f: float64(0)},
-					sample{t: minutes(42), f: float64(0)},
-					sample{t: minutes(43), f: float64(0)},
-					sample{t: minutes(44), f: float64(0)},
-					sample{t: minutes(45), f: float64(0)},
-					sample{t: minutes(50), f: float64(1)},
+					scenario.sampleFunc(minutes(30), 1),
+					scenario.sampleFunc(minutes(41), 0),
+					scenario.sampleFunc(minutes(42), 0),
+					scenario.sampleFunc(minutes(43), 0),
+					scenario.sampleFunc(minutes(44), 0),
+					scenario.sampleFunc(minutes(45), 0),
+					scenario.sampleFunc(minutes(50), 1),
 				},
 			},
 		},
@@ -564,28 +550,28 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) {
 			queryMinT:            minutes(0),
 			queryMaxT:            minutes(100),
 			firstInOrderSampleAt: minutes(120),
-			inputSamples: chunks.SampleSlice{
+			inputSamples: []testValue{
 				// Chunk 0
-				sample{t: minutes(10), f: float64(0)},
-				sample{t: minutes(12), f: float64(0)},
-				sample{t: minutes(14), f: float64(0)},
-				sample{t: minutes(16), f: float64(0)},
-				sample{t: minutes(20), f: float64(0)},
+				{Ts: minutes(10), V: 0},
+				{Ts: minutes(12), V: 0},
+				{Ts: minutes(14), V: 0},
+				{Ts: minutes(16), V: 0},
+				{Ts: minutes(20), V: 0},
 				// Chunk 1
-				sample{t: minutes(20), f: float64(1)},
-				sample{t: minutes(22), f: float64(1)},
-				sample{t: minutes(24), f: float64(1)},
-				sample{t: minutes(26), f: float64(1)},
-				sample{t: minutes(29), f: float64(1)},
-				// Chunk 2
-				sample{t: minutes(30), f: float64(2)},
-				sample{t: minutes(32), f: float64(2)},
-				sample{t: minutes(34), f: float64(2)},
-				sample{t: minutes(36), f: float64(2)},
-				sample{t: minutes(40), f: float64(2)},
+				{Ts: minutes(20), V: 1},
+				{Ts: minutes(22), V: 1},
+				{Ts: minutes(24), V: 1},
+				{Ts: minutes(26), V: 1},
+				{Ts: minutes(29), V: 1},
+				// Chunk 3
+				{Ts: minutes(30), V: 2},
+				{Ts: minutes(32), V: 2},
+				{Ts: minutes(34), V: 2},
+				{Ts: minutes(36), V: 2},
+				{Ts: minutes(40), V: 2},
 				// Head
-				sample{t: minutes(40), f: float64(3)},
-				sample{t: minutes(50), f: float64(3)},
+				{Ts: minutes(40), V: 3},
+				{Ts: minutes(50), V: 3},
 			},
 			expChunkError: false,
 			// ts (in minutes)         0       10       20       30       40       50       60       70       80       90       100
@@ -597,23 +583,23 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) {
 			// Output Graphically               [----------------][-----------------]
 			expChunksSamples: []chunks.SampleSlice{
 				{
-					sample{t: minutes(10), f: float64(0)},
-					sample{t: minutes(12), f: float64(0)},
-					sample{t: minutes(14), f: float64(0)},
-					sample{t: minutes(16), f: float64(0)},
-					sample{t: minutes(20), f: float64(1)},
-					sample{t: minutes(22), f: float64(1)},
-					sample{t: minutes(24), f: float64(1)},
-					sample{t: minutes(26), f: float64(1)},
-					sample{t: minutes(29), f: float64(1)},
+					scenario.sampleFunc(minutes(10), 0),
+					scenario.sampleFunc(minutes(12), 0),
+					scenario.sampleFunc(minutes(14), 0),
+					scenario.sampleFunc(minutes(16), 0),
+					scenario.sampleFunc(minutes(20), 1),
+					scenario.sampleFunc(minutes(22), 1),
+					scenario.sampleFunc(minutes(24), 1),
+					scenario.sampleFunc(minutes(26), 1),
+					scenario.sampleFunc(minutes(29), 1),
 				},
 				{
-					sample{t: minutes(30), f: float64(2)},
-					sample{t: minutes(32), f: float64(2)},
-					sample{t: minutes(34), f: float64(2)},
-					sample{t: minutes(36), f: float64(2)},
-					sample{t: minutes(40), f: float64(3)},
-					sample{t: minutes(50), f: float64(3)},
+					scenario.sampleFunc(minutes(30), 2),
+					scenario.sampleFunc(minutes(32), 2),
+					scenario.sampleFunc(minutes(34), 2),
+					scenario.sampleFunc(minutes(36), 2),
+					scenario.sampleFunc(minutes(40), 3),
+					scenario.sampleFunc(minutes(50), 3),
 				},
 			},
 		},
@@ -622,28 +608,28 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) {
 			queryMinT:            minutes(0),
 			queryMaxT:            minutes(100),
 			firstInOrderSampleAt: minutes(120),
-			inputSamples: chunks.SampleSlice{
+			inputSamples: []testValue{
 				// Chunk 0
-				sample{t: minutes(40), f: float64(0)},
-				sample{t: minutes(42), f: float64(0)},
-				sample{t: minutes(44), f: float64(0)},
-				sample{t: minutes(46), f: float64(0)},
-				sample{t: minutes(50), f: float64(0)},
+				{Ts: minutes(40), V: 0},
+				{Ts: minutes(42), V: 0},
+				{Ts: minutes(44), V: 0},
+				{Ts: minutes(46), V: 0},
+				{Ts: minutes(50), V: 0},
 				// Chunk 1
-				sample{t: minutes(30), f: float64(1)},
-				sample{t: minutes(32), f: float64(1)},
-				sample{t: minutes(34), f: float64(1)},
-				sample{t: minutes(36), f: float64(1)},
-				sample{t: minutes(40), f: float64(1)},
-				// Chunk 2
-				sample{t: minutes(20), f: float64(2)},
-				sample{t: minutes(22), f: float64(2)},
-				sample{t: minutes(24), f: float64(2)},
-				sample{t: minutes(26), f: float64(2)},
-				sample{t: minutes(29), f: float64(2)},
+				{Ts: minutes(30), V: 1},
+				{Ts: minutes(32), V: 1},
+				{Ts: minutes(34), V: 1},
+				{Ts: minutes(36), V: 1},
+				{Ts: minutes(40), V: 1},
+				// Chunk 3
+				{Ts: minutes(20), V: 2},
+				{Ts: minutes(22), V: 2},
+				{Ts: minutes(24), V: 2},
+				{Ts: minutes(26), V: 2},
+				{Ts: minutes(29), V: 2},
 				// Head
-				sample{t: minutes(10), f: float64(3)},
-				sample{t: minutes(20), f: float64(3)},
+				{Ts: minutes(10), V: 3},
+				{Ts: minutes(20), V: 3},
 			},
 			expChunkError: false,
 			// ts (in minutes)         0       10       20       30       40       50       60       70       80       90       100
@@ -655,23 +641,23 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) {
 			// Output Graphically               [----------------][-----------------]
 			expChunksSamples: []chunks.SampleSlice{
 				{
-					sample{t: minutes(10), f: float64(3)},
-					sample{t: minutes(20), f: float64(2)},
-					sample{t: minutes(22), f: float64(2)},
-					sample{t: minutes(24), f: float64(2)},
-					sample{t: minutes(26), f: float64(2)},
-					sample{t: minutes(29), f: float64(2)},
+					scenario.sampleFunc(minutes(10), 3),
+					scenario.sampleFunc(minutes(20), 2),
+					scenario.sampleFunc(minutes(22), 2),
+					scenario.sampleFunc(minutes(24), 2),
+					scenario.sampleFunc(minutes(26), 2),
+					scenario.sampleFunc(minutes(29), 2),
 				},
 				{
-					sample{t: minutes(30), f: float64(1)},
-					sample{t: minutes(32), f: float64(1)},
-					sample{t: minutes(34), f: float64(1)},
-					sample{t: minutes(36), f: float64(1)},
-					sample{t: minutes(40), f: float64(0)},
-					sample{t: minutes(42), f: float64(0)},
-					sample{t: minutes(44), f: float64(0)},
-					sample{t: minutes(46), f: float64(0)},
-					sample{t: minutes(50), f: float64(0)},
+					scenario.sampleFunc(minutes(30), 1),
+					scenario.sampleFunc(minutes(32), 1),
+					scenario.sampleFunc(minutes(34), 1),
+					scenario.sampleFunc(minutes(36), 1),
+					scenario.sampleFunc(minutes(40), 0),
+					scenario.sampleFunc(minutes(42), 0),
+					scenario.sampleFunc(minutes(44), 0),
+					scenario.sampleFunc(minutes(46), 0),
+					scenario.sampleFunc(minutes(50), 0),
 				},
 			},
 		},
@@ -680,28 +666,28 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) {
 			queryMinT:            minutes(0),
 			queryMaxT:            minutes(100),
 			firstInOrderSampleAt: minutes(120),
-			inputSamples: chunks.SampleSlice{
+			inputSamples: []testValue{
 				// Chunk 0
-				sample{t: minutes(10), f: float64(0)},
-				sample{t: minutes(12), f: float64(0)},
-				sample{t: minutes(14), f: float64(0)},
-				sample{t: minutes(16), f: float64(0)},
-				sample{t: minutes(18), f: float64(0)},
+				{Ts: minutes(10), V: 0},
+				{Ts: minutes(12), V: 0},
+				{Ts: minutes(14), V: 0},
+				{Ts: minutes(16), V: 0},
+				{Ts: minutes(18), V: 0},
 				// Chunk 1
-				sample{t: minutes(20), f: float64(1)},
-				sample{t: minutes(22), f: float64(1)},
-				sample{t: minutes(24), f: float64(1)},
-				sample{t: minutes(26), f: float64(1)},
-				sample{t: minutes(28), f: float64(1)},
-				// Chunk 2
-				sample{t: minutes(30), f: float64(2)},
-				sample{t: minutes(32), f: float64(2)},
-				sample{t: minutes(34), f: float64(2)},
-				sample{t: minutes(36), f: float64(2)},
-				sample{t: minutes(38), f: float64(2)},
+				{Ts: minutes(20), V: 1},
+				{Ts: minutes(22), V: 1},
+				{Ts: minutes(24), V: 1},
+				{Ts: minutes(26), V: 1},
+				{Ts: minutes(28), V: 1},
+				// Chunk 3
+				{Ts: minutes(30), V: 2},
+				{Ts: minutes(32), V: 2},
+				{Ts: minutes(34), V: 2},
+				{Ts: minutes(36), V: 2},
+				{Ts: minutes(38), V: 2},
 				// Head
-				sample{t: minutes(40), f: float64(3)},
-				sample{t: minutes(42), f: float64(3)},
+				{Ts: minutes(40), V: 3},
+				{Ts: minutes(42), V: 3},
 			},
 			expChunkError: false,
 			// ts (in minutes)         0       10       20       30       40       50       60       70       80       90       100
@@ -713,29 +699,29 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) {
 			// Output Graphically               [-------][-------][-------][--------]
 			expChunksSamples: []chunks.SampleSlice{
 				{
-					sample{t: minutes(10), f: float64(0)},
-					sample{t: minutes(12), f: float64(0)},
-					sample{t: minutes(14), f: float64(0)},
-					sample{t: minutes(16), f: float64(0)},
-					sample{t: minutes(18), f: float64(0)},
+					scenario.sampleFunc(minutes(10), 0),
+					scenario.sampleFunc(minutes(12), 0),
+					scenario.sampleFunc(minutes(14), 0),
+					scenario.sampleFunc(minutes(16), 0),
+					scenario.sampleFunc(minutes(18), 0),
 				},
 				{
-					sample{t: minutes(20), f: float64(1)},
-					sample{t: minutes(22), f: float64(1)},
-					sample{t: minutes(24), f: float64(1)},
-					sample{t: minutes(26), f: float64(1)},
-					sample{t: minutes(28), f: float64(1)},
+					scenario.sampleFunc(minutes(20), 1),
+					scenario.sampleFunc(minutes(22), 1),
+					scenario.sampleFunc(minutes(24), 1),
+					scenario.sampleFunc(minutes(26), 1),
+					scenario.sampleFunc(minutes(28), 1),
 				},
 				{
-					sample{t: minutes(30), f: float64(2)},
-					sample{t: minutes(32), f: float64(2)},
-					sample{t: minutes(34), f: float64(2)},
-					sample{t: minutes(36), f: float64(2)},
-					sample{t: minutes(38), f: float64(2)},
+					scenario.sampleFunc(minutes(30), 2),
+					scenario.sampleFunc(minutes(32), 2),
+					scenario.sampleFunc(minutes(34), 2),
+					scenario.sampleFunc(minutes(36), 2),
+					scenario.sampleFunc(minutes(38), 2),
 				},
 				{
-					sample{t: minutes(40), f: float64(3)},
-					sample{t: minutes(42), f: float64(3)},
+					scenario.sampleFunc(minutes(40), 3),
+					scenario.sampleFunc(minutes(42), 3),
 				},
 			},
 		},
@@ -744,22 +730,22 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) {
 			queryMinT:            minutes(0),
 			queryMaxT:            minutes(100),
 			firstInOrderSampleAt: minutes(120),
-			inputSamples: chunks.SampleSlice{
+			inputSamples: []testValue{
 				// Chunk 0
-				sample{t: minutes(10), f: float64(0)},
-				sample{t: minutes(15), f: float64(0)},
-				sample{t: minutes(20), f: float64(0)},
-				sample{t: minutes(25), f: float64(0)},
-				sample{t: minutes(30), f: float64(0)},
+				{Ts: minutes(10), V: 0},
+				{Ts: minutes(15), V: 0},
+				{Ts: minutes(20), V: 0},
+				{Ts: minutes(25), V: 0},
+				{Ts: minutes(30), V: 0},
 				// Chunk 1
-				sample{t: minutes(20), f: float64(1)},
-				sample{t: minutes(25), f: float64(1)},
-				sample{t: minutes(30), f: float64(1)},
-				sample{t: minutes(35), f: float64(1)},
-				sample{t: minutes(42), f: float64(1)},
+				{Ts: minutes(20), V: 1},
+				{Ts: minutes(25), V: 1},
+				{Ts: minutes(30), V: 1},
+				{Ts: minutes(35), V: 1},
+				{Ts: minutes(42), V: 1},
 				// Chunk 2 Head
-				sample{t: minutes(32), f: float64(2)},
-				sample{t: minutes(50), f: float64(2)},
+				{Ts: minutes(32), V: 2},
+				{Ts: minutes(50), V: 2},
 			},
 			expChunkError: false,
 			// ts (in minutes)         0       10       20       30       40       50       60       70       80       90       100
@@ -770,15 +756,15 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) {
 			// Output Graphically               [-----------------------------------]
 			expChunksSamples: []chunks.SampleSlice{
 				{
-					sample{t: minutes(10), f: float64(0)},
-					sample{t: minutes(15), f: float64(0)},
-					sample{t: minutes(20), f: float64(1)},
-					sample{t: minutes(25), f: float64(1)},
-					sample{t: minutes(30), f: float64(1)},
-					sample{t: minutes(32), f: float64(2)},
-					sample{t: minutes(35), f: float64(1)},
-					sample{t: minutes(42), f: float64(1)},
-					sample{t: minutes(50), f: float64(2)},
+					scenario.sampleFunc(minutes(10), 0),
+					scenario.sampleFunc(minutes(15), 0),
+					scenario.sampleFunc(minutes(20), 1),
+					scenario.sampleFunc(minutes(25), 1),
+					scenario.sampleFunc(minutes(30), 1),
+					scenario.sampleFunc(minutes(32), 2),
+					scenario.sampleFunc(minutes(35), 1),
+					scenario.sampleFunc(minutes(42), 1),
+					scenario.sampleFunc(minutes(50), 2),
 				},
 			},
 		},
@@ -787,22 +773,22 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) {
 			queryMinT:            minutes(12),
 			queryMaxT:            minutes(33),
 			firstInOrderSampleAt: minutes(120),
-			inputSamples: chunks.SampleSlice{
+			inputSamples: []testValue{
 				// Chunk 0
-				sample{t: minutes(10), f: float64(0)},
-				sample{t: minutes(15), f: float64(0)},
-				sample{t: minutes(20), f: float64(0)},
-				sample{t: minutes(25), f: float64(0)},
-				sample{t: minutes(30), f: float64(0)},
+				{Ts: minutes(10), V: 0},
+				{Ts: minutes(15), V: 0},
+				{Ts: minutes(20), V: 0},
+				{Ts: minutes(25), V: 0},
+				{Ts: minutes(30), V: 0},
 				// Chunk 1
-				sample{t: minutes(20), f: float64(1)},
-				sample{t: minutes(25), f: float64(1)},
-				sample{t: minutes(30), f: float64(1)},
-				sample{t: minutes(35), f: float64(1)},
-				sample{t: minutes(42), f: float64(1)},
+				{Ts: minutes(20), V: 1},
+				{Ts: minutes(25), V: 1},
+				{Ts: minutes(30), V: 1},
+				{Ts: minutes(35), V: 1},
+				{Ts: minutes(42), V: 1},
 				// Chunk 2 Head
-				sample{t: minutes(32), f: float64(2)},
-				sample{t: minutes(50), f: float64(2)},
+				{Ts: minutes(32), V: 2},
+				{Ts: minutes(50), V: 2},
 			},
 			expChunkError: false,
 			// ts (in minutes)         0       10       20       30       40       50       60       70       80       90       100
@@ -813,15 +799,15 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) {
 			// Output Graphically               [-----------------------------------]
 			expChunksSamples: []chunks.SampleSlice{
 				{
-					sample{t: minutes(10), f: float64(0)},
-					sample{t: minutes(15), f: float64(0)},
-					sample{t: minutes(20), f: float64(1)},
-					sample{t: minutes(25), f: float64(1)},
-					sample{t: minutes(30), f: float64(1)},
-					sample{t: minutes(32), f: float64(2)},
-					sample{t: minutes(35), f: float64(1)},
-					sample{t: minutes(42), f: float64(1)},
-					sample{t: minutes(50), f: float64(2)},
+					scenario.sampleFunc(minutes(10), 0),
+					scenario.sampleFunc(minutes(15), 0),
+					scenario.sampleFunc(minutes(20), 1),
+					scenario.sampleFunc(minutes(25), 1),
+					scenario.sampleFunc(minutes(30), 1),
+					scenario.sampleFunc(minutes(32), 2),
+					scenario.sampleFunc(minutes(35), 1),
+					scenario.sampleFunc(minutes(42), 1),
+					scenario.sampleFunc(minutes(50), 2),
 				},
 			},
 		},
@@ -832,22 +818,24 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) {
 			db := newTestDBWithOpts(t, opts)
 
 			app := db.Appender(context.Background())
-			s1Ref := appendSample(app, s1, tc.firstInOrderSampleAt, float64(tc.firstInOrderSampleAt/1*time.Minute.Milliseconds()))
+			s1Ref, _, err := scenario.appendFunc(app, s1, tc.firstInOrderSampleAt, tc.firstInOrderSampleAt/1*time.Minute.Milliseconds())
+			require.NoError(t, err)
 			require.NoError(t, app.Commit())
 
 			// OOO few samples for s1.
 			app = db.Appender(context.Background())
 			for _, s := range tc.inputSamples {
-				appendSample(app, s1, s.T(), s.F())
+				_, _, err := scenario.appendFunc(app, s1, s.Ts, s.V)
+				require.NoError(t, err)
 			}
 			require.NoError(t, app.Commit())
 
-			// The Series method is the one that populates the chunk meta OOO
-			// markers like OOOLastRef. These are then used by the ChunkReader.
+			// The Series method populates the chunk metas, taking a copy of the
+			// head OOO chunk if necessary. These are then used by the ChunkReader.
 			ir := NewOOOHeadIndexReader(db.head, tc.queryMinT, tc.queryMaxT, 0)
 			var chks []chunks.Meta
 			var b labels.ScratchBuilder
-			err := ir.Series(s1Ref, &b, &chks)
+			err = ir.Series(s1Ref, &b, &chks)
 			require.NoError(t, err)
 			require.Equal(t, len(tc.expChunksSamples), len(chks))
 
@@ -858,13 +846,10 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) {
 				require.NoError(t, err)
 				require.Nil(t, c)
 
-				var resultSamples chunks.SampleSlice
 				it := iterable.Iterator(nil)
-				for it.Next() == chunkenc.ValFloat {
-					t, v := it.At()
-					resultSamples = append(resultSamples, sample{t: t, f: v})
-				}
-				require.Equal(t, tc.expChunksSamples[i], resultSamples)
+				resultSamples, err := storage.ExpandSamples(it, nil)
+				require.NoError(t, err)
+				requireEqualSamples(t, s1.String(), tc.expChunksSamples[i], resultSamples, true)
 			}
 		})
 	}
@@ -879,6 +864,15 @@ func TestOOOHeadChunkReader_Chunk(t *testing.T) {
 //   - Response B comes from : Series(), in parallel new samples added to the head, then Chunk()
 //   - A == B
 func TestOOOHeadChunkReader_Chunk_ConsistentQueryResponseDespiteOfHeadExpanding(t *testing.T) {
+	for name, scenario := range sampleTypeScenarios {
+		t.Run(name, func(t *testing.T) {
+			testOOOHeadChunkReader_Chunk_ConsistentQueryResponseDespiteOfHeadExpanding(t, scenario)
+		})
+	}
+}
+
+//nolint:revive // unexported-return.
+func testOOOHeadChunkReader_Chunk_ConsistentQueryResponseDespiteOfHeadExpanding(t *testing.T, scenario sampleTypeScenario) {
 	opts := DefaultOptions()
 	opts.OutOfOrderCapMax = 5
 	opts.OutOfOrderTimeWindow = 120 * time.Minute.Milliseconds()
@@ -886,19 +880,13 @@ func TestOOOHeadChunkReader_Chunk_ConsistentQueryResponseDespiteOfHeadExpanding(
 	s1 := labels.FromStrings("l", "v1")
 	minutes := func(m int64) int64 { return m * time.Minute.Milliseconds() }
 
-	appendSample := func(app storage.Appender, l labels.Labels, timestamp int64, value float64) storage.SeriesRef {
-		ref, err := app.Append(0, l, timestamp, value)
-		require.NoError(t, err)
-		return ref
-	}
-
 	tests := []struct {
 		name                   string
 		queryMinT              int64
 		queryMaxT              int64
 		firstInOrderSampleAt   int64
-		initialSamples         chunks.SampleSlice
-		samplesAfterSeriesCall chunks.SampleSlice
+		initialSamples         []testValue
+		samplesAfterSeriesCall []testValue
 		expChunkError          bool
 		expChunksSamples       []chunks.SampleSlice
 	}{
@@ -907,21 +895,21 @@ func TestOOOHeadChunkReader_Chunk_ConsistentQueryResponseDespiteOfHeadExpanding(
 			queryMinT:            minutes(0),
 			queryMaxT:            minutes(100),
 			firstInOrderSampleAt: minutes(120),
-			initialSamples: chunks.SampleSlice{
+			initialSamples: []testValue{
 				// Chunk 0
-				sample{t: minutes(20), f: float64(0)},
-				sample{t: minutes(22), f: float64(0)},
-				sample{t: minutes(24), f: float64(0)},
-				sample{t: minutes(26), f: float64(0)},
-				sample{t: minutes(30), f: float64(0)},
+				{Ts: minutes(20), V: 0},
+				{Ts: minutes(22), V: 0},
+				{Ts: minutes(24), V: 0},
+				{Ts: minutes(26), V: 0},
+				{Ts: minutes(30), V: 0},
 				// Chunk 1 Head
-				sample{t: minutes(25), f: float64(1)},
-				sample{t: minutes(35), f: float64(1)},
+				{Ts: minutes(25), V: 1},
+				{Ts: minutes(35), V: 1},
 			},
-			samplesAfterSeriesCall: chunks.SampleSlice{
-				sample{t: minutes(10), f: float64(1)},
-				sample{t: minutes(32), f: float64(1)},
-				sample{t: minutes(50), f: float64(1)},
+			samplesAfterSeriesCall: []testValue{
+				{Ts: minutes(10), V: 1},
+				{Ts: minutes(32), V: 1},
+				{Ts: minutes(50), V: 1},
 			},
 			expChunkError: false,
 			// ts (in minutes)         0       10       20       30       40       50       60       70       80       90       100
@@ -933,40 +921,39 @@ func TestOOOHeadChunkReader_Chunk_ConsistentQueryResponseDespiteOfHeadExpanding(
 			// Output Graphically                        [------------] (With 8 samples, samples newer than lastmint or older than lastmaxt are omitted but the ones in between are kept)
 			expChunksSamples: []chunks.SampleSlice{
 				{
-					sample{t: minutes(20), f: float64(0)},
-					sample{t: minutes(22), f: float64(0)},
-					sample{t: minutes(24), f: float64(0)},
-					sample{t: minutes(25), f: float64(1)},
-					sample{t: minutes(26), f: float64(0)},
-					sample{t: minutes(30), f: float64(0)},
-					sample{t: minutes(32), f: float64(1)}, // This sample was added after Series() but before Chunk() and its in between the lastmint and maxt so it should be kept
-					sample{t: minutes(35), f: float64(1)},
+					scenario.sampleFunc(minutes(20), 0),
+					scenario.sampleFunc(minutes(22), 0),
+					scenario.sampleFunc(minutes(24), 0),
+					scenario.sampleFunc(minutes(25), 1),
+					scenario.sampleFunc(minutes(26), 0),
+					scenario.sampleFunc(minutes(30), 0),
+					scenario.sampleFunc(minutes(35), 1),
 				},
 			},
 		},
 		{
-			name:                 "After Series() previous head gets mmapped after getting samples, new head gets new samples also overlapping, none of these should appear in the response.",
+			name:                 "After Series() prev head gets mmapped after getting samples, new head gets new samples also overlapping, none of these should appear in response.",
 			queryMinT:            minutes(0),
 			queryMaxT:            minutes(100),
 			firstInOrderSampleAt: minutes(120),
-			initialSamples: chunks.SampleSlice{
+			initialSamples: []testValue{
 				// Chunk 0
-				sample{t: minutes(20), f: float64(0)},
-				sample{t: minutes(22), f: float64(0)},
-				sample{t: minutes(24), f: float64(0)},
-				sample{t: minutes(26), f: float64(0)},
-				sample{t: minutes(30), f: float64(0)},
+				{Ts: minutes(20), V: 0},
+				{Ts: minutes(22), V: 0},
+				{Ts: minutes(24), V: 0},
+				{Ts: minutes(26), V: 0},
+				{Ts: minutes(30), V: 0},
 				// Chunk 1 Head
-				sample{t: minutes(25), f: float64(1)},
-				sample{t: minutes(35), f: float64(1)},
+				{Ts: minutes(25), V: 1},
+				{Ts: minutes(35), V: 1},
 			},
-			samplesAfterSeriesCall: chunks.SampleSlice{
-				sample{t: minutes(10), f: float64(1)},
-				sample{t: minutes(32), f: float64(1)},
-				sample{t: minutes(50), f: float64(1)},
+			samplesAfterSeriesCall: []testValue{
+				{Ts: minutes(10), V: 1},
+				{Ts: minutes(32), V: 1},
+				{Ts: minutes(50), V: 1},
 				// Chunk 1 gets mmapped and Chunk 2, the new head is born
-				sample{t: minutes(25), f: float64(2)},
-				sample{t: minutes(31), f: float64(2)},
+				{Ts: minutes(25), V: 2},
+				{Ts: minutes(31), V: 2},
 			},
 			expChunkError: false,
 			// ts (in minutes)         0       10       20       30       40       50       60       70       80       90       100
@@ -979,14 +966,13 @@ func TestOOOHeadChunkReader_Chunk_ConsistentQueryResponseDespiteOfHeadExpanding(
 			// Output Graphically                        [------------]  (8 samples) It has 5 from Chunk 0 and 3 from Chunk 1
 			expChunksSamples: []chunks.SampleSlice{
 				{
-					sample{t: minutes(20), f: float64(0)},
-					sample{t: minutes(22), f: float64(0)},
-					sample{t: minutes(24), f: float64(0)},
-					sample{t: minutes(25), f: float64(1)},
-					sample{t: minutes(26), f: float64(0)},
-					sample{t: minutes(30), f: float64(0)},
-					sample{t: minutes(32), f: float64(1)}, // This sample was added after Series() but before Chunk() and its in between the lastmint and maxt so it should be kept
-					sample{t: minutes(35), f: float64(1)},
+					scenario.sampleFunc(minutes(20), 0),
+					scenario.sampleFunc(minutes(22), 0),
+					scenario.sampleFunc(minutes(24), 0),
+					scenario.sampleFunc(minutes(25), 1),
+					scenario.sampleFunc(minutes(26), 0),
+					scenario.sampleFunc(minutes(30), 0),
+					scenario.sampleFunc(minutes(35), 1),
 				},
 			},
 		},
@@ -997,22 +983,24 @@ func TestOOOHeadChunkReader_Chunk_ConsistentQueryResponseDespiteOfHeadExpanding(
 			db := newTestDBWithOpts(t, opts)
 
 			app := db.Appender(context.Background())
-			s1Ref := appendSample(app, s1, tc.firstInOrderSampleAt, float64(tc.firstInOrderSampleAt/1*time.Minute.Milliseconds()))
+			s1Ref, _, err := scenario.appendFunc(app, s1, tc.firstInOrderSampleAt, tc.firstInOrderSampleAt/1*time.Minute.Milliseconds())
+			require.NoError(t, err)
 			require.NoError(t, app.Commit())
 
 			// OOO few samples for s1.
 			app = db.Appender(context.Background())
 			for _, s := range tc.initialSamples {
-				appendSample(app, s1, s.T(), s.F())
+				_, _, err := scenario.appendFunc(app, s1, s.Ts, s.V)
+				require.NoError(t, err)
 			}
 			require.NoError(t, app.Commit())
 
-			// The Series method is the one that populates the chunk meta OOO
-			// markers like OOOLastRef. These are then used by the ChunkReader.
+			// The Series method populates the chunk metas, taking a copy of the
+			// head OOO chunk if necessary. These are then used by the ChunkReader.
 			ir := NewOOOHeadIndexReader(db.head, tc.queryMinT, tc.queryMaxT, 0)
 			var chks []chunks.Meta
 			var b labels.ScratchBuilder
-			err := ir.Series(s1Ref, &b, &chks)
+			err = ir.Series(s1Ref, &b, &chks)
 			require.NoError(t, err)
 			require.Equal(t, len(tc.expChunksSamples), len(chks))
 
@@ -1020,7 +1008,8 @@ func TestOOOHeadChunkReader_Chunk_ConsistentQueryResponseDespiteOfHeadExpanding(
 			// OOO few samples for s1.
 			app = db.Appender(context.Background())
 			for _, s := range tc.samplesAfterSeriesCall {
-				appendSample(app, s1, s.T(), s.F())
+				_, _, err = scenario.appendFunc(app, s1, s.Ts, s.V)
+				require.NoError(t, err)
 			}
 			require.NoError(t, app.Commit())
 
@@ -1031,13 +1020,10 @@ func TestOOOHeadChunkReader_Chunk_ConsistentQueryResponseDespiteOfHeadExpanding(
 				require.NoError(t, err)
 				require.Nil(t, c)
 
-				var resultSamples chunks.SampleSlice
 				it := iterable.Iterator(nil)
-				for it.Next() == chunkenc.ValFloat {
-					ts, v := it.At()
-					resultSamples = append(resultSamples, sample{t: ts, f: v})
-				}
-				require.Equal(t, tc.expChunksSamples[i], resultSamples)
+				resultSamples, err := storage.ExpandSamples(it, nil)
+				require.NoError(t, err)
+				requireEqualSamples(t, s1.String(), tc.expChunksSamples[i], resultSamples, true)
 			}
 		})
 	}

tsdb/querier.go

@@ -77,12 +77,12 @@ func newBlockBaseQuerier(b BlockReader, mint, maxt int64) (*blockBaseQuerier, er
 	}, nil
 }
 
-func (q *blockBaseQuerier) LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+func (q *blockBaseQuerier) LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
 	res, err := q.index.SortedLabelValues(ctx, name, matchers...)
 	return res, nil, err
 }
 
-func (q *blockBaseQuerier) LabelNames(ctx context.Context, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+func (q *blockBaseQuerier) LabelNames(ctx context.Context, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
 	res, err := q.index.LabelNames(ctx, matchers...)
 	return res, nil, err
 }

tsdb/querier_test.go

@@ -3022,7 +3022,7 @@ func TestQuerierIndexQueriesRace(t *testing.T) {
 				q, err := db.Querier(math.MinInt64, math.MaxInt64)
 				require.NoError(t, err)
 
-				values, _, err := q.LabelValues(ctx, "seq", c.matchers...)
+				values, _, err := q.LabelValues(ctx, "seq", nil, c.matchers...)
 				require.NoError(t, err)
 				require.Emptyf(t, values, `label values for label "seq" should be empty`)
 

tsdb/record/record.go

@@ -543,7 +543,7 @@ func (d *Decoder) FloatHistogramSamples(rec []byte, histograms []RefFloatHistogr
 	return histograms, nil
 }
 
-// Decode decodes a Histogram from a byte slice.
+// DecodeFloatHistogram decodes a Histogram from a byte slice.
 func DecodeFloatHistogram(buf *encoding.Decbuf, fh *histogram.FloatHistogram) {
 	fh.CounterResetHint = histogram.CounterResetHint(buf.Byte())
 

tsdb/testutil.go

@@ -0,0 +1,176 @@
+// Copyright 2017 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tsdb
+
+import (
+	"testing"
+
+	prom_testutil "github.com/prometheus/client_golang/prometheus/testutil"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/prometheus/model/histogram"
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/storage"
+	"github.com/prometheus/prometheus/tsdb/chunks"
+)
+
+const (
+	float = "float"
+)
+
+type testValue struct {
+	Ts                 int64
+	V                  int64
+	CounterResetHeader histogram.CounterResetHint
+}
+
+type sampleTypeScenario struct {
+	sampleType string
+	appendFunc func(appender storage.Appender, lbls labels.Labels, ts, value int64) (storage.SeriesRef, sample, error)
+	sampleFunc func(ts, value int64) sample
+}
+
+// TODO: native histogram sample types will be added as part of out-of-order native histogram support; see #11220.
+var sampleTypeScenarios = map[string]sampleTypeScenario{
+	float: {
+		sampleType: sampleMetricTypeFloat,
+		appendFunc: func(appender storage.Appender, lbls labels.Labels, ts, value int64) (storage.SeriesRef, sample, error) {
+			s := sample{t: ts, f: float64(value)}
+			ref, err := appender.Append(0, lbls, ts, s.f)
+			return ref, s, err
+		},
+		sampleFunc: func(ts, value int64) sample {
+			return sample{t: ts, f: float64(value)}
+		},
+	},
+	// intHistogram: {
+	//	 sampleType: sampleMetricTypeHistogram,
+	//	 appendFunc: func(appender storage.Appender, lbls labels.Labels, ts, value int64) (storage.SeriesRef, sample, error) {
+	//		 s := sample{t: ts, h: tsdbutil.GenerateTestHistogram(int(value))}
+	//		 ref, err := appender.AppendHistogram(0, lbls, ts, s.h, nil)
+	//		 return ref, s, err
+	//	 },
+	//	 sampleFunc: func(ts, value int64) sample {
+	//		 return sample{t: ts, h: tsdbutil.GenerateTestHistogram(int(value))}
+	//	 },
+	// },
+	// floatHistogram: {
+	//	 sampleType: sampleMetricTypeHistogram,
+	//	 appendFunc: func(appender storage.Appender, lbls labels.Labels, ts, value int64) (storage.SeriesRef, sample, error) {
+	//		 s := sample{t: ts, fh: tsdbutil.GenerateTestFloatHistogram(int(value))}
+	//		 ref, err := appender.AppendHistogram(0, lbls, ts, nil, s.fh)
+	//		 return ref, s, err
+	//	 },
+	//	 sampleFunc: func(ts, value int64) sample {
+	//		 return sample{t: ts, fh: tsdbutil.GenerateTestFloatHistogram(int(value))}
+	//	 },
+	// },
+	// gaugeIntHistogram: {
+	//	 sampleType: sampleMetricTypeHistogram,
+	//	 appendFunc: func(appender storage.Appender, lbls labels.Labels, ts, value int64) (storage.SeriesRef, sample, error) {
+	//		 s := sample{t: ts, h: tsdbutil.GenerateTestGaugeHistogram(int(value))}
+	//		 ref, err := appender.AppendHistogram(0, lbls, ts, s.h, nil)
+	//		 return ref, s, err
+	//	 },
+	//	 sampleFunc: func(ts, value int64) sample {
+	//		 return sample{t: ts, h: tsdbutil.GenerateTestGaugeHistogram(int(value))}
+	//	 },
+	// },
+	// gaugeFloatHistogram: {
+	//	 sampleType: sampleMetricTypeHistogram,
+	//	 appendFunc: func(appender storage.Appender, lbls labels.Labels, ts, value int64) (storage.SeriesRef, sample, error) {
+	//		 s := sample{t: ts, fh: tsdbutil.GenerateTestGaugeFloatHistogram(int(value))}
+	//		 ref, err := appender.AppendHistogram(0, lbls, ts, nil, s.fh)
+	//		 return ref, s, err
+	//	 },
+	//	 sampleFunc: func(ts, value int64) sample {
+	//		 return sample{t: ts, fh: tsdbutil.GenerateTestGaugeFloatHistogram(int(value))}
+	//	 },
+	// },
+}
+
+// requireEqualSeries checks that the actual series are equal to the expected ones. It ignores the counter reset hints for histograms.
+func requireEqualSeries(t *testing.T, expected, actual map[string][]chunks.Sample, ignoreCounterResets bool) {
+	for name, expectedItem := range expected {
+		actualItem, ok := actual[name]
+		require.True(t, ok, "Expected series %s not found", name)
+		requireEqualSamples(t, name, expectedItem, actualItem, ignoreCounterResets)
+	}
+	for name := range actual {
+		_, ok := expected[name]
+		require.True(t, ok, "Unexpected series %s", name)
+	}
+}
+
+func requireEqualOOOSamples(t *testing.T, expectedSamples int, db *DB) {
+	require.Equal(t, float64(expectedSamples),
+		prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamplesAppended.WithLabelValues(sampleMetricTypeFloat))+
+			prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamplesAppended.WithLabelValues(sampleMetricTypeHistogram)),
+		"number of ooo appended samples mismatch")
+}
+
+func requireEqualSamples(t *testing.T, name string, expected, actual []chunks.Sample, ignoreCounterResets bool) {
+	require.Equal(t, len(expected), len(actual), "Length not equal to expected for %s", name)
+	for i, s := range expected {
+		expectedSample := s
+		actualSample := actual[i]
+		require.Equal(t, expectedSample.T(), actualSample.T(), "Different timestamps for %s[%d]", name, i)
+		require.Equal(t, expectedSample.Type().String(), actualSample.Type().String(), "Different types for %s[%d] at ts %d", name, i, expectedSample.T())
+		switch {
+		case s.H() != nil:
+			{
+				expectedHist := expectedSample.H()
+				actualHist := actualSample.H()
+				if ignoreCounterResets && expectedHist.CounterResetHint != histogram.GaugeType {
+					expectedHist.CounterResetHint = histogram.UnknownCounterReset
+					actualHist.CounterResetHint = histogram.UnknownCounterReset
+				} else {
+					require.Equal(t, expectedHist.CounterResetHint, actualHist.CounterResetHint, "Sample header doesn't match for %s[%d] at ts %d, expected: %s, actual: %s", name, i, expectedSample.T(), counterResetAsString(expectedHist.CounterResetHint), counterResetAsString(actualHist.CounterResetHint))
+				}
+				require.Equal(t, expectedHist, actualHist, "Sample doesn't match for %s[%d] at ts %d", name, i, expectedSample.T())
+			}
+		case s.FH() != nil:
+			{
+				expectedHist := expectedSample.FH()
+				actualHist := actualSample.FH()
+				if ignoreCounterResets {
+					expectedHist.CounterResetHint = histogram.UnknownCounterReset
+					actualHist.CounterResetHint = histogram.UnknownCounterReset
+				} else {
+					require.Equal(t, expectedHist.CounterResetHint, actualHist.CounterResetHint, "Sample header doesn't match for %s[%d] at ts %d, expected: %s, actual: %s", name, i, expectedSample.T(), counterResetAsString(expectedHist.CounterResetHint), counterResetAsString(actualHist.CounterResetHint))
+				}
+				require.Equal(t, expectedHist, actualHist, "Sample doesn't match for %s[%d] at ts %d", name, i, expectedSample.T())
+			}
+		default:
+			expectedFloat := expectedSample.F()
+			actualFloat := actualSample.F()
+			require.Equal(t, expectedFloat, actualFloat, "Sample doesn't match for %s[%d] at ts %d", name, i, expectedSample.T())
+		}
+	}
+}
+
+func counterResetAsString(h histogram.CounterResetHint) string {
+	switch h {
+	case histogram.UnknownCounterReset:
+		return "UnknownCounterReset"
+	case histogram.CounterReset:
+		return "CounterReset"
+	case histogram.NotCounterReset:
+		return "NotCounterReset"
+	case histogram.GaugeType:
+		return "GaugeType"
+	}
+	panic("Unexpected counter reset type")
+}

tsdb/wlog/watcher.go

@@ -57,6 +57,7 @@ type WriteTo interface {
 	AppendHistograms([]record.RefHistogramSample) bool
 	AppendFloatHistograms([]record.RefFloatHistogramSample) bool
 	StoreSeries([]record.RefSeries, int)
+	StoreMetadata([]record.RefMetadata)
 
 	// Next two methods are intended for garbage-collection: first we call
 	// UpdateSeriesSegment on all current series
@@ -88,6 +89,7 @@ type Watcher struct {
 	lastCheckpoint string
 	sendExemplars  bool
 	sendHistograms bool
+	sendMetadata   bool
 	metrics        *WatcherMetrics
 	readerMetrics  *LiveReaderMetrics
 
@@ -170,7 +172,7 @@ func NewWatcherMetrics(reg prometheus.Registerer) *WatcherMetrics {
 }
 
 // NewWatcher creates a new WAL watcher for a given WriteTo.
-func NewWatcher(metrics *WatcherMetrics, readerMetrics *LiveReaderMetrics, logger log.Logger, name string, writer WriteTo, dir string, sendExemplars, sendHistograms bool) *Watcher {
+func NewWatcher(metrics *WatcherMetrics, readerMetrics *LiveReaderMetrics, logger log.Logger, name string, writer WriteTo, dir string, sendExemplars, sendHistograms, sendMetadata bool) *Watcher {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -183,6 +185,7 @@ func NewWatcher(metrics *WatcherMetrics, readerMetrics *LiveReaderMetrics, logge
 		name:           name,
 		sendExemplars:  sendExemplars,
 		sendHistograms: sendHistograms,
+		sendMetadata:   sendMetadata,
 
 		readNotify: make(chan struct{}),
 		quit:       make(chan struct{}),
@@ -262,6 +265,11 @@ func (w *Watcher) loop() {
 // Run the watcher, which will tail the WAL until the quit channel is closed
 // or an error case is hit.
 func (w *Watcher) Run() error {
+	_, lastSegment, err := w.firstAndLast()
+	if err != nil {
+		return fmt.Errorf("wal.Segments: %w", err)
+	}
+
 	// We want to ensure this is false across iterations since
 	// Run will be called again if there was a failure to read the WAL.
 	w.sendSamples = false
@@ -286,20 +294,14 @@ func (w *Watcher) Run() error {
 		return err
 	}
 
-	level.Debug(w.logger).Log("msg", "Tailing WAL", "lastCheckpoint", lastCheckpoint, "checkpointIndex", checkpointIndex, "currentSegment", currentSegment)
+	level.Debug(w.logger).Log("msg", "Tailing WAL", "lastCheckpoint", lastCheckpoint, "checkpointIndex", checkpointIndex, "currentSegment", currentSegment, "lastSegment", lastSegment)
 	for !isClosed(w.quit) {
 		w.currentSegmentMetric.Set(float64(currentSegment))
 
-		// Re-check on each iteration in case a new segment was added,
-		// because watch() will wait for notifications on the last segment.
-		_, lastSegment, err := w.firstAndLast()
-		if err != nil {
-			return fmt.Errorf("wal.Segments: %w", err)
-		}
-		tail := currentSegment >= lastSegment
-
-		level.Debug(w.logger).Log("msg", "Processing segment", "currentSegment", currentSegment, "lastSegment", lastSegment)
-		if err := w.watch(currentSegment, tail); err != nil && !errors.Is(err, ErrIgnorable) {
+		// On start, after reading the existing WAL for series records, we have a pointer to what is the latest segment.
+		// On subsequent calls to this function, currentSegment will have been incremented and we should open that segment.
+		level.Debug(w.logger).Log("msg", "Processing segment", "currentSegment", currentSegment)
+		if err := w.watch(currentSegment, currentSegment >= lastSegment); err != nil && !errors.Is(err, ErrIgnorable) {
 			return err
 		}
 
@@ -541,6 +543,7 @@ func (w *Watcher) readSegment(r *LiveReader, segmentNum int, tail bool) error {
 		histogramsToSend      []record.RefHistogramSample
 		floatHistograms       []record.RefFloatHistogramSample
 		floatHistogramsToSend []record.RefFloatHistogramSample
+		metadata              []record.RefMetadata
 	)
 	for r.Next() && !isClosed(w.quit) {
 		rec := r.Record()
@@ -652,6 +655,17 @@ func (w *Watcher) readSegment(r *LiveReader, segmentNum int, tail bool) error {
 				w.writer.AppendFloatHistograms(floatHistogramsToSend)
 				floatHistogramsToSend = floatHistogramsToSend[:0]
 			}
+
+		case record.Metadata:
+			if !w.sendMetadata || !tail {
+				break
+			}
+			meta, err := dec.Metadata(rec, metadata[:0])
+			if err != nil {
+				w.recordDecodeFailsMetric.Inc()
+				return err
+			}
+			w.writer.StoreMetadata(meta)
 		case record.Tombstones:
 
 		default: