prometheus/storage/remote/write_test.go

// Copyright 2017 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package remote

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"log/slog"
	"math/rand/v2"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"reflect"
	"runtime"
	"strconv"
	"sync"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/prometheus/client_golang/prometheus"
	common_config "github.com/prometheus/common/config"
	"github.com/prometheus/common/model"
	"github.com/stretchr/testify/require"
	"go.opentelemetry.io/collector/pdata/pcommon"
	"go.opentelemetry.io/collector/pdata/pmetric"
	"go.opentelemetry.io/collector/pdata/pmetric/pmetricotlp"

	"github.com/prometheus/prometheus/config"
	"github.com/prometheus/prometheus/model/histogram"
	"github.com/prometheus/prometheus/model/labels"
	"github.com/prometheus/prometheus/model/relabel"
	"github.com/prometheus/prometheus/storage"
)

func testRemoteWriteConfig() *config.RemoteWriteConfig {
	return &config.RemoteWriteConfig{
		Name: "dev",
		URL: &common_config.URL{
			URL: &url.URL{
				Scheme: "http",
				Host:   "localhost",
			},
		},
		QueueConfig:     config.DefaultQueueConfig,
		ProtobufMessage: config.RemoteWriteProtoMsgV1,
	}
}

func TestWriteStorageApplyConfig_NoDuplicateWriteConfigs(t *testing.T) {
	dir := t.TempDir()

	cfg1 := config.RemoteWriteConfig{
		Name: "write-1",
		URL: &common_config.URL{
			URL: &url.URL{
				Scheme: "http",
				Host:   "localhost",
			},
		},
		QueueConfig:     config.DefaultQueueConfig,
		ProtobufMessage: config.RemoteWriteProtoMsgV1,
	}
	cfg2 := config.RemoteWriteConfig{
		Name: "write-2",
		URL: &common_config.URL{
			URL: &url.URL{
				Scheme: "http",
				Host:   "localhost",
			},
		},
		QueueConfig:     config.DefaultQueueConfig,
		ProtobufMessage: config.RemoteWriteProtoMsgV1,
	}
	cfg3 := config.RemoteWriteConfig{
		URL: &common_config.URL{
			URL: &url.URL{
				Scheme: "http",
				Host:   "localhost",
			},
		},
		QueueConfig:     config.DefaultQueueConfig,
		ProtobufMessage: config.RemoteWriteProtoMsgV1,
	}

	for _, tc := range []struct {
		cfgs        []*config.RemoteWriteConfig
		expectedErr error
	}{
		{ // Two duplicates, we should get an error.
			cfgs:        []*config.RemoteWriteConfig{&cfg1, &cfg1},
			expectedErr: errors.New("duplicate remote write configs are not allowed, found duplicate for URL: http://localhost"),
		},
		{ // Duplicates but with different names, we should not get an error.
			cfgs: []*config.RemoteWriteConfig{&cfg1, &cfg2},
		},
		{ // Duplicates but one with no name, we should not get an error.
			cfgs: []*config.RemoteWriteConfig{&cfg1, &cfg3},
		},
		{ // Duplicates both with no name, we should get an error.
			cfgs:        []*config.RemoteWriteConfig{&cfg3, &cfg3},
			expectedErr: errors.New("duplicate remote write configs are not allowed, found duplicate for URL: http://localhost"),
		},
	} {
		t.Run("", func(t *testing.T) {
			s := NewWriteStorage(nil, nil, dir, time.Millisecond, nil, false)
			conf := &config.Config{
				GlobalConfig:       config.DefaultGlobalConfig,
				RemoteWriteConfigs: tc.cfgs,
			}
			err := s.ApplyConfig(conf)
			if tc.expectedErr == nil {
				require.NoError(t, err)
			} else {
				require.Error(t, err)
				require.Equal(t, tc.expectedErr, err)
			}

			require.NoError(t, s.Close())
		})
	}
}

func TestWriteStorageApplyConfig_RestartOnNameChange(t *testing.T) {
	dir := t.TempDir()

	cfg := testRemoteWriteConfig()

	hash, err := toHash(cfg)
	require.NoError(t, err)

	s := NewWriteStorage(nil, nil, dir, time.Millisecond, nil, false)

	conf := &config.Config{
		GlobalConfig:       config.DefaultGlobalConfig,
		RemoteWriteConfigs: []*config.RemoteWriteConfig{cfg},
	}
	require.NoError(t, s.ApplyConfig(conf))
	require.Equal(t, s.queues[hash].client().Name(), cfg.Name)

	// Change the queues name, ensure the queue has been restarted.
	conf.RemoteWriteConfigs[0].Name = "dev-2"
	require.NoError(t, s.ApplyConfig(conf))
	hash, err = toHash(cfg)
	require.NoError(t, err)
	require.Equal(t, s.queues[hash].client().Name(), conf.RemoteWriteConfigs[0].Name)

	require.NoError(t, s.Close())
}

func TestWriteStorageApplyConfig_UpdateWithRegisterer(t *testing.T) {
	dir := t.TempDir()

	s := NewWriteStorage(nil, prometheus.NewRegistry(), dir, time.Millisecond, nil, false)
	c1 := &config.RemoteWriteConfig{
		Name: "named",
		URL: &common_config.URL{
			URL: &url.URL{
				Scheme: "http",
				Host:   "localhost",
			},
		},
		QueueConfig:     config.DefaultQueueConfig,
		ProtobufMessage: config.RemoteWriteProtoMsgV1,
	}
	c2 := &config.RemoteWriteConfig{
		URL: &common_config.URL{
			URL: &url.URL{
				Scheme: "http",
				Host:   "localhost",
			},
		},
		QueueConfig:     config.DefaultQueueConfig,
		ProtobufMessage: config.RemoteWriteProtoMsgV1,
	}
	conf := &config.Config{
		GlobalConfig:       config.DefaultGlobalConfig,
		RemoteWriteConfigs: []*config.RemoteWriteConfig{c1, c2},
	}
	require.NoError(t, s.ApplyConfig(conf))

	c1.QueueConfig.MaxShards = 10
	c2.QueueConfig.MaxShards = 10
	require.NoError(t, s.ApplyConfig(conf))
	for _, queue := range s.queues {
		require.Equal(t, 10, queue.cfg.MaxShards)
	}

	require.NoError(t, s.Close())
}

func TestWriteStorageApplyConfig_Lifecycle(t *testing.T) {
	dir := t.TempDir()

	s := NewWriteStorage(nil, nil, dir, defaultFlushDeadline, nil, false)
	conf := &config.Config{
		GlobalConfig: config.DefaultGlobalConfig,
		RemoteWriteConfigs: []*config.RemoteWriteConfig{
			baseRemoteWriteConfig("http://test-storage.com"),
		},
	}
	require.NoError(t, s.ApplyConfig(conf))
	require.Len(t, s.queues, 1)

	require.NoError(t, s.Close())
}

func TestWriteStorageApplyConfig_UpdateExternalLabels(t *testing.T) {
	dir := t.TempDir()

	s := NewWriteStorage(nil, prometheus.NewRegistry(), dir, time.Second, nil, false)

	externalLabels := labels.FromStrings("external", "true")
	conf := &config.Config{
		GlobalConfig: config.GlobalConfig{},
		RemoteWriteConfigs: []*config.RemoteWriteConfig{
			testRemoteWriteConfig(),
		},
	}
	hash, err := toHash(conf.RemoteWriteConfigs[0])
	require.NoError(t, err)
	require.NoError(t, s.ApplyConfig(conf))
	require.Len(t, s.queues, 1)
	require.Empty(t, s.queues[hash].externalLabels)

	conf.GlobalConfig.ExternalLabels = externalLabels
	hash, err = toHash(conf.RemoteWriteConfigs[0])
	require.NoError(t, err)
	require.NoError(t, s.ApplyConfig(conf))
	require.Len(t, s.queues, 1)
	require.Equal(t, []labels.Label{{Name: "external", Value: "true"}}, s.queues[hash].externalLabels)

	require.NoError(t, s.Close())
}

func TestWriteStorageApplyConfig_Idempotent(t *testing.T) {
	dir := t.TempDir()

	s := NewWriteStorage(nil, nil, dir, defaultFlushDeadline, nil, false)
	conf := &config.Config{
		GlobalConfig: config.GlobalConfig{},
		RemoteWriteConfigs: []*config.RemoteWriteConfig{
			baseRemoteWriteConfig("http://test-storage.com"),
		},
	}
	hash, err := toHash(conf.RemoteWriteConfigs[0])
	require.NoError(t, err)

	require.NoError(t, s.ApplyConfig(conf))
	require.Len(t, s.queues, 1)

	require.NoError(t, s.ApplyConfig(conf))
	require.Len(t, s.queues, 1)
	_, hashExists := s.queues[hash]
	require.True(t, hashExists, "Queue pointer should have remained the same")

	require.NoError(t, s.Close())
}

func TestWriteStorageApplyConfig_PartialUpdate(t *testing.T) {
	dir := t.TempDir()

	s := NewWriteStorage(nil, nil, dir, defaultFlushDeadline, nil, false)

	c0 := &config.RemoteWriteConfig{
		RemoteTimeout: model.Duration(10 * time.Second),
		QueueConfig:   config.DefaultQueueConfig,
		WriteRelabelConfigs: []*relabel.Config{
			{
				Regex: relabel.MustNewRegexp(".+"),
			},
		},
		ProtobufMessage: config.RemoteWriteProtoMsgV1,
	}
	c1 := &config.RemoteWriteConfig{
		RemoteTimeout: model.Duration(20 * time.Second),
		QueueConfig:   config.DefaultQueueConfig,
		HTTPClientConfig: common_config.HTTPClientConfig{
			BearerToken: "foo",
		},
		ProtobufMessage: config.RemoteWriteProtoMsgV1,
	}
	c2 := &config.RemoteWriteConfig{
		RemoteTimeout:   model.Duration(30 * time.Second),
		QueueConfig:     config.DefaultQueueConfig,
		ProtobufMessage: config.RemoteWriteProtoMsgV1,
	}

	conf := &config.Config{
		GlobalConfig:       config.GlobalConfig{},
		RemoteWriteConfigs: []*config.RemoteWriteConfig{c0, c1, c2},
	}
	// We need to set URL's so that metric creation doesn't panic.
	for i := range conf.RemoteWriteConfigs {
		conf.RemoteWriteConfigs[i].URL = &common_config.URL{
			URL: &url.URL{
				Host: "http://test-storage.com",
			},
		}
	}
	require.NoError(t, s.ApplyConfig(conf))
	require.Len(t, s.queues, 3)

	hashes := make([]string, len(conf.RemoteWriteConfigs))
	queues := make([]*QueueManager, len(conf.RemoteWriteConfigs))
	storeHashes := func() {
		for i := range conf.RemoteWriteConfigs {
			hash, err := toHash(conf.RemoteWriteConfigs[i])
			require.NoError(t, err)
			hashes[i] = hash
			queues[i] = s.queues[hash]
		}
	}

	storeHashes()
	// Update c0 and c2.
	c0.WriteRelabelConfigs[0] = &relabel.Config{Regex: relabel.MustNewRegexp("foo")}
	c2.RemoteTimeout = model.Duration(50 * time.Second)
	conf = &config.Config{
		GlobalConfig:       config.GlobalConfig{},
		RemoteWriteConfigs: []*config.RemoteWriteConfig{c0, c1, c2},
	}
	require.NoError(t, s.ApplyConfig(conf))
	require.Len(t, s.queues, 3)

	_, hashExists := s.queues[hashes[0]]
	require.False(t, hashExists, "The queue for the first remote write configuration should have been restarted because the relabel configuration has changed.")
	q, hashExists := s.queues[hashes[1]]
	require.True(t, hashExists, "Hash of unchanged queue should have remained the same")
	require.Equal(t, q, queues[1], "Pointer of unchanged queue should have remained the same")
	_, hashExists = s.queues[hashes[2]]
	require.False(t, hashExists, "The queue for the third remote write configuration should have been restarted because the timeout has changed.")

	storeHashes()
	secondClient := s.queues[hashes[1]].client()
	// Update c1.
	c1.HTTPClientConfig.BearerToken = "bar"
	err := s.ApplyConfig(conf)
	require.NoError(t, err)
	require.Len(t, s.queues, 3)

	_, hashExists = s.queues[hashes[0]]
	require.True(t, hashExists, "Pointer of unchanged queue should have remained the same")
	q, hashExists = s.queues[hashes[1]]
	require.True(t, hashExists, "Hash of queue with secret change should have remained the same")
	require.NotEqual(t, secondClient, q.client(), "Pointer of a client with a secret change should not be the same")
	_, hashExists = s.queues[hashes[2]]
	require.True(t, hashExists, "Pointer of unchanged queue should have remained the same")

	storeHashes()
	// Delete c0.
	conf = &config.Config{
		GlobalConfig:       config.GlobalConfig{},
		RemoteWriteConfigs: []*config.RemoteWriteConfig{c1, c2},
	}
	require.NoError(t, s.ApplyConfig(conf))
	require.Len(t, s.queues, 2)

	_, hashExists = s.queues[hashes[0]]
	require.False(t, hashExists, "If a config is removed, the queue should be stopped and recreated.")
	_, hashExists = s.queues[hashes[1]]
	require.True(t, hashExists, "Pointer of unchanged queue should have remained the same")
	_, hashExists = s.queues[hashes[2]]
	require.True(t, hashExists, "Pointer of unchanged queue should have remained the same")

	require.NoError(t, s.Close())
}

func TestOTLPWriteHandler(t *testing.T) {
	exportRequest := generateOTLPWriteRequest()

	buf, err := exportRequest.MarshalProto()
	require.NoError(t, err)

	req, err := http.NewRequest("", "", bytes.NewReader(buf))
	require.NoError(t, err)
	req.Header.Set("Content-Type", "application/x-protobuf")

	appendable := &mockAppendable{}
	handler := NewOTLPWriteHandler(nil, nil, appendable, func() config.Config {
		return config.Config{
			OTLPConfig: config.DefaultOTLPConfig,
		}
	}, OTLPOptions{})

	recorder := httptest.NewRecorder()
	handler.ServeHTTP(recorder, req)

	resp := recorder.Result()
	require.Equal(t, http.StatusOK, resp.StatusCode)

	require.Len(t, appendable.samples, 12)   // 1 (counter) + 1 (gauge) + 1 (target_info) + 7 (hist_bucket) + 2 (hist_sum, hist_count)
	require.Len(t, appendable.histograms, 1) // 1 (exponential histogram)
	require.Len(t, appendable.exemplars, 1)  // 1 (exemplar)
}

func generateOTLPWriteRequest() pmetricotlp.ExportRequest {
	d := pmetric.NewMetrics()

	// Generate One Counter, One Gauge, One Histogram, One Exponential-Histogram
	// with resource attributes: service.name="test-service", service.instance.id="test-instance", host.name="test-host"
	// with metric attribute: foo.bar="baz"

	timestamp := time.Now()

	resourceMetric := d.ResourceMetrics().AppendEmpty()
	resourceMetric.Resource().Attributes().PutStr("service.name", "test-service")
	resourceMetric.Resource().Attributes().PutStr("service.instance.id", "test-instance")
	resourceMetric.Resource().Attributes().PutStr("host.name", "test-host")

	scopeMetric := resourceMetric.ScopeMetrics().AppendEmpty()

	// Generate One Counter
	counterMetric := scopeMetric.Metrics().AppendEmpty()
	counterMetric.SetName("test-counter")
	counterMetric.SetDescription("test-counter-description")
	counterMetric.SetEmptySum()
	counterMetric.Sum().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative)
	counterMetric.Sum().SetIsMonotonic(true)

	counterDataPoint := counterMetric.Sum().DataPoints().AppendEmpty()
	counterDataPoint.SetTimestamp(pcommon.NewTimestampFromTime(timestamp))
	counterDataPoint.SetDoubleValue(10.0)
	counterDataPoint.Attributes().PutStr("foo.bar", "baz")

	counterExemplar := counterDataPoint.Exemplars().AppendEmpty()

	counterExemplar.SetTimestamp(pcommon.NewTimestampFromTime(timestamp))
	counterExemplar.SetDoubleValue(10.0)
	counterExemplar.SetSpanID(pcommon.SpanID{0, 1, 2, 3, 4, 5, 6, 7})
	counterExemplar.SetTraceID(pcommon.TraceID{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15})

	// Generate One Gauge
	gaugeMetric := scopeMetric.Metrics().AppendEmpty()
	gaugeMetric.SetName("test-gauge")
	gaugeMetric.SetDescription("test-gauge-description")
	gaugeMetric.SetEmptyGauge()

	gaugeDataPoint := gaugeMetric.Gauge().DataPoints().AppendEmpty()
	gaugeDataPoint.SetTimestamp(pcommon.NewTimestampFromTime(timestamp))
	gaugeDataPoint.SetDoubleValue(10.0)
	gaugeDataPoint.Attributes().PutStr("foo.bar", "baz")

	// Generate One Histogram
	histogramMetric := scopeMetric.Metrics().AppendEmpty()
	histogramMetric.SetName("test-histogram")
	histogramMetric.SetDescription("test-histogram-description")
	histogramMetric.SetEmptyHistogram()
	histogramMetric.Histogram().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative)

	histogramDataPoint := histogramMetric.Histogram().DataPoints().AppendEmpty()
	histogramDataPoint.SetTimestamp(pcommon.NewTimestampFromTime(timestamp))
	histogramDataPoint.ExplicitBounds().FromRaw([]float64{0.0, 1.0, 2.0, 3.0, 4.0, 5.0})
	histogramDataPoint.BucketCounts().FromRaw([]uint64{2, 2, 2, 2, 2, 2})
	histogramDataPoint.SetCount(10)
	histogramDataPoint.SetSum(30.0)
	histogramDataPoint.Attributes().PutStr("foo.bar", "baz")

	// Generate One Exponential-Histogram
	exponentialHistogramMetric := scopeMetric.Metrics().AppendEmpty()
	exponentialHistogramMetric.SetName("test-exponential-histogram")
	exponentialHistogramMetric.SetDescription("test-exponential-histogram-description")
	exponentialHistogramMetric.SetEmptyExponentialHistogram()
	exponentialHistogramMetric.ExponentialHistogram().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative)

	exponentialHistogramDataPoint := exponentialHistogramMetric.ExponentialHistogram().DataPoints().AppendEmpty()
	exponentialHistogramDataPoint.SetTimestamp(pcommon.NewTimestampFromTime(timestamp))
	exponentialHistogramDataPoint.SetScale(2.0)
	exponentialHistogramDataPoint.Positive().BucketCounts().FromRaw([]uint64{2, 2, 2, 2, 2})
	exponentialHistogramDataPoint.SetZeroCount(2)
	exponentialHistogramDataPoint.SetCount(10)
	exponentialHistogramDataPoint.SetSum(30.0)
	exponentialHistogramDataPoint.Attributes().PutStr("foo.bar", "baz")

	return pmetricotlp.NewExportRequestFromMetrics(d)
}

func TestOTLPDelta(t *testing.T) {
	log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn}))
	appendable := &mockAppendable{}
	cfg := func() config.Config {
		return config.Config{OTLPConfig: config.DefaultOTLPConfig}
	}
	handler := NewOTLPWriteHandler(log, nil, appendable, cfg, OTLPOptions{ConvertDelta: true})

	md := pmetric.NewMetrics()
	ms := md.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics()

	m := ms.AppendEmpty()
	m.SetName("some.delta.total")

	sum := m.SetEmptySum()
	sum.SetAggregationTemporality(pmetric.AggregationTemporalityDelta)

	ts := time.Date(2000, 1, 2, 3, 4, 0, 0, time.UTC)
	for i := range 3 {
		dp := sum.DataPoints().AppendEmpty()
		dp.SetIntValue(int64(i))
		dp.SetTimestamp(pcommon.NewTimestampFromTime(ts.Add(time.Duration(i) * time.Second)))
	}

	proto, err := pmetricotlp.NewExportRequestFromMetrics(md).MarshalProto()
	require.NoError(t, err)

	req, err := http.NewRequest("", "", bytes.NewReader(proto))
	require.NoError(t, err)
	req.Header.Set("Content-Type", "application/x-protobuf")

	rec := httptest.NewRecorder()
	handler.ServeHTTP(rec, req)
	require.Equal(t, http.StatusOK, rec.Result().StatusCode)

	ls := labels.FromStrings("__name__", "some_delta_total")
	milli := func(sec int) int64 {
		return time.Date(2000, 1, 2, 3, 4, sec, 0, time.UTC).UnixMilli()
	}

	want := []mockSample{
		{t: milli(0), l: ls, v: 0}, // +0
		{t: milli(1), l: ls, v: 1}, // +1
		{t: milli(2), l: ls, v: 3}, // +2
	}
	if diff := cmp.Diff(want, appendable.samples, cmp.Exporter(func(_ reflect.Type) bool { return true })); diff != "" {
		t.Fatal(diff)
	}
}

func BenchmarkOTLP(b *testing.B) {
	start := time.Date(2000, 1, 2, 3, 4, 5, 0, time.UTC)

	type Type struct {
		name string
		data func(mode pmetric.AggregationTemporality, dpc, epoch int) []pmetric.Metric
	}
	types := []Type{{
		name: "sum",
		data: func() func(mode pmetric.AggregationTemporality, dpc, epoch int) []pmetric.Metric {
			cumul := make(map[int]float64)
			return func(mode pmetric.AggregationTemporality, dpc, epoch int) []pmetric.Metric {
				m := pmetric.NewMetric()
				sum := m.SetEmptySum()
				sum.SetAggregationTemporality(mode)
				dps := sum.DataPoints()
				for id := range dpc {
					dp := dps.AppendEmpty()
					dp.SetStartTimestamp(pcommon.NewTimestampFromTime(start))
					dp.SetTimestamp(pcommon.NewTimestampFromTime(start.Add(time.Duration(epoch) * time.Minute)))
					dp.Attributes().PutStr("id", strconv.Itoa(id))
					v := float64(rand.IntN(100)) / 10
					switch mode {
					case pmetric.AggregationTemporalityDelta:
						dp.SetDoubleValue(v)
					case pmetric.AggregationTemporalityCumulative:
						cumul[id] += v
						dp.SetDoubleValue(cumul[id])
					}
				}
				return []pmetric.Metric{m}
			}
		}(),
	}, {
		name: "histogram",
		data: func() func(mode pmetric.AggregationTemporality, dpc, epoch int) []pmetric.Metric {
			bounds := [4]float64{1, 10, 100, 1000}
			type state struct {
				counts [4]uint64
				count  uint64
				sum    float64
			}
			var cumul []state
			return func(mode pmetric.AggregationTemporality, dpc, epoch int) []pmetric.Metric {
				if cumul == nil {
					cumul = make([]state, dpc)
				}
				m := pmetric.NewMetric()
				hist := m.SetEmptyHistogram()
				hist.SetAggregationTemporality(mode)
				dps := hist.DataPoints()
				for id := range dpc {
					dp := dps.AppendEmpty()
					dp.SetStartTimestamp(pcommon.NewTimestampFromTime(start))
					dp.SetTimestamp(pcommon.NewTimestampFromTime(start.Add(time.Duration(epoch) * time.Minute)))
					dp.Attributes().PutStr("id", strconv.Itoa(id))
					dp.ExplicitBounds().FromRaw(bounds[:])

					var obs *state
					switch mode {
					case pmetric.AggregationTemporalityDelta:
						obs = new(state)
					case pmetric.AggregationTemporalityCumulative:
						obs = &cumul[id]
					}

					for i := range obs.counts {
						v := uint64(rand.IntN(10))
						obs.counts[i] += v
						obs.count++
						obs.sum += float64(v)
					}

					dp.SetCount(obs.count)
					dp.SetSum(obs.sum)
					dp.BucketCounts().FromRaw(obs.counts[:])
				}
				return []pmetric.Metric{m}
			}
		}(),
	}, {
		name: "exponential",
		data: func() func(mode pmetric.AggregationTemporality, dpc, epoch int) []pmetric.Metric {
			type state struct {
				counts [4]uint64
				count  uint64
				sum    float64
			}
			var cumul []state
			return func(mode pmetric.AggregationTemporality, dpc, epoch int) []pmetric.Metric {
				if cumul == nil {
					cumul = make([]state, dpc)
				}
				m := pmetric.NewMetric()
				ex := m.SetEmptyExponentialHistogram()
				ex.SetAggregationTemporality(mode)
				dps := ex.DataPoints()
				for id := range dpc {
					dp := dps.AppendEmpty()
					dp.SetStartTimestamp(pcommon.NewTimestampFromTime(start))
					dp.SetTimestamp(pcommon.NewTimestampFromTime(start.Add(time.Duration(epoch) * time.Minute)))
					dp.Attributes().PutStr("id", strconv.Itoa(id))
					dp.SetScale(2)

					var obs *state
					switch mode {
					case pmetric.AggregationTemporalityDelta:
						obs = new(state)
					case pmetric.AggregationTemporalityCumulative:
						obs = &cumul[id]
					}

					for i := range obs.counts {
						v := uint64(rand.IntN(10))
						obs.counts[i] += v
						obs.count++
						obs.sum += float64(v)
					}

					dp.Positive().BucketCounts().FromRaw(obs.counts[:])
					dp.SetCount(obs.count)
					dp.SetSum(obs.sum)
				}

				return []pmetric.Metric{m}
			}
		}(),
	}}

	modes := []struct {
		name string
		data func(func(pmetric.AggregationTemporality, int, int) []pmetric.Metric, int) []pmetric.Metric
	}{{
		name: "cumulative",
		data: func(data func(pmetric.AggregationTemporality, int, int) []pmetric.Metric, epoch int) []pmetric.Metric {
			return data(pmetric.AggregationTemporalityCumulative, 10, epoch)
		},
	}, {
		name: "delta",
		data: func(data func(pmetric.AggregationTemporality, int, int) []pmetric.Metric, epoch int) []pmetric.Metric {
			return data(pmetric.AggregationTemporalityDelta, 10, epoch)
		},
	}, {
		name: "mixed",
		data: func(data func(pmetric.AggregationTemporality, int, int) []pmetric.Metric, epoch int) []pmetric.Metric {
			cumul := data(pmetric.AggregationTemporalityCumulative, 5, epoch)
			delta := data(pmetric.AggregationTemporalityDelta, 5, epoch)
			out := append(cumul, delta...)
			rand.Shuffle(len(out), func(i, j int) { out[i], out[j] = out[j], out[i] })
			return out
		},
	}}

	configs := []struct {
		name string
		opts OTLPOptions
	}{
		{name: "default"},
		{name: "convert", opts: OTLPOptions{ConvertDelta: true}},
	}

	Workers := runtime.GOMAXPROCS(0)
	for _, cs := range types {
		for _, mode := range modes {
			for _, cfg := range configs {
				b.Run(fmt.Sprintf("type=%s/temporality=%s/cfg=%s", cs.name, mode.name, cfg.name), func(b *testing.B) {
					if !cfg.opts.ConvertDelta && (mode.name == "delta" || mode.name == "mixed") {
						b.Skip("not possible")
					}

					var total int

					// reqs is a [b.N]*http.Request, divided across the workers.
					// deltatocumulative requires timestamps to be strictly in
					// order on a per-series basis. to ensure this, each reqs[k]
					// contains samples of differently named series, sorted
					// strictly in time order
					reqs := make([][]*http.Request, Workers)
					for n := range b.N {
						k := n % Workers

						md := pmetric.NewMetrics()
						ms := md.ResourceMetrics().AppendEmpty().
							ScopeMetrics().AppendEmpty().
							Metrics()

						for i, m := range mode.data(cs.data, n) {
							m.SetName(fmt.Sprintf("benchmark_%d_%d", k, i))
							m.MoveTo(ms.AppendEmpty())
						}

						total += sampleCount(md)

						ex := pmetricotlp.NewExportRequestFromMetrics(md)
						data, err := ex.MarshalProto()
						require.NoError(b, err)

						req, err := http.NewRequest("", "", bytes.NewReader(data))
						require.NoError(b, err)
						req.Header.Set("Content-Type", "application/x-protobuf")

						reqs[k] = append(reqs[k], req)
					}

					log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn}))
					mock := new(mockAppendable)
					appendable := syncAppendable{Appendable: mock, lock: new(sync.Mutex)}
					cfgfn := func() config.Config {
						return config.Config{OTLPConfig: config.DefaultOTLPConfig}
					}
					handler := NewOTLPWriteHandler(log, nil, appendable, cfgfn, cfg.opts)

					fail := make(chan struct{})
					done := make(chan struct{})

					b.ResetTimer()
					b.ReportAllocs()

					// we use multiple workers to mimic a real-world scenario
					// where multiple OTel collectors are sending their
					// time-series in parallel.
					// this is necessary to exercise potential lock-contention
					// in this benchmark
					for k := range Workers {
						go func() {
							rec := httptest.NewRecorder()
							for _, req := range reqs[k] {
								handler.ServeHTTP(rec, req)
								if rec.Result().StatusCode != http.StatusOK {
									fail <- struct{}{}
									return
								}
							}
							done <- struct{}{}
						}()
					}

					for range Workers {
						select {
						case <-fail:
							b.FailNow()
						case <-done:
						}
					}

					require.Equal(b, total, len(mock.samples)+len(mock.histograms))
				})
			}
		}
	}
}

func sampleCount(md pmetric.Metrics) int {
	var total int
	rms := md.ResourceMetrics()
	for i := range rms.Len() {
		sms := rms.At(i).ScopeMetrics()
		for i := range sms.Len() {
			ms := sms.At(i).Metrics()
			for i := range ms.Len() {
				m := ms.At(i)
				switch m.Type() {
				case pmetric.MetricTypeSum:
					total += m.Sum().DataPoints().Len()
				case pmetric.MetricTypeGauge:
					total += m.Gauge().DataPoints().Len()
				case pmetric.MetricTypeHistogram:
					dps := m.Histogram().DataPoints()
					for i := range dps.Len() {
						total += dps.At(i).BucketCounts().Len()
						total++ // le=+Inf series
						total++ // _sum series
						total++ // _count series
					}
				case pmetric.MetricTypeExponentialHistogram:
					total += m.ExponentialHistogram().DataPoints().Len()
				case pmetric.MetricTypeSummary:
					total += m.Summary().DataPoints().Len()
				}
			}
		}
	}
	return total
}

type syncAppendable struct {
	lock sync.Locker
	storage.Appendable
}

type syncAppender struct {
	lock sync.Locker
	storage.Appender
}

func (s syncAppendable) Appender(ctx context.Context) storage.Appender {
	return syncAppender{Appender: s.Appendable.Appender(ctx), lock: s.lock}
}

func (s syncAppender) Append(ref storage.SeriesRef, l labels.Labels, t int64, v float64) (storage.SeriesRef, error) {
	s.lock.Lock()
	defer s.lock.Unlock()
	return s.Appender.Append(ref, l, t, v)
}

func (s syncAppender) AppendHistogram(ref storage.SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, f *histogram.FloatHistogram) (storage.SeriesRef, error) {
	s.lock.Lock()
	defer s.lock.Unlock()
	return s.Appender.AppendHistogram(ref, l, t, h, f)
}