diff --git a/CHANGELOG.md b/CHANGELOG.md index ead292fc7d..9f9050cc96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## unreleased +* [ENHANCEMENT] OTLP receiver: Convert also metric metadata. #15416 + ## 3.0.0 / 2024-11-14 This release includes new features such as a brand new UI and UTF-8 support enabled by default. As this marks the first new major version in seven years, several breaking changes are introduced. The breaking changes are mainly around the removal of deprecated feature flags and CLI arguments, and the full list can be found below. For users that want to upgrade we recommend to read through our [migration guide](https://prometheus.io/docs/prometheus/3.0/migration/). diff --git a/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go b/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go index 4f8baf3100..63862c4a70 100644 --- a/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go +++ b/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go @@ -37,7 +37,6 @@ type Settings struct { DisableTargetInfo bool ExportCreatedMetric bool AddMetricSuffixes bool - SendMetadata bool AllowUTF8 bool PromoteResourceAttributes []string } @@ -47,6 +46,7 @@ type PrometheusConverter struct { unique map[uint64]*prompb.TimeSeries conflicts map[uint64][]*prompb.TimeSeries everyN everyNTimes + metadata []prompb.MetricMetadata } func NewPrometheusConverter() *PrometheusConverter { @@ -60,6 +60,16 @@ func NewPrometheusConverter() *PrometheusConverter { func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metrics, settings Settings) (annots annotations.Annotations, errs error) { c.everyN = everyNTimes{n: 128} resourceMetricsSlice := md.ResourceMetrics() + + numMetrics := 0 + for i := 0; i < resourceMetricsSlice.Len(); i++ { + scopeMetricsSlice := resourceMetricsSlice.At(i).ScopeMetrics() + for j := 0; j < scopeMetricsSlice.Len(); j++ { + numMetrics += scopeMetricsSlice.At(j).Metrics().Len() + } + } + c.metadata = make([]prompb.MetricMetadata, 0, numMetrics) + for i := 0; i < resourceMetricsSlice.Len(); i++ { resourceMetrics := resourceMetricsSlice.At(i) resource := resourceMetrics.Resource() @@ -86,6 +96,12 @@ func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metric } promName := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes, settings.AllowUTF8) + c.metadata = append(c.metadata, prompb.MetricMetadata{ + Type: otelMetricTypeToPromMetricType(metric), + MetricFamilyName: promName, + Help: metric.Description(), + Unit: metric.Unit(), + }) // handle individual metrics based on type //exhaustive:enforce diff --git a/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw_test.go b/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw_test.go index 641437632d..64d0ebd6f5 100644 --- a/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw_test.go +++ b/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw_test.go @@ -22,20 +22,46 @@ import ( "testing" "time" + "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" "go.opentelemetry.io/collector/pdata/pmetric/pmetricotlp" + + "github.com/prometheus/prometheus/prompb" + prometheustranslator "github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus" ) func TestFromMetrics(t *testing.T) { t.Run("successful", func(t *testing.T) { converter := NewPrometheusConverter() payload := createExportRequest(5, 128, 128, 2, 0) + var expMetadata []prompb.MetricMetadata + resourceMetricsSlice := payload.Metrics().ResourceMetrics() + for i := 0; i < resourceMetricsSlice.Len(); i++ { + scopeMetricsSlice := resourceMetricsSlice.At(i).ScopeMetrics() + for j := 0; j < scopeMetricsSlice.Len(); j++ { + metricSlice := scopeMetricsSlice.At(j).Metrics() + for k := 0; k < metricSlice.Len(); k++ { + metric := metricSlice.At(k) + promName := prometheustranslator.BuildCompliantName(metric, "", false, false) + expMetadata = append(expMetadata, prompb.MetricMetadata{ + Type: otelMetricTypeToPromMetricType(metric), + MetricFamilyName: promName, + Help: metric.Description(), + Unit: metric.Unit(), + }) + } + } + } annots, err := converter.FromMetrics(context.Background(), payload.Metrics(), Settings{}) require.NoError(t, err) require.Empty(t, annots) + + if diff := cmp.Diff(expMetadata, converter.Metadata()); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } }) t.Run("context cancellation", func(t *testing.T) { @@ -115,13 +141,15 @@ func BenchmarkPrometheusConverter_FromMetrics(b *testing.B) { for _, exemplarsPerSeries := range []int{0, 5, 10} { b.Run(fmt.Sprintf("exemplars per series: %v", exemplarsPerSeries), func(b *testing.B) { payload := createExportRequest(resourceAttributeCount, histogramCount, nonHistogramCount, labelsPerMetric, exemplarsPerSeries) + b.ResetTimer() - for i := 0; i < b.N; i++ { + for range b.N { converter := NewPrometheusConverter() annots, err := converter.FromMetrics(context.Background(), payload.Metrics(), Settings{}) require.NoError(b, err) require.Empty(b, annots) require.NotNil(b, converter.TimeSeries()) + require.NotNil(b, converter.Metadata()) } }) } @@ -148,6 +176,8 @@ func createExportRequest(resourceAttributeCount, histogramCount, nonHistogramCou m := metrics.AppendEmpty() m.SetEmptyHistogram() m.SetName(fmt.Sprintf("histogram-%v", i)) + m.SetDescription("histogram") + m.SetUnit("unit") m.Histogram().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) h := m.Histogram().DataPoints().AppendEmpty() h.SetTimestamp(ts) @@ -166,6 +196,8 @@ func createExportRequest(resourceAttributeCount, histogramCount, nonHistogramCou m := metrics.AppendEmpty() m.SetEmptySum() m.SetName(fmt.Sprintf("sum-%v", i)) + m.SetDescription("sum") + m.SetUnit("unit") m.Sum().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) point := m.Sum().DataPoints().AppendEmpty() point.SetTimestamp(ts) @@ -178,6 +210,8 @@ func createExportRequest(resourceAttributeCount, histogramCount, nonHistogramCou m := metrics.AppendEmpty() m.SetEmptyGauge() m.SetName(fmt.Sprintf("gauge-%v", i)) + m.SetDescription("gauge") + m.SetUnit("unit") point := m.Gauge().DataPoints().AppendEmpty() point.SetTimestamp(ts) point.SetDoubleValue(1.23) diff --git a/storage/remote/otlptranslator/prometheusremotewrite/otlp_to_openmetrics_metadata.go b/storage/remote/otlptranslator/prometheusremotewrite/otlp_to_openmetrics_metadata.go index b423d2cc6e..359fc52522 100644 --- a/storage/remote/otlptranslator/prometheusremotewrite/otlp_to_openmetrics_metadata.go +++ b/storage/remote/otlptranslator/prometheusremotewrite/otlp_to_openmetrics_metadata.go @@ -20,7 +20,6 @@ import ( "go.opentelemetry.io/collector/pdata/pmetric" "github.com/prometheus/prometheus/prompb" - prometheustranslator "github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus" ) func otelMetricTypeToPromMetricType(otelMetric pmetric.Metric) prompb.MetricMetadata_MetricType { @@ -42,36 +41,3 @@ func otelMetricTypeToPromMetricType(otelMetric pmetric.Metric) prompb.MetricMeta } return prompb.MetricMetadata_UNKNOWN } - -func OtelMetricsToMetadata(md pmetric.Metrics, addMetricSuffixes, allowUTF8 bool) []*prompb.MetricMetadata { - resourceMetricsSlice := md.ResourceMetrics() - - metadataLength := 0 - for i := 0; i < resourceMetricsSlice.Len(); i++ { - scopeMetricsSlice := resourceMetricsSlice.At(i).ScopeMetrics() - for j := 0; j < scopeMetricsSlice.Len(); j++ { - metadataLength += scopeMetricsSlice.At(j).Metrics().Len() - } - } - - var metadata = make([]*prompb.MetricMetadata, 0, metadataLength) - for i := 0; i < resourceMetricsSlice.Len(); i++ { - resourceMetrics := resourceMetricsSlice.At(i) - scopeMetricsSlice := resourceMetrics.ScopeMetrics() - - for j := 0; j < scopeMetricsSlice.Len(); j++ { - scopeMetrics := scopeMetricsSlice.At(j) - for k := 0; k < scopeMetrics.Metrics().Len(); k++ { - metric := scopeMetrics.Metrics().At(k) - entry := prompb.MetricMetadata{ - Type: otelMetricTypeToPromMetricType(metric), - MetricFamilyName: prometheustranslator.BuildCompliantName(metric, "", addMetricSuffixes, allowUTF8), - Help: metric.Description(), - } - metadata = append(metadata, &entry) - } - } - } - - return metadata -} diff --git a/storage/remote/otlptranslator/prometheusremotewrite/timeseries.go b/storage/remote/otlptranslator/prometheusremotewrite/timeseries.go index fe973761a2..abffbe6105 100644 --- a/storage/remote/otlptranslator/prometheusremotewrite/timeseries.go +++ b/storage/remote/otlptranslator/prometheusremotewrite/timeseries.go @@ -39,3 +39,8 @@ func (c *PrometheusConverter) TimeSeries() []prompb.TimeSeries { return allTS } + +// Metadata returns a slice of the prompb.Metadata that were converted from OTel format. +func (c *PrometheusConverter) Metadata() []prompb.MetricMetadata { + return c.metadata +} diff --git a/storage/remote/write_handler.go b/storage/remote/write_handler.go index 87102a374b..f719ea6a1f 100644 --- a/storage/remote/write_handler.go +++ b/storage/remote/write_handler.go @@ -526,6 +526,7 @@ func (h *otlpWriteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { err = h.rwHandler.write(r.Context(), &prompb.WriteRequest{ Timeseries: converter.TimeSeries(), + Metadata: converter.Metadata(), }) switch {