diff --git a/config/config.go b/config/config.go index a395c3f53e..f47cc0d31a 100644 --- a/config/config.go +++ b/config/config.go @@ -110,7 +110,7 @@ func Load(s string, logger *slog.Logger) (*Config, error) { switch cfg.OTLPConfig.TranslationStrategy { case UnderscoreEscapingWithSuffixes: case "": - case NoUTF8EscapingWithSuffixes: + case NoTranslation, NoUTF8EscapingWithSuffixes: if cfg.GlobalConfig.MetricNameValidationScheme == LegacyValidationConfig { return nil, errors.New("OTLP translation strategy NoUTF8EscapingWithSuffixes is not allowed when UTF8 is disabled") } @@ -1436,6 +1436,15 @@ var ( // and label name characters that are not alphanumerics/underscores to underscores. // Unit and type suffixes may be appended to metric names, according to certain rules. UnderscoreEscapingWithSuffixes translationStrategyOption = "UnderscoreEscapingWithSuffixes" + // NoTranslation (EXPERIMENTAL): disables all translation of incoming metric + // and label names. Note that because metrics in Open Telemetry are considered + // distinct if they share the same name but have different Type or Units, for + // instance "foo.bar" with units Seconds is a separate series from "foo.bar" + // with units Milliseconds. Because prometheus does not yet support type and + // unit metadata, these two series would be conflated in Prometheus. + // Therefore this setting is experimental and should not be used in + // production systems. + NoTranslation translationStrategyOption = "NoTranslation" ) // OTLPConfig is the configuration for writing to the OTLP endpoint. diff --git a/config/config_test.go b/config/config_test.go index faca7dda12..9928f38204 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1590,6 +1590,26 @@ func TestOTLPAllowUTF8(t *testing.T) { }) }) + t.Run("good config, no translation", func(t *testing.T) { + fpath := filepath.Join("testdata", "otlp_no_translation.good.yml") + verify := func(t *testing.T, conf *Config, err error) { + t.Helper() + require.NoError(t, err) + require.Equal(t, NoTranslation, conf.OTLPConfig.TranslationStrategy) + } + + t.Run("LoadFile", func(t *testing.T) { + conf, err := LoadFile(fpath, false, promslog.NewNopLogger()) + verify(t, conf, err) + }) + t.Run("Load", func(t *testing.T) { + content, err := os.ReadFile(fpath) + require.NoError(t, err) + conf, err := Load(string(content), promslog.NewNopLogger()) + verify(t, conf, err) + }) + }) + t.Run("incompatible config", func(t *testing.T) { fpath := filepath.Join("testdata", "otlp_allow_utf8.incompatible.yml") verify := func(t *testing.T, err error) { diff --git a/config/testdata/otlp_no_translation.good.yml b/config/testdata/otlp_no_translation.good.yml new file mode 100644 index 0000000000..e5c4460842 --- /dev/null +++ b/config/testdata/otlp_no_translation.good.yml @@ -0,0 +1,2 @@ +otlp: + translation_strategy: NoTranslation diff --git a/storage/remote/write_handler.go b/storage/remote/write_handler.go index 21e3693e50..5e03d29643 100644 --- a/storage/remote/write_handler.go +++ b/storage/remote/write_handler.go @@ -577,8 +577,8 @@ func (rw *rwExporter) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) er converter := otlptranslator.NewPrometheusConverter() annots, err := converter.FromMetrics(ctx, md, otlptranslator.Settings{ - AddMetricSuffixes: true, - AllowUTF8: otlpCfg.TranslationStrategy == config.NoUTF8EscapingWithSuffixes, + AddMetricSuffixes: otlpCfg.TranslationStrategy != config.NoTranslation, + AllowUTF8: otlpCfg.TranslationStrategy != config.UnderscoreEscapingWithSuffixes, PromoteResourceAttributes: otlpCfg.PromoteResourceAttributes, KeepIdentifyingResourceAttributes: otlpCfg.KeepIdentifyingResourceAttributes, }) diff --git a/storage/remote/write_handler_test.go b/storage/remote/write_handler_test.go index 5bf6f5632e..4cd611106f 100644 --- a/storage/remote/write_handler_test.go +++ b/storage/remote/write_handler_test.go @@ -844,6 +844,19 @@ func requireEqual(t *testing.T, expected, actual interface{}, msgAndArgs ...inte msgAndArgs...) } +func requireContainsSample(t *testing.T, actual []mockSample, expected mockSample) { + t.Helper() + + for _, got := range actual { + if labels.Equal(expected.l, got.l) && expected.t == got.t && expected.v == got.v { + return + } + } + require.Fail(t, fmt.Sprintf("Sample not found: \n"+ + "expected: %v\n"+ + "actual : %v", expected, actual)) +} + func (m *mockAppendable) Appender(_ context.Context) storage.Appender { if m.latestSample == nil { m.latestSample = map[uint64]int64{} diff --git a/storage/remote/write_test.go b/storage/remote/write_test.go index a3b30b6425..49da9db787 100644 --- a/storage/remote/write_test.go +++ b/storage/remote/write_test.go @@ -382,7 +382,44 @@ func TestWriteStorageApplyConfig_PartialUpdate(t *testing.T) { func TestOTLPWriteHandler(t *testing.T) { exportRequest := generateOTLPWriteRequest() + resp, appendable := handleOtlp(t, exportRequest) + require.Equal(t, http.StatusOK, resp.StatusCode) + require.Len(t, appendable.samples, 12) // 1 (counter) + 1 (gauge) + 1 (target_info) + 7 (hist_bucket) + 2 (hist_sum, hist_count) + require.Len(t, appendable.histograms, 1) // 1 (exponential histogram) + require.Len(t, appendable.exemplars, 1) // 1 (exemplar) +} + +func TestOTLPWriteHandlerNoTranslation(t *testing.T) { + timestamp := time.Now() + exportRequest := generateCounterOTLPWriteRequest(timestamp) + resp, appendable := handleOtlp(t, exportRequest) + require.Equal(t, http.StatusOK, resp.StatusCode) + + requireContainsSample(t, appendable.samples, mockSample{ + l: labels.New( + labels.Label{Name: "__name__", Value: "test.counter"}, + labels.Label{Name: "foo.bar", Value: "baz"}, + labels.Label{Name: "instance", Value: "test-instance"}, + labels.Label{Name: "job", Value: "test-service"}, + ), + t: timestamp.UnixMilli(), + v: 10, + }) + + requireContainsSample(t, appendable.samples, mockSample{ + l: labels.New( + labels.Label{Name: "__name__", Value: "target_info"}, + labels.Label{Name: "host.name", Value: "test-host"}, + labels.Label{Name: "instance", Value: "test-instance"}, + labels.Label{Name: "job", Value: "test-service"}, + ), + t: timestamp.UnixMilli(), + v: 1, + }) +} + +func handleOtlp(t *testing.T, exportRequest pmetricotlp.ExportRequest) (*http.Response, *mockAppendable) { buf, err := exportRequest.MarshalProto() require.NoError(t, err) @@ -391,9 +428,11 @@ func TestOTLPWriteHandler(t *testing.T) { req.Header.Set("Content-Type", "application/x-protobuf") appendable := &mockAppendable{} + conf := config.DefaultOTLPConfig + conf.TranslationStrategy = config.NoTranslation handler := NewOTLPWriteHandler(nil, nil, appendable, func() config.Config { return config.Config{ - OTLPConfig: config.DefaultOTLPConfig, + OTLPConfig: conf, } }, OTLPOptions{}) @@ -401,11 +440,39 @@ func TestOTLPWriteHandler(t *testing.T) { handler.ServeHTTP(recorder, req) resp := recorder.Result() - require.Equal(t, http.StatusOK, resp.StatusCode) + return resp, appendable +} - require.Len(t, appendable.samples, 12) // 1 (counter) + 1 (gauge) + 1 (target_info) + 7 (hist_bucket) + 2 (hist_sum, hist_count) - require.Len(t, appendable.histograms, 1) // 1 (exponential histogram) - require.Len(t, appendable.exemplars, 1) // 1 (exemplar) +func generateCounterOTLPWriteRequest(timestamp time.Time) pmetricotlp.ExportRequest { + d := pmetric.NewMetrics() + + resourceMetric := d.ResourceMetrics().AppendEmpty() + resourceMetric.Resource().Attributes().PutStr("service.name", "test-service") + resourceMetric.Resource().Attributes().PutStr("service.instance.id", "test-instance") + resourceMetric.Resource().Attributes().PutStr("host.name", "test-host") + + scopeMetric := resourceMetric.ScopeMetrics().AppendEmpty() + + counterMetric := scopeMetric.Metrics().AppendEmpty() + counterMetric.SetName("test.counter") + counterMetric.SetDescription("test-counter-description") + counterMetric.SetEmptySum() + counterMetric.Sum().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + counterMetric.Sum().SetIsMonotonic(true) + + counterDataPoint := counterMetric.Sum().DataPoints().AppendEmpty() + counterDataPoint.SetTimestamp(pcommon.NewTimestampFromTime(timestamp)) + counterDataPoint.SetDoubleValue(10.0) + counterDataPoint.Attributes().PutStr("foo.bar", "baz") + + counterExemplar := counterDataPoint.Exemplars().AppendEmpty() + + counterExemplar.SetTimestamp(pcommon.NewTimestampFromTime(timestamp)) + counterExemplar.SetDoubleValue(10.0) + counterExemplar.SetSpanID(pcommon.SpanID{0, 1, 2, 3, 4, 5, 6, 7}) + counterExemplar.SetTraceID(pcommon.TraceID{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}) + + return pmetricotlp.NewExportRequestFromMetrics(d) } func generateOTLPWriteRequest() pmetricotlp.ExportRequest { @@ -426,7 +493,7 @@ func generateOTLPWriteRequest() pmetricotlp.ExportRequest { // Generate One Counter counterMetric := scopeMetric.Metrics().AppendEmpty() - counterMetric.SetName("test-counter") + counterMetric.SetName("test.counter") counterMetric.SetDescription("test-counter-description") counterMetric.SetEmptySum() counterMetric.Sum().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative)