otlp/translator: Use separate function for metric names with UTF8 characters (#15664)

BuildCompliantName was renamed to BuildCompliantMetricName, and it no longer takes UTF8 support into consideration. It focuses on building a metric name that follows Prometheus conventions.

A new function, BuildMetricName, was added to optionally add unit and type suffixes to OTLP metric names without translating any characters to underscores(_).
This commit is contained in:
Arthur Silva Sens 2025-01-06 11:30:39 -03:00 committed by GitHub
parent 56094197b5
commit 5fdec31401
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 391 additions and 404 deletions

View file

@ -1,106 +0,0 @@
// Copyright 2024 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Provenance-includes-location: https://github.com/golang/go/blob/f2d118fd5f7e872804a5825ce29797f81a28b0fa/src/strings/strings.go
// Provenance-includes-license: BSD-3-Clause
// Provenance-includes-copyright: Copyright The Go Authors.
package prometheus
import "strings"
// fieldsFunc is a copy of strings.FieldsFunc from the Go standard library,
// but it also returns the separators as part of the result.
func fieldsFunc(s string, f func(rune) bool) ([]string, []string) {
// A span is used to record a slice of s of the form s[start:end].
// The start index is inclusive and the end index is exclusive.
type span struct {
start int
end int
}
spans := make([]span, 0, 32)
separators := make([]string, 0, 32)
// Find the field start and end indices.
// Doing this in a separate pass (rather than slicing the string s
// and collecting the result substrings right away) is significantly
// more efficient, possibly due to cache effects.
start := -1 // valid span start if >= 0
for end, rune := range s {
if f(rune) {
if start >= 0 {
spans = append(spans, span{start, end})
// Set start to a negative value.
// Note: using -1 here consistently and reproducibly
// slows down this code by a several percent on amd64.
start = ^start
separators = append(separators, string(s[end]))
}
} else {
if start < 0 {
start = end
}
}
}
// Last field might end at EOF.
if start >= 0 {
spans = append(spans, span{start, len(s)})
}
// Create strings from recorded field indices.
a := make([]string, len(spans))
for i, span := range spans {
a[i] = s[span.start:span.end]
}
return a, separators
}
// join is a copy of strings.Join from the Go standard library,
// but it also accepts a slice of separators to join the elements with.
// If the slice of separators is shorter than the slice of elements, use a default value.
// We also don't check for integer overflow.
func join(elems []string, separators []string, def string) string {
switch len(elems) {
case 0:
return ""
case 1:
return elems[0]
}
var n int
var sep string
sepLen := len(separators)
for i, elem := range elems {
if i >= sepLen {
sep = def
} else {
sep = separators[i]
}
n += len(sep) + len(elem)
}
var b strings.Builder
b.Grow(n)
b.WriteString(elems[0])
for i, s := range elems[1:] {
if i >= sepLen {
sep = def
} else {
sep = separators[i]
}
b.WriteString(sep)
b.WriteString(s)
}
return b.String()
}

View file

@ -78,7 +78,7 @@ var perUnitMap = map[string]string{
"y": "year",
}
// BuildCompliantName builds a Prometheus-compliant metric name for the specified metric.
// BuildCompliantMetricName builds a Prometheus-compliant metric name for the specified metric.
//
// Metric name is prefixed with specified namespace and underscore (if any).
// Namespace is not cleaned up. Make sure specified namespace follows Prometheus
@ -87,29 +87,24 @@ var perUnitMap = map[string]string{
// See rules at https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels,
// https://prometheus.io/docs/practices/naming/#metric-and-label-naming
// and https://github.com/open-telemetry/opentelemetry-specification/blob/v1.38.0/specification/compatibility/prometheus_and_openmetrics.md#otlp-metric-points-to-prometheus.
func BuildCompliantName(metric pmetric.Metric, namespace string, addMetricSuffixes, allowUTF8 bool) string {
func BuildCompliantMetricName(metric pmetric.Metric, namespace string, addMetricSuffixes bool) string {
// Full normalization following standard Prometheus naming conventions
if addMetricSuffixes {
return normalizeName(metric, namespace, allowUTF8)
return normalizeName(metric, namespace)
}
var metricName string
if !allowUTF8 {
// Simple case (no full normalization, no units, etc.).
metricName = strings.Join(strings.FieldsFunc(metric.Name(), func(r rune) bool {
return invalidMetricCharRE.MatchString(string(r))
}), "_")
} else {
metricName = metric.Name()
}
// Simple case (no full normalization, no units, etc.).
metricName := strings.Join(strings.FieldsFunc(metric.Name(), func(r rune) bool {
return invalidMetricCharRE.MatchString(string(r))
}), "_")
// Namespace?
if namespace != "" {
return namespace + "_" + metricName
}
// Metric name starts with a digit and utf8 not allowed? Prefix it with an underscore.
if metricName != "" && unicode.IsDigit(rune(metricName[0])) && !allowUTF8 {
// Metric name starts with a digit? Prefix it with an underscore.
if metricName != "" && unicode.IsDigit(rune(metricName[0])) {
metricName = "_" + metricName
}
@ -124,70 +119,17 @@ var (
)
// Build a normalized name for the specified metric.
func normalizeName(metric pmetric.Metric, namespace string, allowUTF8 bool) string {
var nameTokens []string
var separators []string
if !allowUTF8 {
// Split metric name into "tokens" (of supported metric name runes).
// Note that this has the side effect of replacing multiple consecutive underscores with a single underscore.
// This is part of the OTel to Prometheus specification: https://github.com/open-telemetry/opentelemetry-specification/blob/v1.38.0/specification/compatibility/prometheus_and_openmetrics.md#otlp-metric-points-to-prometheus.
nameTokens = strings.FieldsFunc(
metric.Name(),
func(r rune) bool { return nonMetricNameCharRE.MatchString(string(r)) },
)
} else {
translationFunc := func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) && r != ':' }
// Split metric name into "tokens" (of supported metric name runes).
nameTokens, separators = fieldsFunc(metric.Name(), translationFunc)
}
func normalizeName(metric pmetric.Metric, namespace string) string {
// Split metric name into "tokens" (of supported metric name runes).
// Note that this has the side effect of replacing multiple consecutive underscores with a single underscore.
// This is part of the OTel to Prometheus specification: https://github.com/open-telemetry/opentelemetry-specification/blob/v1.38.0/specification/compatibility/prometheus_and_openmetrics.md#otlp-metric-points-to-prometheus.
nameTokens := strings.FieldsFunc(
metric.Name(),
func(r rune) bool { return nonMetricNameCharRE.MatchString(string(r)) },
)
// Split unit at the '/' if any
unitTokens := strings.SplitN(metric.Unit(), "/", 2)
// Main unit
// Append if not blank, doesn't contain '{}', and is not present in metric name already
if len(unitTokens) > 0 {
var mainUnitProm, perUnitProm string
mainUnitOTel := strings.TrimSpace(unitTokens[0])
if mainUnitOTel != "" && !strings.ContainsAny(mainUnitOTel, "{}") {
mainUnitProm = unitMapGetOrDefault(mainUnitOTel)
if !allowUTF8 {
mainUnitProm = cleanUpUnit(mainUnitProm)
}
if slices.Contains(nameTokens, mainUnitProm) {
mainUnitProm = ""
}
}
// Per unit
// Append if not blank, doesn't contain '{}', and is not present in metric name already
if len(unitTokens) > 1 && unitTokens[1] != "" {
perUnitOTel := strings.TrimSpace(unitTokens[1])
if perUnitOTel != "" && !strings.ContainsAny(perUnitOTel, "{}") {
perUnitProm = perUnitMapGetOrDefault(perUnitOTel)
if !allowUTF8 {
perUnitProm = cleanUpUnit(perUnitProm)
}
}
if perUnitProm != "" {
perUnitProm = "per_" + perUnitProm
if slices.Contains(nameTokens, perUnitProm) {
perUnitProm = ""
}
}
}
if perUnitProm != "" {
mainUnitProm = strings.TrimSuffix(mainUnitProm, "_")
}
if mainUnitProm != "" {
nameTokens = append(nameTokens, mainUnitProm)
}
if perUnitProm != "" {
nameTokens = append(nameTokens, perUnitProm)
}
}
mainUnitSuffix, perUnitSuffix := buildUnitSuffixes(metric.Unit())
nameTokens = addUnitTokens(nameTokens, cleanUpUnit(mainUnitSuffix), cleanUpUnit(perUnitSuffix))
// Append _total for Counters
if metric.Type() == pmetric.MetricTypeSum && metric.Sum().IsMonotonic() {
@ -208,14 +150,8 @@ func normalizeName(metric pmetric.Metric, namespace string, allowUTF8 bool) stri
nameTokens = append([]string{namespace}, nameTokens...)
}
var normalizedName string
if !allowUTF8 {
// Build the string from the tokens, separated with underscores
normalizedName = strings.Join(nameTokens, "_")
} else {
// Build the string from the tokens + separators.
normalizedName = join(nameTokens, separators, "_")
}
// Build the string from the tokens, separated with underscores
normalizedName := strings.Join(nameTokens, "_")
// Metric name cannot start with a digit, so prefix it with "_" in this case
if normalizedName != "" && unicode.IsDigit(rune(normalizedName[0])) {
@ -225,6 +161,39 @@ func normalizeName(metric pmetric.Metric, namespace string, allowUTF8 bool) stri
return normalizedName
}
// addUnitTokens will add the suffixes to the nameTokens if they are not already present.
// It will also remove trailing underscores from the main suffix to avoid double underscores
// when joining the tokens.
//
// If the 'per' unit ends with underscore, the underscore will be removed. If the per unit is just
// 'per_', it will be entirely removed.
func addUnitTokens(nameTokens []string, mainUnitSuffix, perUnitSuffix string) []string {
if slices.Contains(nameTokens, mainUnitSuffix) {
mainUnitSuffix = ""
}
if perUnitSuffix == "per_" {
perUnitSuffix = ""
} else {
perUnitSuffix = strings.TrimSuffix(perUnitSuffix, "_")
if slices.Contains(nameTokens, perUnitSuffix) {
perUnitSuffix = ""
}
}
if perUnitSuffix != "" {
mainUnitSuffix = strings.TrimSuffix(mainUnitSuffix, "_")
}
if mainUnitSuffix != "" {
nameTokens = append(nameTokens, mainUnitSuffix)
}
if perUnitSuffix != "" {
nameTokens = append(nameTokens, perUnitSuffix)
}
return nameTokens
}
// cleanUpUnit cleans up unit so it matches model.LabelNameRE.
func cleanUpUnit(unit string) string {
// Multiple consecutive underscores are replaced with a single underscore.
@ -263,3 +232,75 @@ func removeItem(slice []string, value string) []string {
}
return newSlice
}
// BuildMetricName builds a valid metric name but without following Prometheus naming conventions.
// It doesn't do any character transformation, it only prefixes the metric name with the namespace, if any,
// and adds metric type suffixes, e.g. "_total" for counters and unit suffixes.
//
// Differently from BuildCompliantMetricName, it doesn't check for the presence of unit and type suffixes.
// If "addMetricSuffixes" is true, it will add them anyway.
//
// Please use BuildCompliantMetricName for a metric name that follows Prometheus naming conventions.
func BuildMetricName(metric pmetric.Metric, namespace string, addMetricSuffixes bool) string {
metricName := metric.Name()
if namespace != "" {
metricName = namespace + "_" + metricName
}
if addMetricSuffixes {
mainUnitSuffix, perUnitSuffix := buildUnitSuffixes(metric.Unit())
if mainUnitSuffix != "" {
metricName = metricName + "_" + mainUnitSuffix
}
if perUnitSuffix != "" {
metricName = metricName + "_" + perUnitSuffix
}
// Append _total for Counters
if metric.Type() == pmetric.MetricTypeSum && metric.Sum().IsMonotonic() {
metricName = metricName + "_total"
}
// Append _ratio for metrics with unit "1"
// Some OTel receivers improperly use unit "1" for counters of objects
// See https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aissue+some+metric+units+don%27t+follow+otel+semantic+conventions
// Until these issues have been fixed, we're appending `_ratio` for gauges ONLY
// Theoretically, counters could be ratios as well, but it's absurd (for mathematical reasons)
if metric.Unit() == "1" && metric.Type() == pmetric.MetricTypeGauge {
metricName = metricName + "_ratio"
}
}
return metricName
}
// buildUnitSuffixes builds the main and per unit suffixes for the specified unit
// but doesn't do any special character transformation to accommodate Prometheus naming conventions.
// Removing trailing underscores or appending suffixes is done in the caller.
func buildUnitSuffixes(unit string) (mainUnitSuffix, perUnitSuffix string) {
// Split unit at the '/' if any
unitTokens := strings.SplitN(unit, "/", 2)
if len(unitTokens) > 0 {
// Main unit
// Update if not blank and doesn't contain '{}'
mainUnitOTel := strings.TrimSpace(unitTokens[0])
if mainUnitOTel != "" && !strings.ContainsAny(mainUnitOTel, "{}") {
mainUnitSuffix = unitMapGetOrDefault(mainUnitOTel)
}
// Per unit
// Update if not blank and doesn't contain '{}'
if len(unitTokens) > 1 && unitTokens[1] != "" {
perUnitOTel := strings.TrimSpace(unitTokens[1])
if perUnitOTel != "" && !strings.ContainsAny(perUnitOTel, "{}") {
perUnitSuffix = perUnitMapGetOrDefault(perUnitOTel)
}
if perUnitSuffix != "" {
perUnitSuffix = "per_" + perUnitSuffix
}
}
}
return mainUnitSuffix, perUnitSuffix
}

View file

@ -0,0 +1,257 @@
// Copyright 2024 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Provenance-includes-location: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/95e8f8fdc2a9dc87230406c9a3cf02be4fd68bea/pkg/translator/prometheus/normalize_name_test.go
// Provenance-includes-license: Apache-2.0
// Provenance-includes-copyright: Copyright The OpenTelemetry Authors.
package prometheus
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestByte(t *testing.T) {
require.Equal(t, "system_filesystem_usage_bytes", normalizeName(createGauge("system.filesystem.usage", "By"), ""))
}
func TestByteCounter(t *testing.T) {
require.Equal(t, "system_io_bytes_total", normalizeName(createCounter("system.io", "By"), ""))
require.Equal(t, "network_transmitted_bytes_total", normalizeName(createCounter("network_transmitted_bytes_total", "By"), ""))
}
func TestWhiteSpaces(t *testing.T) {
require.Equal(t, "system_filesystem_usage_bytes", normalizeName(createGauge("\t system.filesystem.usage ", " By\t"), ""))
}
func TestNonStandardUnit(t *testing.T) {
require.Equal(t, "system_network_dropped", normalizeName(createGauge("system.network.dropped", "{packets}"), ""))
// The normal metric name character set is allowed in non-standard units.
require.Equal(t, "system_network_dropped_nonstandard:_1", normalizeName(createGauge("system.network.dropped", "nonstandard:_1"), ""))
}
func TestNonStandardUnitCounter(t *testing.T) {
require.Equal(t, "system_network_dropped_total", normalizeName(createCounter("system.network.dropped", "{packets}"), ""))
}
func TestBrokenUnit(t *testing.T) {
require.Equal(t, "system_network_dropped_packets", normalizeName(createGauge("system.network.dropped", "packets"), ""))
require.Equal(t, "system_network_packets_dropped", normalizeName(createGauge("system.network.packets.dropped", "packets"), ""))
require.Equal(t, "system_network_packets", normalizeName(createGauge("system.network.packets", "packets"), ""))
}
func TestBrokenUnitCounter(t *testing.T) {
require.Equal(t, "system_network_dropped_packets_total", normalizeName(createCounter("system.network.dropped", "packets"), ""))
require.Equal(t, "system_network_packets_dropped_total", normalizeName(createCounter("system.network.packets.dropped", "packets"), ""))
require.Equal(t, "system_network_packets_total", normalizeName(createCounter("system.network.packets", "packets"), ""))
}
func TestRatio(t *testing.T) {
require.Equal(t, "hw_gpu_memory_utilization_ratio", normalizeName(createGauge("hw.gpu.memory.utilization", "1"), ""))
require.Equal(t, "hw_fan_speed_ratio", normalizeName(createGauge("hw.fan.speed_ratio", "1"), ""))
require.Equal(t, "objects_total", normalizeName(createCounter("objects", "1"), ""))
}
func TestHertz(t *testing.T) {
require.Equal(t, "hw_cpu_speed_limit_hertz", normalizeName(createGauge("hw.cpu.speed_limit", "Hz"), ""))
}
func TestPer(t *testing.T) {
require.Equal(t, "broken_metric_speed_km_per_hour", normalizeName(createGauge("broken.metric.speed", "km/h"), ""))
require.Equal(t, "astro_light_speed_limit_meters_per_second", normalizeName(createGauge("astro.light.speed_limit", "m/s"), ""))
// The normal metric name character set is allowed in non-standard units.
require.Equal(t, "system_network_dropped_non_per_standard:_1", normalizeName(createGauge("system.network.dropped", "non/standard:_1"), ""))
t.Run("invalid per unit", func(t *testing.T) {
require.Equal(t, "broken_metric_speed_km", normalizeName(createGauge("broken.metric.speed", "km/°"), ""))
})
}
func TestPercent(t *testing.T) {
require.Equal(t, "broken_metric_success_ratio_percent", normalizeName(createGauge("broken.metric.success_ratio", "%"), ""))
require.Equal(t, "broken_metric_success_percent", normalizeName(createGauge("broken.metric.success_percent", "%"), ""))
}
func TestEmpty(t *testing.T) {
require.Equal(t, "test_metric_no_unit", normalizeName(createGauge("test.metric.no_unit", ""), ""))
require.Equal(t, "test_metric_spaces", normalizeName(createGauge("test.metric.spaces", " \t "), ""))
}
func TestOTelReceivers(t *testing.T) {
require.Equal(t, "active_directory_ds_replication_network_io_bytes_total", normalizeName(createCounter("active_directory.ds.replication.network.io", "By"), ""))
require.Equal(t, "active_directory_ds_replication_sync_object_pending_total", normalizeName(createCounter("active_directory.ds.replication.sync.object.pending", "{objects}"), ""))
require.Equal(t, "active_directory_ds_replication_object_rate_per_second", normalizeName(createGauge("active_directory.ds.replication.object.rate", "{objects}/s"), ""))
require.Equal(t, "active_directory_ds_name_cache_hit_rate_percent", normalizeName(createGauge("active_directory.ds.name_cache.hit_rate", "%"), ""))
require.Equal(t, "active_directory_ds_ldap_bind_last_successful_time_milliseconds", normalizeName(createGauge("active_directory.ds.ldap.bind.last_successful.time", "ms"), ""))
require.Equal(t, "apache_current_connections", normalizeName(createGauge("apache.current_connections", "connections"), ""))
require.Equal(t, "apache_workers_connections", normalizeName(createGauge("apache.workers", "connections"), ""))
require.Equal(t, "apache_requests_total", normalizeName(createCounter("apache.requests", "1"), ""))
require.Equal(t, "bigip_virtual_server_request_count_total", normalizeName(createCounter("bigip.virtual_server.request.count", "{requests}"), ""))
require.Equal(t, "system_cpu_utilization_ratio", normalizeName(createGauge("system.cpu.utilization", "1"), ""))
require.Equal(t, "system_disk_operation_time_seconds_total", normalizeName(createCounter("system.disk.operation_time", "s"), ""))
require.Equal(t, "system_cpu_load_average_15m_ratio", normalizeName(createGauge("system.cpu.load_average.15m", "1"), ""))
require.Equal(t, "memcached_operation_hit_ratio_percent", normalizeName(createGauge("memcached.operation_hit_ratio", "%"), ""))
require.Equal(t, "mongodbatlas_process_asserts_per_second", normalizeName(createGauge("mongodbatlas.process.asserts", "{assertions}/s"), ""))
require.Equal(t, "mongodbatlas_process_journaling_data_files_mebibytes", normalizeName(createGauge("mongodbatlas.process.journaling.data_files", "MiBy"), ""))
require.Equal(t, "mongodbatlas_process_network_io_bytes_per_second", normalizeName(createGauge("mongodbatlas.process.network.io", "By/s"), ""))
require.Equal(t, "mongodbatlas_process_oplog_rate_gibibytes_per_hour", normalizeName(createGauge("mongodbatlas.process.oplog.rate", "GiBy/h"), ""))
require.Equal(t, "mongodbatlas_process_db_query_targeting_scanned_per_returned", normalizeName(createGauge("mongodbatlas.process.db.query_targeting.scanned_per_returned", "{scanned}/{returned}"), ""))
require.Equal(t, "nginx_requests", normalizeName(createGauge("nginx.requests", "requests"), ""))
require.Equal(t, "nginx_connections_accepted", normalizeName(createGauge("nginx.connections_accepted", "connections"), ""))
require.Equal(t, "nsxt_node_memory_usage_kilobytes", normalizeName(createGauge("nsxt.node.memory.usage", "KBy"), ""))
require.Equal(t, "redis_latest_fork_microseconds", normalizeName(createGauge("redis.latest_fork", "us"), ""))
}
func TestNamespace(t *testing.T) {
require.Equal(t, "space_test", normalizeName(createGauge("test", ""), "space"))
require.Equal(t, "space_test", normalizeName(createGauge("#test", ""), "space"))
}
func TestCleanUpUnit(t *testing.T) {
require.Equal(t, "", cleanUpUnit(""))
require.Equal(t, "a_b", cleanUpUnit("a b"))
require.Equal(t, "hello_world", cleanUpUnit("hello, world"))
require.Equal(t, "hello_you_2", cleanUpUnit("hello you 2"))
require.Equal(t, "1000", cleanUpUnit("$1000"))
require.Equal(t, "", cleanUpUnit("*+$^=)"))
}
func TestUnitMapGetOrDefault(t *testing.T) {
require.Equal(t, "", unitMapGetOrDefault(""))
require.Equal(t, "seconds", unitMapGetOrDefault("s"))
require.Equal(t, "invalid", unitMapGetOrDefault("invalid"))
}
func TestPerUnitMapGetOrDefault(t *testing.T) {
require.Equal(t, "", perUnitMapGetOrDefault(""))
require.Equal(t, "second", perUnitMapGetOrDefault("s"))
require.Equal(t, "invalid", perUnitMapGetOrDefault("invalid"))
}
func TestBuildUnitSuffixes(t *testing.T) {
tests := []struct {
unit string
expectedMain string
expectedPer string
}{
{"", "", ""},
{"s", "seconds", ""},
{"By/s", "bytes", "per_second"},
{"requests/m", "requests", "per_minute"},
{"{invalid}/second", "", "per_second"},
{"bytes/{invalid}", "bytes", ""},
}
for _, test := range tests {
mainUnitSuffix, perUnitSuffix := buildUnitSuffixes(test.unit)
require.Equal(t, test.expectedMain, mainUnitSuffix)
require.Equal(t, test.expectedPer, perUnitSuffix)
}
}
func TestAddUnitTokens(t *testing.T) {
tests := []struct {
nameTokens []string
mainUnitSuffix string
perUnitSuffix string
expected []string
}{
{[]string{}, "", "", []string{}},
{[]string{"token1"}, "main", "", []string{"token1", "main"}},
{[]string{"token1"}, "", "per", []string{"token1", "per"}},
{[]string{"token1"}, "main", "per", []string{"token1", "main", "per"}},
{[]string{"token1", "per"}, "main", "per", []string{"token1", "per", "main"}},
{[]string{"token1", "main"}, "main", "per", []string{"token1", "main", "per"}},
{[]string{"token1"}, "main_", "per", []string{"token1", "main", "per"}},
{[]string{"token1"}, "main_unit", "per_seconds_", []string{"token1", "main_unit", "per_seconds"}}, // trailing underscores are removed
{[]string{"token1"}, "main_unit", "per_", []string{"token1", "main_unit"}}, // 'per_' is removed entirely
}
for _, test := range tests {
result := addUnitTokens(test.nameTokens, test.mainUnitSuffix, test.perUnitSuffix)
require.Equal(t, test.expected, result)
}
}
func TestRemoveItem(t *testing.T) {
require.Equal(t, []string{}, removeItem([]string{}, "test"))
require.Equal(t, []string{}, removeItem([]string{}, ""))
require.Equal(t, []string{"a", "b", "c"}, removeItem([]string{"a", "b", "c"}, "d"))
require.Equal(t, []string{"a", "b", "c"}, removeItem([]string{"a", "b", "c"}, ""))
require.Equal(t, []string{"a", "b"}, removeItem([]string{"a", "b", "c"}, "c"))
require.Equal(t, []string{"a", "c"}, removeItem([]string{"a", "b", "c"}, "b"))
require.Equal(t, []string{"b", "c"}, removeItem([]string{"a", "b", "c"}, "a"))
}
func TestBuildCompliantMetricNameWithSuffixes(t *testing.T) {
require.Equal(t, "system_io_bytes_total", BuildCompliantMetricName(createCounter("system.io", "By"), "", true))
require.Equal(t, "system_network_io_bytes_total", BuildCompliantMetricName(createCounter("network.io", "By"), "system", true))
require.Equal(t, "_3_14_digits", BuildCompliantMetricName(createGauge("3.14 digits", ""), "", true))
require.Equal(t, "envoy_rule_engine_zlib_buf_error", BuildCompliantMetricName(createGauge("envoy__rule_engine_zlib_buf_error", ""), "", true))
require.Equal(t, ":foo::bar", BuildCompliantMetricName(createGauge(":foo::bar", ""), "", true))
require.Equal(t, ":foo::bar_total", BuildCompliantMetricName(createCounter(":foo::bar", ""), "", true))
// Gauges with unit 1 are considered ratios.
require.Equal(t, "foo_bar_ratio", BuildCompliantMetricName(createGauge("foo.bar", "1"), "", true))
// Slashes in units are converted.
require.Equal(t, "system_io_foo_per_bar_total", BuildCompliantMetricName(createCounter("system.io", "foo/bar"), "", true))
require.Equal(t, "metric_with_foreign_characters_total", BuildCompliantMetricName(createCounter("metric_with_字符_foreign_characters", ""), "", true))
// Removes non aplhanumerical characters from units, but leaves colons.
require.Equal(t, "temperature_:C", BuildCompliantMetricName(createGauge("temperature", "%*()°:C"), "", true))
}
func TestBuildCompliantMetricNameWithoutSuffixes(t *testing.T) {
require.Equal(t, "system_io", BuildCompliantMetricName(createCounter("system.io", "By"), "", false))
require.Equal(t, "system_network_io", BuildCompliantMetricName(createCounter("network.io", "By"), "system", false))
require.Equal(t, "system_network_I_O", BuildCompliantMetricName(createCounter("network (I/O)", "By"), "system", false))
require.Equal(t, "_3_14_digits", BuildCompliantMetricName(createGauge("3.14 digits", "By"), "", false))
require.Equal(t, "envoy__rule_engine_zlib_buf_error", BuildCompliantMetricName(createGauge("envoy__rule_engine_zlib_buf_error", ""), "", false))
require.Equal(t, ":foo::bar", BuildCompliantMetricName(createGauge(":foo::bar", ""), "", false))
require.Equal(t, ":foo::bar", BuildCompliantMetricName(createCounter(":foo::bar", ""), "", false))
require.Equal(t, "foo_bar", BuildCompliantMetricName(createGauge("foo.bar", "1"), "", false))
require.Equal(t, "system_io", BuildCompliantMetricName(createCounter("system.io", "foo/bar"), "", false))
require.Equal(t, "metric_with___foreign_characters", BuildCompliantMetricName(createCounter("metric_with_字符_foreign_characters", ""), "", false))
}
func TestBuildMetricNameWithSuffixes(t *testing.T) {
require.Equal(t, "system.io_bytes_total", BuildMetricName(createCounter("system.io", "By"), "", true))
require.Equal(t, "system_network.io_bytes_total", BuildMetricName(createCounter("network.io", "By"), "system", true))
require.Equal(t, "3.14 digits", BuildMetricName(createGauge("3.14 digits", ""), "", true))
require.Equal(t, "envoy__rule_engine_zlib_buf_error", BuildMetricName(createGauge("envoy__rule_engine_zlib_buf_error", ""), "", true))
require.Equal(t, ":foo::bar", BuildMetricName(createGauge(":foo::bar", ""), "", true))
require.Equal(t, ":foo::bar_total", BuildMetricName(createCounter(":foo::bar", ""), "", true))
// Gauges with unit 1 are considered ratios.
require.Equal(t, "foo.bar_ratio", BuildMetricName(createGauge("foo.bar", "1"), "", true))
// Slashes in units are converted.
require.Equal(t, "system.io_foo_per_bar_total", BuildMetricName(createCounter("system.io", "foo/bar"), "", true))
require.Equal(t, "metric_with_字符_foreign_characters_total", BuildMetricName(createCounter("metric_with_字符_foreign_characters", ""), "", true))
require.Equal(t, "temperature_%*()°C", BuildMetricName(createGauge("temperature", "%*()°C"), "", true)) // Keeps the all characters in unit
// Tests below show weird interactions that users can have with the metric names.
// With BuildMetricName we don't check if units/type suffixes are already present in the metric name, we always add them.
require.Equal(t, "system_io_seconds_seconds", BuildMetricName(createGauge("system_io_seconds", "s"), "", true))
require.Equal(t, "system_io_total_total", BuildMetricName(createCounter("system_io_total", ""), "", true))
}
func TestBuildMetricNameWithoutSuffixes(t *testing.T) {
require.Equal(t, "system.io", BuildMetricName(createCounter("system.io", "By"), "", false))
require.Equal(t, "system_network.io", BuildMetricName(createCounter("network.io", "By"), "system", false))
require.Equal(t, "3.14 digits", BuildMetricName(createGauge("3.14 digits", ""), "", false))
require.Equal(t, "envoy__rule_engine_zlib_buf_error", BuildMetricName(createGauge("envoy__rule_engine_zlib_buf_error", ""), "", false))
require.Equal(t, ":foo::bar", BuildMetricName(createGauge(":foo::bar", ""), "", false))
require.Equal(t, ":foo::bar", BuildMetricName(createCounter(":foo::bar", ""), "", false))
// Gauges with unit 1 are considered ratios.
require.Equal(t, "foo.bar", BuildMetricName(createGauge("foo.bar", "1"), "", false))
require.Equal(t, "metric_with_字符_foreign_characters", BuildMetricName(createCounter("metric_with_字符_foreign_characters", ""), "", false))
require.Equal(t, "system_io_seconds", BuildMetricName(createGauge("system_io_seconds", "s"), "", false))
require.Equal(t, "system_io_total", BuildMetricName(createCounter("system_io_total", ""), "", false))
}

View file

@ -1,210 +0,0 @@
// Copyright 2024 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Provenance-includes-location: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/95e8f8fdc2a9dc87230406c9a3cf02be4fd68bea/pkg/translator/prometheus/normalize_name_test.go
// Provenance-includes-license: Apache-2.0
// Provenance-includes-copyright: Copyright The OpenTelemetry Authors.
package prometheus
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestByte(t *testing.T) {
require.Equal(t, "system_filesystem_usage_bytes", normalizeName(createGauge("system.filesystem.usage", "By"), "", false))
}
func TestByteCounter(t *testing.T) {
require.Equal(t, "system_io_bytes_total", normalizeName(createCounter("system.io", "By"), "", false))
require.Equal(t, "network_transmitted_bytes_total", normalizeName(createCounter("network_transmitted_bytes_total", "By"), "", false))
}
func TestWhiteSpaces(t *testing.T) {
require.Equal(t, "system_filesystem_usage_bytes", normalizeName(createGauge("\t system.filesystem.usage ", " By\t"), "", false))
}
func TestNonStandardUnit(t *testing.T) {
require.Equal(t, "system_network_dropped", normalizeName(createGauge("system.network.dropped", "{packets}"), "", false))
// The normal metric name character set is allowed in non-standard units.
require.Equal(t, "system_network_dropped_nonstandard:_1", normalizeName(createGauge("system.network.dropped", "nonstandard:_1"), "", false))
}
func TestNonStandardUnitCounter(t *testing.T) {
require.Equal(t, "system_network_dropped_total", normalizeName(createCounter("system.network.dropped", "{packets}"), "", false))
}
func TestBrokenUnit(t *testing.T) {
require.Equal(t, "system_network_dropped_packets", normalizeName(createGauge("system.network.dropped", "packets"), "", false))
require.Equal(t, "system_network_packets_dropped", normalizeName(createGauge("system.network.packets.dropped", "packets"), "", false))
require.Equal(t, "system_network_packets", normalizeName(createGauge("system.network.packets", "packets"), "", false))
}
func TestBrokenUnitCounter(t *testing.T) {
require.Equal(t, "system_network_dropped_packets_total", normalizeName(createCounter("system.network.dropped", "packets"), "", false))
require.Equal(t, "system_network_packets_dropped_total", normalizeName(createCounter("system.network.packets.dropped", "packets"), "", false))
require.Equal(t, "system_network_packets_total", normalizeName(createCounter("system.network.packets", "packets"), "", false))
}
func TestRatio(t *testing.T) {
require.Equal(t, "hw_gpu_memory_utilization_ratio", normalizeName(createGauge("hw.gpu.memory.utilization", "1"), "", false))
require.Equal(t, "hw_fan_speed_ratio", normalizeName(createGauge("hw.fan.speed_ratio", "1"), "", false))
require.Equal(t, "objects_total", normalizeName(createCounter("objects", "1"), "", false))
}
func TestHertz(t *testing.T) {
require.Equal(t, "hw_cpu_speed_limit_hertz", normalizeName(createGauge("hw.cpu.speed_limit", "Hz"), "", false))
}
func TestPer(t *testing.T) {
require.Equal(t, "broken_metric_speed_km_per_hour", normalizeName(createGauge("broken.metric.speed", "km/h"), "", false))
require.Equal(t, "astro_light_speed_limit_meters_per_second", normalizeName(createGauge("astro.light.speed_limit", "m/s"), "", false))
// The normal metric name character set is allowed in non-standard units.
require.Equal(t, "system_network_dropped_non_per_standard:_1", normalizeName(createGauge("system.network.dropped", "non/standard:_1"), "", false))
t.Run("invalid per unit", func(t *testing.T) {
require.Equal(t, "broken_metric_speed_km", normalizeName(createGauge("broken.metric.speed", "km/°"), "", false))
})
}
func TestPercent(t *testing.T) {
require.Equal(t, "broken_metric_success_ratio_percent", normalizeName(createGauge("broken.metric.success_ratio", "%"), "", false))
require.Equal(t, "broken_metric_success_percent", normalizeName(createGauge("broken.metric.success_percent", "%"), "", false))
}
func TestEmpty(t *testing.T) {
require.Equal(t, "test_metric_no_unit", normalizeName(createGauge("test.metric.no_unit", ""), "", false))
require.Equal(t, "test_metric_spaces", normalizeName(createGauge("test.metric.spaces", " \t "), "", false))
}
func TestAllowUTF8(t *testing.T) {
t.Run("allow UTF8", func(t *testing.T) {
require.Equal(t, "unsupported.metric.temperature_°F", normalizeName(createGauge("unsupported.metric.temperature", "°F"), "", true))
require.Equal(t, "unsupported.metric.weird_+=.:,!* & #", normalizeName(createGauge("unsupported.metric.weird", "+=.:,!* & #"), "", true))
require.Equal(t, "unsupported.metric.redundant___test $_per_°C", normalizeName(createGauge("unsupported.metric.redundant", "__test $/°C"), "", true))
require.Equal(t, "metric_with_字符_foreign_characters_ど", normalizeName(createGauge("metric_with_字符_foreign_characters", "ど"), "", true))
})
t.Run("disallow UTF8", func(t *testing.T) {
require.Equal(t, "unsupported_metric_temperature_F", normalizeName(createGauge("unsupported.metric.temperature", "°F"), "", false))
require.Equal(t, "unsupported_metric_weird", normalizeName(createGauge("unsupported.metric.weird", "+=.,!* & #"), "", false))
require.Equal(t, "unsupported_metric_redundant_test_per_C", normalizeName(createGauge("unsupported.metric.redundant", "__test $/°C"), "", false))
require.Equal(t, "metric_with_foreign_characters", normalizeName(createGauge("metric_with_字符_foreign_characters", "ど"), "", false))
})
}
func TestAllowUTF8KnownBugs(t *testing.T) {
// Due to historical reasons, the translator code was copied from OpenTelemetry collector codebase.
// Over there, they tried to provide means to translate metric names following Prometheus conventions that are documented here:
// https://prometheus.io/docs/practices/naming/
//
// Althogh not explicitly said, it was implied that words should be separated by a single underscore and the codebase was written
// with that in mind.
//
// Now that we're allowing OTel users to have their original names stored in prometheus without any transformation, we're facing problems
// where two (or more) UTF-8 characters are being used to separate words.
// TODO(arthursens): Fix it!
// We're asserting on 'NotEqual', which proves the bug.
require.NotEqual(t, "metric....split_=+by_//utf8characters", normalizeName(createGauge("metric....split_=+by_//utf8characters", ""), "", true))
// Here we're asserting on 'Equal', showing the current behavior.
require.Equal(t, "metric.split_by_utf8characters", normalizeName(createGauge("metric....split_=+by_//utf8characters", ""), "", true))
}
func TestOTelReceivers(t *testing.T) {
require.Equal(t, "active_directory_ds_replication_network_io_bytes_total", normalizeName(createCounter("active_directory.ds.replication.network.io", "By"), "", false))
require.Equal(t, "active_directory_ds_replication_sync_object_pending_total", normalizeName(createCounter("active_directory.ds.replication.sync.object.pending", "{objects}"), "", false))
require.Equal(t, "active_directory_ds_replication_object_rate_per_second", normalizeName(createGauge("active_directory.ds.replication.object.rate", "{objects}/s"), "", false))
require.Equal(t, "active_directory_ds_name_cache_hit_rate_percent", normalizeName(createGauge("active_directory.ds.name_cache.hit_rate", "%"), "", false))
require.Equal(t, "active_directory_ds_ldap_bind_last_successful_time_milliseconds", normalizeName(createGauge("active_directory.ds.ldap.bind.last_successful.time", "ms"), "", false))
require.Equal(t, "apache_current_connections", normalizeName(createGauge("apache.current_connections", "connections"), "", false))
require.Equal(t, "apache_workers_connections", normalizeName(createGauge("apache.workers", "connections"), "", false))
require.Equal(t, "apache_requests_total", normalizeName(createCounter("apache.requests", "1"), "", false))
require.Equal(t, "bigip_virtual_server_request_count_total", normalizeName(createCounter("bigip.virtual_server.request.count", "{requests}"), "", false))
require.Equal(t, "system_cpu_utilization_ratio", normalizeName(createGauge("system.cpu.utilization", "1"), "", false))
require.Equal(t, "system_disk_operation_time_seconds_total", normalizeName(createCounter("system.disk.operation_time", "s"), "", false))
require.Equal(t, "system_cpu_load_average_15m_ratio", normalizeName(createGauge("system.cpu.load_average.15m", "1"), "", false))
require.Equal(t, "memcached_operation_hit_ratio_percent", normalizeName(createGauge("memcached.operation_hit_ratio", "%"), "", false))
require.Equal(t, "mongodbatlas_process_asserts_per_second", normalizeName(createGauge("mongodbatlas.process.asserts", "{assertions}/s"), "", false))
require.Equal(t, "mongodbatlas_process_journaling_data_files_mebibytes", normalizeName(createGauge("mongodbatlas.process.journaling.data_files", "MiBy"), "", false))
require.Equal(t, "mongodbatlas_process_network_io_bytes_per_second", normalizeName(createGauge("mongodbatlas.process.network.io", "By/s"), "", false))
require.Equal(t, "mongodbatlas_process_oplog_rate_gibibytes_per_hour", normalizeName(createGauge("mongodbatlas.process.oplog.rate", "GiBy/h"), "", false))
require.Equal(t, "mongodbatlas_process_db_query_targeting_scanned_per_returned", normalizeName(createGauge("mongodbatlas.process.db.query_targeting.scanned_per_returned", "{scanned}/{returned}"), "", false))
require.Equal(t, "nginx_requests", normalizeName(createGauge("nginx.requests", "requests"), "", false))
require.Equal(t, "nginx_connections_accepted", normalizeName(createGauge("nginx.connections_accepted", "connections"), "", false))
require.Equal(t, "nsxt_node_memory_usage_kilobytes", normalizeName(createGauge("nsxt.node.memory.usage", "KBy"), "", false))
require.Equal(t, "redis_latest_fork_microseconds", normalizeName(createGauge("redis.latest_fork", "us"), "", false))
}
func TestNamespace(t *testing.T) {
require.Equal(t, "space_test", normalizeName(createGauge("test", ""), "space", false))
require.Equal(t, "space_test", normalizeName(createGauge("#test", ""), "space", false))
}
func TestCleanUpUnit(t *testing.T) {
require.Equal(t, "", cleanUpUnit(""))
require.Equal(t, "a_b", cleanUpUnit("a b"))
require.Equal(t, "hello_world", cleanUpUnit("hello, world"))
require.Equal(t, "hello_you_2", cleanUpUnit("hello you 2"))
require.Equal(t, "1000", cleanUpUnit("$1000"))
require.Equal(t, "", cleanUpUnit("*+$^=)"))
}
func TestUnitMapGetOrDefault(t *testing.T) {
require.Equal(t, "", unitMapGetOrDefault(""))
require.Equal(t, "seconds", unitMapGetOrDefault("s"))
require.Equal(t, "invalid", unitMapGetOrDefault("invalid"))
}
func TestPerUnitMapGetOrDefault(t *testing.T) {
require.Equal(t, "", perUnitMapGetOrDefault(""))
require.Equal(t, "second", perUnitMapGetOrDefault("s"))
require.Equal(t, "invalid", perUnitMapGetOrDefault("invalid"))
}
func TestRemoveItem(t *testing.T) {
require.Equal(t, []string{}, removeItem([]string{}, "test"))
require.Equal(t, []string{}, removeItem([]string{}, ""))
require.Equal(t, []string{"a", "b", "c"}, removeItem([]string{"a", "b", "c"}, "d"))
require.Equal(t, []string{"a", "b", "c"}, removeItem([]string{"a", "b", "c"}, ""))
require.Equal(t, []string{"a", "b"}, removeItem([]string{"a", "b", "c"}, "c"))
require.Equal(t, []string{"a", "c"}, removeItem([]string{"a", "b", "c"}, "b"))
require.Equal(t, []string{"b", "c"}, removeItem([]string{"a", "b", "c"}, "a"))
}
func TestBuildCompliantNameWithSuffixes(t *testing.T) {
require.Equal(t, "system_io_bytes_total", BuildCompliantName(createCounter("system.io", "By"), "", true, false))
require.Equal(t, "system_network_io_bytes_total", BuildCompliantName(createCounter("network.io", "By"), "system", true, false))
require.Equal(t, "_3_14_digits", BuildCompliantName(createGauge("3.14 digits", ""), "", true, false))
require.Equal(t, "envoy_rule_engine_zlib_buf_error", BuildCompliantName(createGauge("envoy__rule_engine_zlib_buf_error", ""), "", true, false))
require.Equal(t, ":foo::bar", BuildCompliantName(createGauge(":foo::bar", ""), "", true, false))
require.Equal(t, ":foo::bar_total", BuildCompliantName(createCounter(":foo::bar", ""), "", true, false))
// Gauges with unit 1 are considered ratios.
require.Equal(t, "foo_bar_ratio", BuildCompliantName(createGauge("foo.bar", "1"), "", true, false))
// Slashes in units are converted.
require.Equal(t, "system_io_foo_per_bar_total", BuildCompliantName(createCounter("system.io", "foo/bar"), "", true, false))
require.Equal(t, "metric_with_foreign_characters_total", BuildCompliantName(createCounter("metric_with_字符_foreign_characters", ""), "", true, false))
}
func TestBuildCompliantNameWithoutSuffixes(t *testing.T) {
require.Equal(t, "system_io", BuildCompliantName(createCounter("system.io", "By"), "", false, false))
require.Equal(t, "system_network_io", BuildCompliantName(createCounter("network.io", "By"), "system", false, false))
require.Equal(t, "system_network_I_O", BuildCompliantName(createCounter("network (I/O)", "By"), "system", false, false))
require.Equal(t, "_3_14_digits", BuildCompliantName(createGauge("3.14 digits", "By"), "", false, false))
require.Equal(t, "envoy__rule_engine_zlib_buf_error", BuildCompliantName(createGauge("envoy__rule_engine_zlib_buf_error", ""), "", false, false))
require.Equal(t, ":foo::bar", BuildCompliantName(createGauge(":foo::bar", ""), "", false, false))
require.Equal(t, ":foo::bar", BuildCompliantName(createCounter(":foo::bar", ""), "", false, false))
require.Equal(t, "foo_bar", BuildCompliantName(createGauge("foo.bar", "1"), "", false, false))
require.Equal(t, "system_io", BuildCompliantName(createCounter("system.io", "foo/bar"), "", false, false))
require.Equal(t, "metric_with___foreign_characters", BuildCompliantName(createCounter("metric_with_字符_foreign_characters", ""), "", false, false))
}

View file

@ -762,7 +762,7 @@ func TestPrometheusConverter_addExponentialHistogramDataPoints(t *testing.T) {
Settings{
ExportCreatedMetric: true,
},
prometheustranslator.BuildCompliantName(metric, "", true, true),
prometheustranslator.BuildCompliantMetricName(metric, "", true),
)
require.NoError(t, err)
require.Empty(t, annots)

View file

@ -96,7 +96,12 @@ func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metric
continue
}
promName := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes, settings.AllowUTF8)
var promName string
if settings.AllowUTF8 {
promName = prometheustranslator.BuildMetricName(metric, settings.Namespace, settings.AddMetricSuffixes)
} else {
promName = prometheustranslator.BuildCompliantMetricName(metric, settings.Namespace, settings.AddMetricSuffixes)
}
c.metadata = append(c.metadata, prompb.MetricMetadata{
Type: otelMetricTypeToPromMetricType(metric),
MetricFamilyName: promName,

View file

@ -46,7 +46,7 @@ func TestFromMetrics(t *testing.T) {
metricSlice := scopeMetricsSlice.At(j).Metrics()
for k := 0; k < metricSlice.Len(); k++ {
metric := metricSlice.At(k)
promName := prometheustranslator.BuildCompliantName(metric, "", false, false)
promName := prometheustranslator.BuildCompliantMetricName(metric, "", false)
expMetadata = append(expMetadata, prompb.MetricMetadata{
Type: otelMetricTypeToPromMetricType(metric),
MetricFamilyName: promName,