mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2024-12-25 05:34:05 -08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Merge pull request #13627 from aknuds1/arve/sync-otlp

Browse source 
otlptranslator: Upgrade to v0.95.0
This commit is contained in:
Goutham Veeramachaneni 2024-02-22 22:34:18 +05:30 committed by GitHub
parent 9738e48a7d bf5ca8cf38
commit bf2658bec1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 145 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

80
storage/remote/otlptranslator/prometheusremotewrite/helper.go
View file

@ -28,7 +28,6 @@ import (
)
const (
nameStr = "__name__"
sumStr = "_sum"
countStr = "_count"
bucketStr = "_bucket"
@ -72,15 +71,14 @@ func (a ByLabelName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
// tsMap will be unmodified if either labels or sample is nil, but can still be modified if the exemplar is nil.
func addSample(tsMap map[string]*prompb.TimeSeries, sample *prompb.Sample, labels []prompb.Label,
datatype string) string {
if sample == nil || labels == nil || tsMap == nil {
// This shouldn't happen
return ""
}
sig := timeSeriesSignature(datatype, &labels)
ts, ok := tsMap[sig]
if ok {
sig := timeSeriesSignature(datatype, labels)
ts := tsMap[sig]
if ts != nil {
ts.Samples = append(ts.Samples, *sample)
} else {
newTs := &prompb.TimeSeries{
@ -97,7 +95,7 @@ func addSample(tsMap map[string]*prompb.TimeSeries, sample *prompb.Sample, label
// we only add exemplars if samples are presents
// tsMap is unmodified if either of its parameters is nil and samples are nil.
func addExemplars(tsMap map[string]*prompb.TimeSeries, exemplars []prompb.Exemplar, bucketBoundsData []bucketBoundsData) {
if tsMap == nil || bucketBoundsData == nil || exemplars == nil {
if len(tsMap) == 0 || len(bucketBoundsData) == 0 || len(exemplars) == 0 {
return
}
@ -113,14 +111,10 @@ func addExemplar(tsMap map[string]*prompb.TimeSeries, bucketBounds []bucketBound
sig := bucketBound.sig
bound := bucketBound.bound
_, ok := tsMap[sig]
if ok {
if tsMap[sig].Samples != nil {
if exemplar.Value <= bound {
tsMap[sig].Exemplars = append(tsMap[sig].Exemplars, exemplar)
return
}
}
ts := tsMap[sig]
if ts != nil && len(ts.Samples) > 0 && exemplar.Value <= bound {
ts.Exemplars = append(ts.Exemplars, exemplar)
return
}
}
}
@ -131,10 +125,10 @@ func addExemplar(tsMap map[string]*prompb.TimeSeries, bucketBounds []bucketBound
//
// the label slice should not contain duplicate label names; this method sorts the slice by label name before creating
// the signature.
func timeSeriesSignature(datatype string, labels *[]prompb.Label) string {
func timeSeriesSignature(datatype string, labels []prompb.Label) string {
length := len(datatype)
for _, lb := range *labels {
for _, lb := range labels {
length += 2 + len(lb.GetName()) + len(lb.GetValue())
}
@ -142,9 +136,9 @@ func timeSeriesSignature(datatype string, labels *[]prompb.Label) string {
b.Grow(length)
b.WriteString(datatype)
sort.Sort(ByLabelName(*labels))
sort.Sort(ByLabelName(labels))
for _, lb := range *labels {
for _, lb := range labels {
b.WriteString("-")
b.WriteString(lb.GetName())
b.WriteString("-")
@ -154,9 +148,9 @@ func timeSeriesSignature(datatype string, labels *[]prompb.Label) string {
return b.String()
}
// createAttributes creates a slice of Cortex Label with OTLP attributes and pairs of string values.
// Unpaired string value is ignored. String pairs overwrites OTLP labels if collision happens, and the overwrite is
// logged. Resultant label names are sanitized.
// createAttributes creates a slice of Prometheus Labels with OTLP attributes and pairs of string values.
// Unpaired string values are ignored. String pairs overwrite OTLP labels if collisions happen, and overwrites are
// logged. Resulting label names are sanitized.
func createAttributes(resource pcommon.Resource, attributes pcommon.Map, externalLabels map[string]string, extras ...string) []prompb.Label {
serviceName, haveServiceName := resource.Attributes().Get(conventions.AttributeServiceName)
instance, haveInstanceID := resource.Attributes().Get(conventions.AttributeServiceInstanceID)
@ -186,8 +180,8 @@ func createAttributes(resource pcommon.Resource, attributes pcommon.Map, externa
for _, label := range labels {
var finalKey = prometheustranslator.NormalizeLabel(label.Name)
if existingLabel, alreadyExists := l[finalKey]; alreadyExists {
l[finalKey] = existingLabel + ";" + label.Value
if existingValue, alreadyExists := l[finalKey]; alreadyExists {
l[finalKey] = existingValue + ";" + label.Value
} else {
l[finalKey] = label.Value
}
@ -257,10 +251,8 @@ func isValidAggregationTemporality(metric pmetric.Metric) bool {
// addSingleHistogramDataPoint converts pt to 2 + min(len(ExplicitBounds), len(BucketCount)) + 1 samples. It
// ignore extra buckets if len(ExplicitBounds) > len(BucketCounts)
func addSingleHistogramDataPoint(pt pmetric.HistogramDataPoint, resource pcommon.Resource, metric pmetric.Metric, settings Settings, tsMap map[string]*prompb.TimeSeries) {
func addSingleHistogramDataPoint(pt pmetric.HistogramDataPoint, resource pcommon.Resource, metric pmetric.Metric, settings Settings, tsMap map[string]*prompb.TimeSeries, baseName string) {
timestamp := convertTimeStamp(pt.Timestamp())
// sum, count, and buckets of the histogram should append suffix to baseName
baseName := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes)
baseLabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels)
createLabels := func(nameSuffix string, extras ...string) []prompb.Label {
@ -272,7 +264,8 @@ func addSingleHistogramDataPoint(pt pmetric.HistogramDataPoint, resource pcommon
labels = append(labels, prompb.Label{Name: extras[extrasIdx], Value: extras[extrasIdx+1]})
}
labels = append(labels, prompb.Label{Name: nameStr, Value: baseName + nameSuffix})
// sum, count, and buckets of the histogram should append suffix to baseName
labels = append(labels, prompb.Label{Name: model.MetricNameLabel, Value: baseName + nameSuffix})
return labels
}
@ -349,7 +342,7 @@ func addSingleHistogramDataPoint(pt pmetric.HistogramDataPoint, resource pcommon
startTimestamp := pt.StartTimestamp()
if settings.ExportCreatedMetric && startTimestamp != 0 {
labels := createLabels(createdSuffix)
addCreatedTimeSeriesIfNeeded(tsMap, labels, startTimestamp, metric.Type().String())
addCreatedTimeSeriesIfNeeded(tsMap, labels, startTimestamp, pt.Timestamp(), metric.Type().String())
}
}
@ -359,13 +352,12 @@ type exemplarType interface {
}
func getPromExemplars[T exemplarType](pt T) []prompb.Exemplar {
var promExemplars []prompb.Exemplar
promExemplars := make([]prompb.Exemplar, 0, pt.Exemplars().Len())
for i := 0; i < pt.Exemplars().Len(); i++ {
exemplar := pt.Exemplars().At(i)
exemplarRunes := 0
promExemplar := &prompb.Exemplar{
promExemplar := prompb.Exemplar{
Value: exemplar.DoubleValue(),
Timestamp: timestamp.FromTime(exemplar.Timestamp().AsTime()),
}
@ -387,9 +379,10 @@ func getPromExemplars[T exemplarType](pt T) []prompb.Exemplar {
}
promExemplar.Labels = append(promExemplar.Labels, promLabel)
}
var labelsFromAttributes []prompb.Label
exemplar.FilteredAttributes().Range(func(key string, value pcommon.Value) bool {
attrs := exemplar.FilteredAttributes()
labelsFromAttributes := make([]prompb.Label, 0, attrs.Len())
attrs.Range(func(key string, value pcommon.Value) bool {
val := value.AsString()
exemplarRunes += utf8.RuneCountInString(key) + utf8.RuneCountInString(val)
promLabel := prompb.Label{
@ -407,7 +400,7 @@ func getPromExemplars[T exemplarType](pt T) []prompb.Exemplar {
promExemplar.Labels = append(promExemplar.Labels, labelsFromAttributes...)
}
promExemplars = append(promExemplars, *promExemplar)
promExemplars = append(promExemplars, promExemplar)
}
return promExemplars
@ -457,10 +450,8 @@ func maxTimestamp(a, b pcommon.Timestamp) pcommon.Timestamp {
// addSingleSummaryDataPoint converts pt to len(QuantileValues) + 2 samples.
func addSingleSummaryDataPoint(pt pmetric.SummaryDataPoint, resource pcommon.Resource, metric pmetric.Metric, settings Settings,
tsMap map[string]*prompb.TimeSeries) {
tsMap map[string]*prompb.TimeSeries, baseName string) {
timestamp := convertTimeStamp(pt.Timestamp())
// sum and count of the summary should append suffix to baseName
baseName := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes)
baseLabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels)
createLabels := func(name string, extras ...string) []prompb.Label {
@ -472,7 +463,7 @@ func addSingleSummaryDataPoint(pt pmetric.SummaryDataPoint, resource pcommon.Res
labels = append(labels, prompb.Label{Name: extras[extrasIdx], Value: extras[extrasIdx+1]})
}
labels = append(labels, prompb.Label{Name: nameStr, Value: name})
labels = append(labels, prompb.Label{Name: model.MetricNameLabel, Value: name})
return labels
}
@ -485,6 +476,7 @@ func addSingleSummaryDataPoint(pt pmetric.SummaryDataPoint, resource pcommon.Res
if pt.Flags().NoRecordedValue() {
sum.Value = math.Float64frombits(value.StaleNaN)
}
// sum and count of the summary should append suffix to baseName
sumlabels := createLabels(baseName + sumStr)
addSample(tsMap, sum, sumlabels, metric.Type().String())
@ -518,7 +510,7 @@ func addSingleSummaryDataPoint(pt pmetric.SummaryDataPoint, resource pcommon.Res
startTimestamp := pt.StartTimestamp()
if settings.ExportCreatedMetric && startTimestamp != 0 {
createdLabels := createLabels(baseName + createdSuffix)
addCreatedTimeSeriesIfNeeded(tsMap, createdLabels, startTimestamp, metric.Type().String())
addCreatedTimeSeriesIfNeeded(tsMap, createdLabels, startTimestamp, pt.Timestamp(), metric.Type().String())
}
}
@ -528,15 +520,17 @@ func addCreatedTimeSeriesIfNeeded(
series map[string]*prompb.TimeSeries,
labels []prompb.Label,
startTimestamp pcommon.Timestamp,
timestamp pcommon.Timestamp,
metricType string,
) {
sig := timeSeriesSignature(metricType, &labels)
sig := timeSeriesSignature(metricType, labels)
if _, ok := series[sig]; !ok {
series[sig] = &prompb.TimeSeries{
Labels: labels,
Samples: []prompb.Sample{
{ // convert ns to ms
Value: float64(convertTimeStamp(startTimestamp)),
Value: float64(convertTimeStamp(startTimestamp)),
Timestamp: convertTimeStamp(timestamp),
},
},
}
@ -570,7 +564,7 @@ func addResourceTargetInfo(resource pcommon.Resource, settings Settings, timesta
if len(settings.Namespace) > 0 {
name = settings.Namespace + "_" + name
}
labels := createAttributes(resource, attributes, settings.ExternalLabels, nameStr, name)
labels := createAttributes(resource, attributes, settings.ExternalLabels, model.MetricNameLabel, name)
sample := &prompb.Sample{
Value: float64(1),
// convert ns to ms

17
storage/remote/otlptranslator/prometheusremotewrite/histograms.go
View file

@ -29,12 +29,13 @@ func addSingleExponentialHistogramDataPoint(
resource,
pt.Attributes(),
settings.ExternalLabels,
model.MetricNameLabel, metric,
model.MetricNameLabel,
metric,
)
sig := timeSeriesSignature(
pmetric.MetricTypeExponentialHistogram.String(),
&labels,
labels,
)
ts, ok := series[sig]
if !ok {
@ -76,7 +77,17 @@ func exponentialToNativeHistogram(p pmetric.ExponentialHistogramDataPoint) (prom
nSpans, nDeltas := convertBucketsLayout(p.Negative(), scaleDown)
h := prompb.Histogram{
Schema: scale,
// The counter reset detection must be compatible with Prometheus to
// safely set ResetHint to NO. This is not ensured currently.
// Sending a sample that triggers counter reset but with ResetHint==NO
// would lead to Prometheus panic as it does not double check the hint.
// Thus we're explicitly saying UNKNOWN here, which is always safe.
// TODO: using created time stamp should be accurate, but we
// need to know here if it was used for the detection.
// Ref: https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/28663#issuecomment-1810577303
// Counter reset detection in Prometheus: https://github.com/prometheus/prometheus/blob/f997c72f294c0f18ca13fa06d51889af04135195/tsdb/chunkenc/histogram.go#L232
ResetHint: prompb.Histogram_UNKNOWN,
Schema: scale,
ZeroCount: &prompb.Histogram_ZeroCountInt{ZeroCountInt: p.ZeroCount()},
// TODO use zero_threshold, if set, see

16
storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go
View file

@ -23,9 +23,10 @@ type Settings struct {
DisableTargetInfo bool
ExportCreatedMetric bool
AddMetricSuffixes bool
SendMetadata bool
}
// FromMetrics converts pmetric.Metrics to prometheus remote write format.
// FromMetrics converts pmetric.Metrics to Prometheus remote write format.
func FromMetrics(md pmetric.Metrics, settings Settings) (tsMap map[string]*prompb.TimeSeries, errs error) {
tsMap = make(map[string]*prompb.TimeSeries)
@ -51,6 +52,8 @@ func FromMetrics(md pmetric.Metrics, settings Settings) (tsMap map[string]*promp
continue
}
promName := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes)
// handle individual metric based on type
//exhaustive:enforce
switch metric.Type() {
@ -60,7 +63,7 @@ func FromMetrics(md pmetric.Metrics, settings Settings) (tsMap map[string]*promp
errs = multierr.Append(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name()))
}
for x := 0; x < dataPoints.Len(); x++ {
addSingleGaugeNumberDataPoint(dataPoints.At(x), resource, metric, settings, tsMap)
addSingleGaugeNumberDataPoint(dataPoints.At(x), resource, metric, settings, tsMap, promName)
}
case pmetric.MetricTypeSum:
dataPoints := metric.Sum().DataPoints()
@ -68,7 +71,7 @@ func FromMetrics(md pmetric.Metrics, settings Settings) (tsMap map[string]*promp
errs = multierr.Append(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name()))
}
for x := 0; x < dataPoints.Len(); x++ {
addSingleSumNumberDataPoint(dataPoints.At(x), resource, metric, settings, tsMap)
addSingleSumNumberDataPoint(dataPoints.At(x), resource, metric, settings, tsMap, promName)
}
case pmetric.MetricTypeHistogram:
dataPoints := metric.Histogram().DataPoints()
@ -76,19 +79,18 @@ func FromMetrics(md pmetric.Metrics, settings Settings) (tsMap map[string]*promp
errs = multierr.Append(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name()))
}
for x := 0; x < dataPoints.Len(); x++ {
addSingleHistogramDataPoint(dataPoints.At(x), resource, metric, settings, tsMap)
addSingleHistogramDataPoint(dataPoints.At(x), resource, metric, settings, tsMap, promName)
}
case pmetric.MetricTypeExponentialHistogram:
dataPoints := metric.ExponentialHistogram().DataPoints()
if dataPoints.Len() == 0 {
errs = multierr.Append(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name()))
}
name := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes)
for x := 0; x < dataPoints.Len(); x++ {
errs = multierr.Append(
errs,
addSingleExponentialHistogramDataPoint(
name,
promName,
dataPoints.At(x),
resource,
settings,
@ -102,7 +104,7 @@ func FromMetrics(md pmetric.Metrics, settings Settings) (tsMap map[string]*promp
errs = multierr.Append(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name()))
}
for x := 0; x < dataPoints.Len(); x++ {
addSingleSummaryDataPoint(dataPoints.At(x), resource, metric, settings, tsMap)
addSingleSummaryDataPoint(dataPoints.At(x), resource, metric, settings, tsMap, promName)
}
default:
errs = multierr.Append(errs, errors.New("unsupported metric type"))

34
storage/remote/otlptranslator/prometheusremotewrite/number_data_points.go
View file

@ -13,11 +13,9 @@ import (
"github.com/prometheus/prometheus/prompb"
"go.opentelemetry.io/collector/pdata/pcommon"
"go.opentelemetry.io/collector/pdata/pmetric"
prometheustranslator "github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus"
)
// addSingleSumNumberDataPoint converts the Gauge metric data point to a
// addSingleGaugeNumberDataPoint converts the Gauge metric data point to a
// Prometheus time series with samples and labels. The result is stored in the
// series map.
func addSingleGaugeNumberDataPoint(
@ -26,13 +24,14 @@ func addSingleGaugeNumberDataPoint(
metric pmetric.Metric,
settings Settings,
series map[string]*prompb.TimeSeries,
name string,
) {
name := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes)
labels := createAttributes(
resource,
pt.Attributes(),
settings.ExternalLabels,
model.MetricNameLabel, name,
model.MetricNameLabel,
name,
)
sample := &prompb.Sample{
// convert ns to ms
@ -59,8 +58,8 @@ func addSingleSumNumberDataPoint(
metric pmetric.Metric,
settings Settings,
series map[string]*prompb.TimeSeries,
name string,
) {
name := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes)
labels := createAttributes(
resource,
pt.Attributes(),
@ -82,7 +81,7 @@ func addSingleSumNumberDataPoint(
}
sig := addSample(series, sample, labels, metric.Type().String())
if ts, ok := series[sig]; sig != "" && ok {
if ts := series[sig]; sig != "" && ts != nil {
exemplars := getPromExemplars[pmetric.NumberDataPoint](pt)
ts.Exemplars = append(ts.Exemplars, exemplars...)
}
@ -90,15 +89,18 @@ func addSingleSumNumberDataPoint(
// add _created time series if needed
if settings.ExportCreatedMetric && metric.Sum().IsMonotonic() {
startTimestamp := pt.StartTimestamp()
if startTimestamp != 0 {
createdLabels := createAttributes(
resource,
pt.Attributes(),
settings.ExternalLabels,
nameStr,
name+createdSuffix,
)
addCreatedTimeSeriesIfNeeded(series, createdLabels, startTimestamp, metric.Type().String())
if startTimestamp == 0 {
return
}
createdLabels := make([]prompb.Label, len(labels))
copy(createdLabels, labels)
for i, l := range createdLabels {
if l.Name == model.MetricNameLabel {
createdLabels[i].Value = name + createdSuffix
break
}
}
addCreatedTimeSeriesIfNeeded(series, createdLabels, startTimestamp, pt.Timestamp(), metric.Type().String())
}
}

66
storage/remote/otlptranslator/prometheusremotewrite/otlp_to_openmetrics_metadata.go Normal file
View file

@ -0,0 +1,66 @@
// DO NOT EDIT. COPIED AS-IS. SEE ../README.md
// Copyright The OpenTelemetry Authors
// SPDX-License-Identifier: Apache-2.0
package prometheusremotewrite // import "github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite"
import (
"github.com/prometheus/prometheus/prompb"
"go.opentelemetry.io/collector/pdata/pmetric"
prometheustranslator "github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus"
)
func otelMetricTypeToPromMetricType(otelMetric pmetric.Metric) prompb.MetricMetadata_MetricType {
switch otelMetric.Type() {
case pmetric.MetricTypeGauge:
return prompb.MetricMetadata_GAUGE
case pmetric.MetricTypeSum:
metricType := prompb.MetricMetadata_GAUGE
if otelMetric.Sum().IsMonotonic() {
metricType = prompb.MetricMetadata_COUNTER
}
return metricType
case pmetric.MetricTypeHistogram:
return prompb.MetricMetadata_HISTOGRAM
case pmetric.MetricTypeSummary:
return prompb.MetricMetadata_SUMMARY
case pmetric.MetricTypeExponentialHistogram:
return prompb.MetricMetadata_HISTOGRAM
}
return prompb.MetricMetadata_UNKNOWN
}
func OtelMetricsToMetadata(md pmetric.Metrics, addMetricSuffixes bool) []*prompb.MetricMetadata {
resourceMetricsSlice := md.ResourceMetrics()
metadataLength := 0
for i := 0; i < resourceMetricsSlice.Len(); i++ {
scopeMetricsSlice := resourceMetricsSlice.At(i).ScopeMetrics()
for j := 0; j < scopeMetricsSlice.Len(); j++ {
metadataLength += scopeMetricsSlice.At(j).Metrics().Len()
}
}
var metadata = make([]*prompb.MetricMetadata, 0, metadataLength)
for i := 0; i < resourceMetricsSlice.Len(); i++ {
resourceMetrics := resourceMetricsSlice.At(i)
scopeMetricsSlice := resourceMetrics.ScopeMetrics()
for j := 0; j < scopeMetricsSlice.Len(); j++ {
scopeMetrics := scopeMetricsSlice.At(j)
for k := 0; k < scopeMetrics.Metrics().Len(); k++ {
metric := scopeMetrics.Metrics().At(k)
entry := prompb.MetricMetadata{
Type: otelMetricTypeToPromMetricType(metric),
MetricFamilyName: prometheustranslator.BuildCompliantName(metric, "", addMetricSuffixes),
Help: metric.Description(),
}
metadata = append(metadata, &entry)
}
}
}
return metadata
}

2
storage/remote/otlptranslator/update-copy.sh
View file

@ -1,7 +1,7 @@
#!/bin/bash
set -xe
OTEL_VERSION=v0.88.0
OTEL_VERSION=v0.95.0
git clone https://github.com/open-telemetry/opentelemetry-collector-contrib ./tmp
cd ./tmp