diff --git a/model/textparse/openmetricsparse.go b/model/textparse/openmetricsparse.go index ddfbe4fc5..3efe5ea4a 100644 --- a/model/textparse/openmetricsparse.go +++ b/model/textparse/openmetricsparse.go @@ -17,6 +17,7 @@ package textparse import ( + "bytes" "errors" "fmt" "io" @@ -24,12 +25,14 @@ import ( "strings" "unicode/utf8" + "github.com/gogo/protobuf/jsonpb" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/exemplar" "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/value" + dto "github.com/prometheus/prometheus/prompb/io/prometheus/client" ) type openMetricsLexer struct { @@ -77,7 +80,9 @@ type OpenMetricsParser struct { series []byte text []byte mtype model.MetricType + mname []byte val float64 + h *histogram.Histogram ts int64 hasTS bool start int @@ -105,10 +110,13 @@ func (p *OpenMetricsParser) Series() ([]byte, *int64, float64) { return p.series, nil, p.val } -// Histogram returns (nil, nil, nil, nil) for now because OpenMetrics does not -// support sparse histograms yet. +// Histogram returns the bytes of the series, the timestamp if set, and the parsed histogram. Currently float histograms are not supported in the text format. func (p *OpenMetricsParser) Histogram() ([]byte, *int64, *histogram.Histogram, *histogram.FloatHistogram) { - return nil, nil, nil, nil + if p.hasTS { + ts := p.ts + return p.series, &ts, p.h, nil + } + return p.series, nil, p.h, nil } // Help returns the metric name and help text in the current entry. @@ -129,7 +137,7 @@ func (p *OpenMetricsParser) Help() ([]byte, []byte) { // Must only be called after Next returned a type entry. // The returned byte slices become invalid after the next call to Next. func (p *OpenMetricsParser) Type() ([]byte, model.MetricType) { - return p.l.b[p.offsets[0]:p.offsets[1]], p.mtype + return p.mname, p.mtype } // Unit returns the metric name and unit in the current entry. @@ -242,6 +250,7 @@ func (p *OpenMetricsParser) Next() (Entry, error) { p.offsets = p.offsets[:0] p.eOffsets = p.eOffsets[:0] p.exemplar = p.exemplar[:0] + p.h = nil p.exemplarVal = 0 p.hasExemplarTs = false @@ -292,6 +301,7 @@ func (p *OpenMetricsParser) Next() (Entry, error) { default: return EntryInvalid, fmt.Errorf("invalid metric type %q", s) } + p.mname = p.l.b[p.offsets[0]:p.offsets[1]] case tHelp: if !utf8.Valid(p.text) { return EntryInvalid, fmt.Errorf("help text %q is not a valid utf8 string", p.text) @@ -315,7 +325,8 @@ func (p *OpenMetricsParser) Next() (Entry, error) { case tMName: p.offsets = append(p.offsets, p.l.i) - p.series = p.l.b[p.start:p.l.i] + name := p.l.b[p.start:p.l.i] + p.series = name t2 := p.nextToken() if t2 == tBraceOpen { @@ -326,7 +337,14 @@ func (p *OpenMetricsParser) Next() (Entry, error) { p.series = p.l.b[p.start:p.l.i] t2 = p.nextToken() } - p.val, err = p.getFloatValue(t2, "metric") + // We are parsing a native histogram if the name of this series matches + // the name from the type metadata. + if (p.mtype == model.MetricTypeGaugeHistogram || p.mtype == model.MetricTypeHistogram) && + bytes.Equal(name, p.mname) { + p.h, err = p.getHistogramValue(t2) + } else { + p.val, err = p.getFloatValue(t2, "metric") + } if err != nil { return EntryInvalid, err } @@ -364,6 +382,9 @@ func (p *OpenMetricsParser) Next() (Entry, error) { default: return EntryInvalid, p.parseError("expected timestamp or # symbol", t2) } + if p.h != nil { + return EntryHistogram, nil + } return EntrySeries, nil default: @@ -477,3 +498,23 @@ func (p *OpenMetricsParser) getFloatValue(t token, after string) (float64, error } return val, nil } + +func (p *OpenMetricsParser) getHistogramValue(t token) (*histogram.Histogram, error) { + if t != tValue { + return nil, p.parseError("expected value after metric", t) + } + + h := dto.Histogram{} + unparsed := yoloString(p.l.buf()[1:]) + err := jsonpb.UnmarshalString(unparsed, &h) + if err != nil { + return nil, err + } + + ht := dto.MetricType_HISTOGRAM + if p.mtype == model.MetricTypeGaugeHistogram { + ht = dto.MetricType_GAUGE_HISTOGRAM + } + sh := convertHistogram(&h, ht) + return &sh, nil +} diff --git a/model/textparse/openmetricsparse_test.go b/model/textparse/openmetricsparse_test.go index 2b1d909f3..c09f5a3e4 100644 --- a/model/textparse/openmetricsparse_test.go +++ b/model/textparse/openmetricsparse_test.go @@ -22,6 +22,7 @@ import ( "github.com/stretchr/testify/require" "github.com/prometheus/prometheus/model/exemplar" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" ) @@ -65,7 +66,9 @@ _metric_starting_with_underscore 1 testmetric{_label_starting_with_underscore="foo"} 1 testmetric{label="\"bar\""} 1 # TYPE foo counter -foo_total 17.0 1520879607.789 # {id="counter-test"} 5` +foo_total 17.0 1520879607.789 # {id="counter-test"} 5 +# TYPE nativehistogram histogram +nativehistogram {"sample_count":24,"sample_sum":100,"schema":0,"zero_threshold":0.001,"zero_count":4,"positive_span":[{"offset":0,"length":2},{"offset":1,"length":2}],"negative_span":[{"offset":0,"length":2},{"offset":1,"length":2}],"positive_delta":[2,1,-2,3],"negative_delta":[2,1,-2,3]}` input += "\n# HELP metric foo\x00bar" input += "\nnull_byte_metric{a=\"abc\x00\"} 1" @@ -78,6 +81,7 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5` m string t *int64 v float64 + h *histogram.Histogram typ model.MetricType help string unit string @@ -236,6 +240,22 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5` lset: labels.FromStrings("__name__", "foo_total"), t: int64p(1520879607789), e: &exemplar.Exemplar{Labels: labels.FromStrings("id", "counter-test"), Value: 5}, + }, { + m: "nativehistogram", + typ: model.MetricTypeHistogram, + }, { + m: "nativehistogram", + h: &histogram.Histogram{ + Schema: 0, + ZeroThreshold: 0.001, + ZeroCount: 4, + Sum: 100.0, + Count: 24, + PositiveSpans: []histogram.Span{{Offset: 0, Length: 2}, {Offset: 1, Length: 2}}, + NegativeSpans: []histogram.Span{{Offset: 0, Length: 2}, {Offset: 1, Length: 2}}, + PositiveBuckets: []int64{2, 1, -2, 3}, + NegativeBuckets: []int64{2, 1, -2, 3}, + }, }, { m: "metric", help: "foo\x00bar", @@ -276,6 +296,12 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5` require.Equal(t, *exp[i].e, e) } + case EntryHistogram: + m, ts, h, _ := p.Histogram() + require.Equal(t, exp[i].m, string(m)) + require.Equal(t, exp[i].t, ts) + require.Equal(t, exp[i].h, h) + case EntryType: m, typ := p.Type() require.Equal(t, exp[i].m, string(m)) @@ -619,7 +645,7 @@ func TestOMNullByteHandling(t *testing.T) { }, { input: "a{b\x00=\"hiih\"} 1", - err: "expected equal, got \"\\x00\" (\"INVALID\") while parsing: \"a{b\\x00\"", + err: "expected equal, got \"\\x00\" (\"INVALID\") while parsing: \"a{b\\x00\"", }, { input: "a\x00{b=\"ddd\"} 1", diff --git a/model/textparse/protobufparse.go b/model/textparse/protobufparse.go index 534bbebb2..44d82cbca 100644 --- a/model/textparse/protobufparse.go +++ b/model/textparse/protobufparse.go @@ -181,29 +181,7 @@ func (p *ProtobufParser) Histogram() ([]byte, *int64, *histogram.Histogram, *his } if h.GetSampleCountFloat() > 0 || h.GetZeroCountFloat() > 0 { // It is a float histogram. - fh := histogram.FloatHistogram{ - Count: h.GetSampleCountFloat(), - Sum: h.GetSampleSum(), - ZeroThreshold: h.GetZeroThreshold(), - ZeroCount: h.GetZeroCountFloat(), - Schema: h.GetSchema(), - PositiveSpans: make([]histogram.Span, len(h.GetPositiveSpan())), - PositiveBuckets: h.GetPositiveCount(), - NegativeSpans: make([]histogram.Span, len(h.GetNegativeSpan())), - NegativeBuckets: h.GetNegativeCount(), - } - for i, span := range h.GetPositiveSpan() { - fh.PositiveSpans[i].Offset = span.GetOffset() - fh.PositiveSpans[i].Length = span.GetLength() - } - for i, span := range h.GetNegativeSpan() { - fh.NegativeSpans[i].Offset = span.GetOffset() - fh.NegativeSpans[i].Length = span.GetLength() - } - if p.mf.GetType() == dto.MetricType_GAUGE_HISTOGRAM { - fh.CounterResetHint = histogram.GaugeType - } - fh.Compact(0) + fh := convertFloatHistogram(h, p.mf.GetType()) if ts != 0 { return p.metricBytes.Bytes(), &ts, nil, &fh } @@ -213,6 +191,41 @@ func (p *ProtobufParser) Histogram() ([]byte, *int64, *histogram.Histogram, *his return p.metricBytes.Bytes(), nil, nil, &fh } + sh := convertHistogram(h, p.mf.GetType()) + if ts != 0 { + return p.metricBytes.Bytes(), &ts, &sh, nil + } + return p.metricBytes.Bytes(), nil, &sh, nil +} + +func convertFloatHistogram(h *dto.Histogram, t dto.MetricType) histogram.FloatHistogram { + fh := histogram.FloatHistogram{ + Count: h.GetSampleCountFloat(), + Sum: h.GetSampleSum(), + ZeroThreshold: h.GetZeroThreshold(), + ZeroCount: h.GetZeroCountFloat(), + Schema: h.GetSchema(), + PositiveSpans: make([]histogram.Span, len(h.GetPositiveSpan())), + PositiveBuckets: h.GetPositiveCount(), + NegativeSpans: make([]histogram.Span, len(h.GetNegativeSpan())), + NegativeBuckets: h.GetNegativeCount(), + } + for i, span := range h.GetPositiveSpan() { + fh.PositiveSpans[i].Offset = span.GetOffset() + fh.PositiveSpans[i].Length = span.GetLength() + } + for i, span := range h.GetNegativeSpan() { + fh.NegativeSpans[i].Offset = span.GetOffset() + fh.NegativeSpans[i].Length = span.GetLength() + } + if t == dto.MetricType_GAUGE_HISTOGRAM { + fh.CounterResetHint = histogram.GaugeType + } + fh.Compact(0) + return fh +} + +func convertHistogram(h *dto.Histogram, t dto.MetricType) histogram.Histogram { sh := histogram.Histogram{ Count: h.GetSampleCount(), Sum: h.GetSampleSum(), @@ -232,14 +245,11 @@ func (p *ProtobufParser) Histogram() ([]byte, *int64, *histogram.Histogram, *his sh.NegativeSpans[i].Offset = span.GetOffset() sh.NegativeSpans[i].Length = span.GetLength() } - if p.mf.GetType() == dto.MetricType_GAUGE_HISTOGRAM { + if t == dto.MetricType_GAUGE_HISTOGRAM { sh.CounterResetHint = histogram.GaugeType } sh.Compact(0) - if ts != 0 { - return p.metricBytes.Bytes(), &ts, &sh, nil - } - return p.metricBytes.Bytes(), nil, &sh, nil + return sh } // Help returns the metric name and help text in the current entry.