Parse JSON based native histogram

Signed-off-by: Chris Marchbanks <csmarchbanks@gmail.com>

model/textparse/openmetricsparse.go

@@ -17,6 +17,7 @@
 package textparse
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"io"
@@ -24,12 +25,14 @@ import (
 	"strings"
 	"unicode/utf8"
 
+	"github.com/gogo/protobuf/jsonpb"
 	"github.com/prometheus/common/model"
 
 	"github.com/prometheus/prometheus/model/exemplar"
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/value"
+	dto "github.com/prometheus/prometheus/prompb/io/prometheus/client"
 )
 
 type openMetricsLexer struct {
@@ -77,7 +80,9 @@ type OpenMetricsParser struct {
 	series  []byte
 	text    []byte
 	mtype   model.MetricType
+	mname   []byte
 	val     float64
+	h       *histogram.Histogram
 	ts      int64
 	hasTS   bool
 	start   int
@@ -105,10 +110,13 @@ func (p *OpenMetricsParser) Series() ([]byte, *int64, float64) {
 	return p.series, nil, p.val
 }
 
-// Histogram returns (nil, nil, nil, nil) for now because OpenMetrics does not
+// Histogram returns the bytes of the series, the timestamp if set, and the parsed histogram. Currently float histograms are not supported in the text format.
-// support sparse histograms yet.
 func (p *OpenMetricsParser) Histogram() ([]byte, *int64, *histogram.Histogram, *histogram.FloatHistogram) {
-	return nil, nil, nil, nil
+	if p.hasTS {
+		ts := p.ts
+		return p.series, &ts, p.h, nil
+	}
+	return p.series, nil, p.h, nil
 }
 
 // Help returns the metric name and help text in the current entry.
@@ -129,7 +137,7 @@ func (p *OpenMetricsParser) Help() ([]byte, []byte) {
 // Must only be called after Next returned a type entry.
 // The returned byte slices become invalid after the next call to Next.
 func (p *OpenMetricsParser) Type() ([]byte, model.MetricType) {
-	return p.l.b[p.offsets[0]:p.offsets[1]], p.mtype
+	return p.mname, p.mtype
 }
 
 // Unit returns the metric name and unit in the current entry.
@@ -242,6 +250,7 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
 	p.offsets = p.offsets[:0]
 	p.eOffsets = p.eOffsets[:0]
 	p.exemplar = p.exemplar[:0]
+	p.h = nil
 	p.exemplarVal = 0
 	p.hasExemplarTs = false
 
@@ -292,6 +301,7 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
 			default:
 				return EntryInvalid, fmt.Errorf("invalid metric type %q", s)
 			}
+			p.mname = p.l.b[p.offsets[0]:p.offsets[1]]
 		case tHelp:
 			if !utf8.Valid(p.text) {
 				return EntryInvalid, fmt.Errorf("help text %q is not a valid utf8 string", p.text)
@@ -315,7 +325,8 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
 
 	case tMName:
 		p.offsets = append(p.offsets, p.l.i)
-		p.series = p.l.b[p.start:p.l.i]
+		name := p.l.b[p.start:p.l.i]
+		p.series = name
 
 		t2 := p.nextToken()
 		if t2 == tBraceOpen {
@@ -326,7 +337,14 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
 			p.series = p.l.b[p.start:p.l.i]
 			t2 = p.nextToken()
 		}
+		// We are parsing a native histogram if the name of this series matches
+		// the name from the type metadata.
+		if (p.mtype == model.MetricTypeGaugeHistogram || p.mtype == model.MetricTypeHistogram) &&
+			bytes.Equal(name, p.mname) {
+			p.h, err = p.getHistogramValue(t2)
+		} else {
 			p.val, err = p.getFloatValue(t2, "metric")
+		}
 		if err != nil {
 			return EntryInvalid, err
 		}
@@ -364,6 +382,9 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
 		default:
 			return EntryInvalid, p.parseError("expected timestamp or # symbol", t2)
 		}
+		if p.h != nil {
+			return EntryHistogram, nil
+		}
 		return EntrySeries, nil
 
 	default:
@@ -477,3 +498,23 @@ func (p *OpenMetricsParser) getFloatValue(t token, after string) (float64, error
 	}
 	return val, nil
 }
+
+func (p *OpenMetricsParser) getHistogramValue(t token) (*histogram.Histogram, error) {
+	if t != tValue {
+		return nil, p.parseError("expected value after metric", t)
+	}
+
+	h := dto.Histogram{}
+	unparsed := yoloString(p.l.buf()[1:])
+	err := jsonpb.UnmarshalString(unparsed, &h)
+	if err != nil {
+		return nil, err
+	}
+
+	ht := dto.MetricType_HISTOGRAM
+	if p.mtype == model.MetricTypeGaugeHistogram {
+		ht = dto.MetricType_GAUGE_HISTOGRAM
+	}
+	sh := convertHistogram(&h, ht)
+	return &sh, nil
+}

model/textparse/openmetricsparse_test.go

@@ -22,6 +22,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/prometheus/prometheus/model/exemplar"
+	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
 )
 
@@ -65,7 +66,9 @@ _metric_starting_with_underscore 1
 testmetric{_label_starting_with_underscore="foo"} 1
 testmetric{label="\"bar\""} 1
 # TYPE foo counter
-foo_total 17.0 1520879607.789 # {id="counter-test"} 5`
+foo_total 17.0 1520879607.789 # {id="counter-test"} 5
+# TYPE nativehistogram histogram
+nativehistogram {"sample_count":24,"sample_sum":100,"schema":0,"zero_threshold":0.001,"zero_count":4,"positive_span":[{"offset":0,"length":2},{"offset":1,"length":2}],"negative_span":[{"offset":0,"length":2},{"offset":1,"length":2}],"positive_delta":[2,1,-2,3],"negative_delta":[2,1,-2,3]}`
 
 	input += "\n# HELP metric foo\x00bar"
 	input += "\nnull_byte_metric{a=\"abc\x00\"} 1"
@@ -78,6 +81,7 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5`
 		m       string
 		t       *int64
 		v       float64
+		h       *histogram.Histogram
 		typ     model.MetricType
 		help    string
 		unit    string
@@ -236,6 +240,22 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5`
 			lset: labels.FromStrings("__name__", "foo_total"),
 			t:    int64p(1520879607789),
 			e:    &exemplar.Exemplar{Labels: labels.FromStrings("id", "counter-test"), Value: 5},
+		}, {
+			m:   "nativehistogram",
+			typ: model.MetricTypeHistogram,
+		}, {
+			m: "nativehistogram",
+			h: &histogram.Histogram{
+				Schema:          0,
+				ZeroThreshold:   0.001,
+				ZeroCount:       4,
+				Sum:             100.0,
+				Count:           24,
+				PositiveSpans:   []histogram.Span{{Offset: 0, Length: 2}, {Offset: 1, Length: 2}},
+				NegativeSpans:   []histogram.Span{{Offset: 0, Length: 2}, {Offset: 1, Length: 2}},
+				PositiveBuckets: []int64{2, 1, -2, 3},
+				NegativeBuckets: []int64{2, 1, -2, 3},
+			},
 		}, {
 			m:    "metric",
 			help: "foo\x00bar",
@@ -276,6 +296,12 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5`
 				require.Equal(t, *exp[i].e, e)
 			}
 
+		case EntryHistogram:
+			m, ts, h, _ := p.Histogram()
+			require.Equal(t, exp[i].m, string(m))
+			require.Equal(t, exp[i].t, ts)
+			require.Equal(t, exp[i].h, h)
+
 		case EntryType:
 			m, typ := p.Type()
 			require.Equal(t, exp[i].m, string(m))

model/textparse/protobufparse.go

@@ -181,6 +181,24 @@ func (p *ProtobufParser) Histogram() ([]byte, *int64, *histogram.Histogram, *his
 	}
 	if h.GetSampleCountFloat() > 0 || h.GetZeroCountFloat() > 0 {
 		// It is a float histogram.
+		fh := convertFloatHistogram(h, p.mf.GetType())
+		if ts != 0 {
+			return p.metricBytes.Bytes(), &ts, nil, &fh
+		}
+		// Nasty hack: Assume that ts==0 means no timestamp. That's not true in
+		// general, but proto3 has no distinction between unset and
+		// default. Need to avoid in the final format.
+		return p.metricBytes.Bytes(), nil, nil, &fh
+	}
+
+	sh := convertHistogram(h, p.mf.GetType())
+	if ts != 0 {
+		return p.metricBytes.Bytes(), &ts, &sh, nil
+	}
+	return p.metricBytes.Bytes(), nil, &sh, nil
+}
+
+func convertFloatHistogram(h *dto.Histogram, t dto.MetricType) histogram.FloatHistogram {
 	fh := histogram.FloatHistogram{
 		Count:           h.GetSampleCountFloat(),
 		Sum:             h.GetSampleSum(),
@@ -200,19 +218,14 @@ func (p *ProtobufParser) Histogram() ([]byte, *int64, *histogram.Histogram, *his
 		fh.NegativeSpans[i].Offset = span.GetOffset()
 		fh.NegativeSpans[i].Length = span.GetLength()
 	}
-		if p.mf.GetType() == dto.MetricType_GAUGE_HISTOGRAM {
+	if t == dto.MetricType_GAUGE_HISTOGRAM {
 		fh.CounterResetHint = histogram.GaugeType
 	}
 	fh.Compact(0)
-		if ts != 0 {
+	return fh
-			return p.metricBytes.Bytes(), &ts, nil, &fh
+}
-		}
-		// Nasty hack: Assume that ts==0 means no timestamp. That's not true in
-		// general, but proto3 has no distinction between unset and
-		// default. Need to avoid in the final format.
-		return p.metricBytes.Bytes(), nil, nil, &fh
-	}
 
+func convertHistogram(h *dto.Histogram, t dto.MetricType) histogram.Histogram {
 	sh := histogram.Histogram{
 		Count:           h.GetSampleCount(),
 		Sum:             h.GetSampleSum(),
@@ -232,14 +245,11 @@ func (p *ProtobufParser) Histogram() ([]byte, *int64, *histogram.Histogram, *his
 		sh.NegativeSpans[i].Offset = span.GetOffset()
 		sh.NegativeSpans[i].Length = span.GetLength()
 	}
-	if p.mf.GetType() == dto.MetricType_GAUGE_HISTOGRAM {
+	if t == dto.MetricType_GAUGE_HISTOGRAM {
 		sh.CounterResetHint = histogram.GaugeType
 	}
 	sh.Compact(0)
-	if ts != 0 {
+	return sh
-		return p.metricBytes.Bytes(), &ts, &sh, nil
-	}
-	return p.metricBytes.Bytes(), nil, &sh, nil
 }
 
 // Help returns the metric name and help text in the current entry.