mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2025-03-05 20:59:13 -08:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Parse JSON based native histogram

Browse source 
Signed-off-by: Chris Marchbanks <csmarchbanks@gmail.com>
This commit is contained in:
Chris Marchbanks 2024-01-02 16:33:28 -07:00
parent 756202aa4f
commit a87c7b5084
No known key found for this signature in database
GPG key ID: B7FD940BC86A8E7A
3 changed files with 113 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
model/textparse/openmetricsparse.go
View file

@ -17,6 +17,7 @@
package textparse
import (
"bytes"
"errors"
"fmt"
"io"
@ -24,12 +25,14 @@ import (
"strings"
"unicode/utf8"
"github.com/gogo/protobuf/jsonpb"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/model/exemplar"
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/model/value"
dto "github.com/prometheus/prometheus/prompb/io/prometheus/client"
)
type openMetricsLexer struct {
@ -77,7 +80,9 @@ type OpenMetricsParser struct {
series []byte
text []byte
mtype model.MetricType
mname []byte
val float64
h *histogram.Histogram
ts int64
hasTS bool
start int
@ -105,10 +110,13 @@ func (p *OpenMetricsParser) Series() ([]byte, *int64, float64) {
return p.series, nil, p.val
}
// Histogram returns (nil, nil, nil, nil) for now because OpenMetrics does not
// support sparse histograms yet.
// Histogram returns the bytes of the series, the timestamp if set, and the parsed histogram. Currently float histograms are not supported in the text format.
func (p *OpenMetricsParser) Histogram() ([]byte, *int64, *histogram.Histogram, *histogram.FloatHistogram) {
return nil, nil, nil, nil
if p.hasTS {
ts := p.ts
return p.series, &ts, p.h, nil
}
return p.series, nil, p.h, nil
}
// Help returns the metric name and help text in the current entry.
@ -129,7 +137,7 @@ func (p *OpenMetricsParser) Help() ([]byte, []byte) {
// Must only be called after Next returned a type entry.
// The returned byte slices become invalid after the next call to Next.
func (p *OpenMetricsParser) Type() ([]byte, model.MetricType) {
return p.l.b[p.offsets[0]:p.offsets[1]], p.mtype
return p.mname, p.mtype
}
// Unit returns the metric name and unit in the current entry.
@ -242,6 +250,7 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
p.offsets = p.offsets[:0]
p.eOffsets = p.eOffsets[:0]
p.exemplar = p.exemplar[:0]
p.h = nil
p.exemplarVal = 0
p.hasExemplarTs = false
@ -292,6 +301,7 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
default:
return EntryInvalid, fmt.Errorf("invalid metric type %q", s)
}
p.mname = p.l.b[p.offsets[0]:p.offsets[1]]
case tHelp:
if !utf8.Valid(p.text) {
return EntryInvalid, fmt.Errorf("help text %q is not a valid utf8 string", p.text)
@ -315,7 +325,8 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
case tMName:
p.offsets = append(p.offsets, p.l.i)
p.series = p.l.b[p.start:p.l.i]
name := p.l.b[p.start:p.l.i]
p.series = name
t2 := p.nextToken()
if t2 == tBraceOpen {
@ -326,7 +337,14 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
p.series = p.l.b[p.start:p.l.i]
t2 = p.nextToken()
}
p.val, err = p.getFloatValue(t2, "metric")
// We are parsing a native histogram if the name of this series matches
// the name from the type metadata.
if (p.mtype == model.MetricTypeGaugeHistogram || p.mtype == model.MetricTypeHistogram) &&
bytes.Equal(name, p.mname) {
p.h, err = p.getHistogramValue(t2)
} else {
p.val, err = p.getFloatValue(t2, "metric")
}
if err != nil {
return EntryInvalid, err
}
@ -364,6 +382,9 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
default:
return EntryInvalid, p.parseError("expected timestamp or # symbol", t2)
}
if p.h != nil {
return EntryHistogram, nil
}
return EntrySeries, nil
default:
@ -477,3 +498,23 @@ func (p *OpenMetricsParser) getFloatValue(t token, after string) (float64, error
}
return val, nil
}
func (p *OpenMetricsParser) getHistogramValue(t token) (*histogram.Histogram, error) {
if t != tValue {
return nil, p.parseError("expected value after metric", t)
}
h := dto.Histogram{}
unparsed := yoloString(p.l.buf()[1:])
err := jsonpb.UnmarshalString(unparsed, &h)
if err != nil {
return nil, err
}
ht := dto.MetricType_HISTOGRAM
if p.mtype == model.MetricTypeGaugeHistogram {
ht = dto.MetricType_GAUGE_HISTOGRAM
}
sh := convertHistogram(&h, ht)
return &sh, nil
}

30
model/textparse/openmetricsparse_test.go
View file

@ -22,6 +22,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/model/exemplar"
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/model/labels"
)
@ -65,7 +66,9 @@ _metric_starting_with_underscore 1
testmetric{_label_starting_with_underscore="foo"} 1
testmetric{label="\"bar\""} 1
# TYPE foo counter
foo_total 17.0 1520879607.789 # {id="counter-test"} 5`
foo_total 17.0 1520879607.789 # {id="counter-test"} 5
# TYPE nativehistogram histogram
nativehistogram {"sample_count":24,"sample_sum":100,"schema":0,"zero_threshold":0.001,"zero_count":4,"positive_span":[{"offset":0,"length":2},{"offset":1,"length":2}],"negative_span":[{"offset":0,"length":2},{"offset":1,"length":2}],"positive_delta":[2,1,-2,3],"negative_delta":[2,1,-2,3]}`
input += "\n# HELP metric foo\x00bar"
input += "\nnull_byte_metric{a=\"abc\x00\"} 1"
@ -78,6 +81,7 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5`
m string
t *int64
v float64
h *histogram.Histogram
typ model.MetricType
help string
unit string
@ -236,6 +240,22 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5`
lset: labels.FromStrings("__name__", "foo_total"),
t: int64p(1520879607789),
e: &exemplar.Exemplar{Labels: labels.FromStrings("id", "counter-test"), Value: 5},
}, {
m: "nativehistogram",
typ: model.MetricTypeHistogram,
}, {
m: "nativehistogram",
h: &histogram.Histogram{
Schema: 0,
ZeroThreshold: 0.001,
ZeroCount: 4,
Sum: 100.0,
Count: 24,
PositiveSpans: []histogram.Span{{Offset: 0, Length: 2}, {Offset: 1, Length: 2}},
NegativeSpans: []histogram.Span{{Offset: 0, Length: 2}, {Offset: 1, Length: 2}},
PositiveBuckets: []int64{2, 1, -2, 3},
NegativeBuckets: []int64{2, 1, -2, 3},
},
}, {
m: "metric",
help: "foo\x00bar",
@ -276,6 +296,12 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5`
require.Equal(t, *exp[i].e, e)
}
case EntryHistogram:
m, ts, h, _ := p.Histogram()
require.Equal(t, exp[i].m, string(m))
require.Equal(t, exp[i].t, ts)
require.Equal(t, exp[i].h, h)
case EntryType:
m, typ := p.Type()
require.Equal(t, exp[i].m, string(m))
@ -619,7 +645,7 @@ func TestOMNullByteHandling(t *testing.T) {
},
{
input: "a{b\x00=\"hiih\"} 1",
err: "expected equal, got \"\\x00\" (\"INVALID\") while parsing: \"a{b\\x00\"",
err: "expected equal, got \"\\x00\" (\"INVALID\") while parsing: \"a{b\\x00\"",
},
{
input: "a\x00{b=\"ddd\"} 1",

66
model/textparse/protobufparse.go
View file

@ -181,29 +181,7 @@ func (p *ProtobufParser) Histogram() ([]byte, *int64, *histogram.Histogram, *his
}
if h.GetSampleCountFloat() > 0 || h.GetZeroCountFloat() > 0 {
// It is a float histogram.
fh := histogram.FloatHistogram{
Count: h.GetSampleCountFloat(),
Sum: h.GetSampleSum(),
ZeroThreshold: h.GetZeroThreshold(),
ZeroCount: h.GetZeroCountFloat(),
Schema: h.GetSchema(),
PositiveSpans: make([]histogram.Span, len(h.GetPositiveSpan())),
PositiveBuckets: h.GetPositiveCount(),
NegativeSpans: make([]histogram.Span, len(h.GetNegativeSpan())),
NegativeBuckets: h.GetNegativeCount(),
}
for i, span := range h.GetPositiveSpan() {
fh.PositiveSpans[i].Offset = span.GetOffset()
fh.PositiveSpans[i].Length = span.GetLength()
}
for i, span := range h.GetNegativeSpan() {
fh.NegativeSpans[i].Offset = span.GetOffset()
fh.NegativeSpans[i].Length = span.GetLength()
}
if p.mf.GetType() == dto.MetricType_GAUGE_HISTOGRAM {
fh.CounterResetHint = histogram.GaugeType
}
fh.Compact(0)
fh := convertFloatHistogram(h, p.mf.GetType())
if ts != 0 {
return p.metricBytes.Bytes(), &ts, nil, &fh
}
@ -213,6 +191,41 @@ func (p *ProtobufParser) Histogram() ([]byte, *int64, *histogram.Histogram, *his
return p.metricBytes.Bytes(), nil, nil, &fh
}
sh := convertHistogram(h, p.mf.GetType())
if ts != 0 {
return p.metricBytes.Bytes(), &ts, &sh, nil
}
return p.metricBytes.Bytes(), nil, &sh, nil
}
func convertFloatHistogram(h *dto.Histogram, t dto.MetricType) histogram.FloatHistogram {
fh := histogram.FloatHistogram{
Count: h.GetSampleCountFloat(),
Sum: h.GetSampleSum(),
ZeroThreshold: h.GetZeroThreshold(),
ZeroCount: h.GetZeroCountFloat(),
Schema: h.GetSchema(),
PositiveSpans: make([]histogram.Span, len(h.GetPositiveSpan())),
PositiveBuckets: h.GetPositiveCount(),
NegativeSpans: make([]histogram.Span, len(h.GetNegativeSpan())),
NegativeBuckets: h.GetNegativeCount(),
}
for i, span := range h.GetPositiveSpan() {
fh.PositiveSpans[i].Offset = span.GetOffset()
fh.PositiveSpans[i].Length = span.GetLength()
}
for i, span := range h.GetNegativeSpan() {
fh.NegativeSpans[i].Offset = span.GetOffset()
fh.NegativeSpans[i].Length = span.GetLength()
}
if t == dto.MetricType_GAUGE_HISTOGRAM {
fh.CounterResetHint = histogram.GaugeType
}
fh.Compact(0)
return fh
}
func convertHistogram(h *dto.Histogram, t dto.MetricType) histogram.Histogram {
sh := histogram.Histogram{
Count: h.GetSampleCount(),
Sum: h.GetSampleSum(),
@ -232,14 +245,11 @@ func (p *ProtobufParser) Histogram() ([]byte, *int64, *histogram.Histogram, *his
sh.NegativeSpans[i].Offset = span.GetOffset()
sh.NegativeSpans[i].Length = span.GetLength()
}
if p.mf.GetType() == dto.MetricType_GAUGE_HISTOGRAM {
if t == dto.MetricType_GAUGE_HISTOGRAM {
sh.CounterResetHint = histogram.GaugeType
}
sh.Compact(0)
if ts != 0 {
return p.metricBytes.Bytes(), &ts, &sh, nil
}
return p.metricBytes.Bytes(), nil, &sh, nil
return sh
}
// Help returns the metric name and help text in the current entry.