mirror of
https://github.com/prometheus/prometheus.git
synced 2024-11-13 17:14:05 -08:00
f7df3b86ba
* histograms: parse float histograms from proto definition Signed-off-by: Marc Tuduri <marctc@protonmail.com> * Improve comment Signed-off-by: Marc Tuduri <marctc@protonmail.com> * Ignore float buckets Signed-off-by: Marc Tuduri <marctc@protonmail.com> * Refactor Histogram() function Signed-off-by: Marc Tuduri <marctc@protonmail.com> * Fix test_float_histogram Signed-off-by: Marc Tuduri <marctc@protonmail.com> * Update model/textparse/protobufparse.go Co-authored-by: Ganesh Vernekar <15064823+codesome@users.noreply.github.com> Signed-off-by: Marc Tudurí <marctc@protonmail.com> * Update protobufparse.go Signed-off-by: Marc Tudurí <marctc@protonmail.com> * Update scrape.go Signed-off-by: Marc Tudurí <marctc@protonmail.com> * Update scrape/scrape.go Co-authored-by: Ganesh Vernekar <15064823+codesome@users.noreply.github.com> Signed-off-by: Marc Tudurí <marctc@protonmail.com> Signed-off-by: Marc Tuduri <marctc@protonmail.com> Signed-off-by: Marc Tudurí <marctc@protonmail.com> Co-authored-by: Ganesh Vernekar <15064823+codesome@users.noreply.github.com>
682 lines
15 KiB
Go
682 lines
15 KiB
Go
// Copyright 2021 The Prometheus Authors
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package textparse
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/binary"
|
|
"io"
|
|
"testing"
|
|
|
|
"github.com/gogo/protobuf/proto"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/prometheus/prometheus/model/exemplar"
|
|
"github.com/prometheus/prometheus/model/histogram"
|
|
"github.com/prometheus/prometheus/model/labels"
|
|
|
|
dto "github.com/prometheus/prometheus/prompb/io/prometheus/client"
|
|
)
|
|
|
|
func TestProtobufParse(t *testing.T) {
|
|
textMetricFamilies := []string{
|
|
`name: "go_build_info"
|
|
help: "Build information about the main Go module."
|
|
type: GAUGE
|
|
metric: <
|
|
label: <
|
|
name: "checksum"
|
|
value: ""
|
|
>
|
|
label: <
|
|
name: "path"
|
|
value: "github.com/prometheus/client_golang"
|
|
>
|
|
label: <
|
|
name: "version"
|
|
value: "(devel)"
|
|
>
|
|
gauge: <
|
|
value: 1
|
|
>
|
|
>
|
|
|
|
`,
|
|
`name: "go_memstats_alloc_bytes_total"
|
|
help: "Total number of bytes allocated, even if freed."
|
|
type: COUNTER
|
|
metric: <
|
|
counter: <
|
|
value: 1.546544e+06
|
|
exemplar: <
|
|
label: <
|
|
name: "dummyID"
|
|
value: "42"
|
|
>
|
|
value: 12
|
|
timestamp: <
|
|
seconds: 1625851151
|
|
nanos: 233181499
|
|
>
|
|
>
|
|
>
|
|
>
|
|
|
|
`,
|
|
`name: "something_untyped"
|
|
help: "Just to test the untyped type."
|
|
type: UNTYPED
|
|
metric: <
|
|
untyped: <
|
|
value: 42
|
|
>
|
|
timestamp_ms: 1234567
|
|
>
|
|
|
|
`,
|
|
`name: "test_histogram"
|
|
help: "Test histogram with many buckets removed to keep it manageable in size."
|
|
type: HISTOGRAM
|
|
metric: <
|
|
histogram: <
|
|
sample_count: 175
|
|
sample_sum: 0.0008280461746287094
|
|
bucket: <
|
|
cumulative_count: 2
|
|
upper_bound: -0.0004899999999999998
|
|
>
|
|
bucket: <
|
|
cumulative_count: 4
|
|
upper_bound: -0.0003899999999999998
|
|
exemplar: <
|
|
label: <
|
|
name: "dummyID"
|
|
value: "59727"
|
|
>
|
|
value: -0.00039
|
|
timestamp: <
|
|
seconds: 1625851155
|
|
nanos: 146848499
|
|
>
|
|
>
|
|
>
|
|
bucket: <
|
|
cumulative_count: 16
|
|
upper_bound: -0.0002899999999999998
|
|
exemplar: <
|
|
label: <
|
|
name: "dummyID"
|
|
value: "5617"
|
|
>
|
|
value: -0.00029
|
|
>
|
|
>
|
|
schema: 3
|
|
zero_threshold: 2.938735877055719e-39
|
|
zero_count: 2
|
|
negative_span: <
|
|
offset: -162
|
|
length: 1
|
|
>
|
|
negative_span: <
|
|
offset: 23
|
|
length: 4
|
|
>
|
|
negative_delta: 1
|
|
negative_delta: 3
|
|
negative_delta: -2
|
|
negative_delta: -1
|
|
negative_delta: 1
|
|
positive_span: <
|
|
offset: -161
|
|
length: 1
|
|
>
|
|
positive_span: <
|
|
offset: 8
|
|
length: 3
|
|
>
|
|
positive_delta: 1
|
|
positive_delta: 2
|
|
positive_delta: -1
|
|
positive_delta: -1
|
|
>
|
|
timestamp_ms: 1234568
|
|
>
|
|
|
|
`,
|
|
|
|
`name: "test_float_histogram"
|
|
help: "Test float histogram with many buckets removed to keep it manageable in size."
|
|
type: HISTOGRAM
|
|
metric: <
|
|
histogram: <
|
|
sample_count: 175
|
|
sample_count_float: 175.0
|
|
sample_sum: 0.0008280461746287094
|
|
bucket: <
|
|
cumulative_count_float: 2.0
|
|
upper_bound: -0.0004899999999999998
|
|
>
|
|
bucket: <
|
|
cumulative_count_float: 4.0
|
|
upper_bound: -0.0003899999999999998
|
|
exemplar: <
|
|
label: <
|
|
name: "dummyID"
|
|
value: "59727"
|
|
>
|
|
value: -0.00039
|
|
timestamp: <
|
|
seconds: 1625851155
|
|
nanos: 146848499
|
|
>
|
|
>
|
|
>
|
|
bucket: <
|
|
cumulative_count_float: 16
|
|
upper_bound: -0.0002899999999999998
|
|
exemplar: <
|
|
label: <
|
|
name: "dummyID"
|
|
value: "5617"
|
|
>
|
|
value: -0.00029
|
|
>
|
|
>
|
|
schema: 3
|
|
zero_threshold: 2.938735877055719e-39
|
|
zero_count_float: 2.0
|
|
negative_span: <
|
|
offset: -162
|
|
length: 1
|
|
>
|
|
negative_span: <
|
|
offset: 23
|
|
length: 4
|
|
>
|
|
negative_count: 1.0
|
|
negative_count: 3.0
|
|
negative_count: -2.0
|
|
negative_count: -1.0
|
|
negative_count: 1.0
|
|
positive_span: <
|
|
offset: -161
|
|
length: 1
|
|
>
|
|
positive_span: <
|
|
offset: 8
|
|
length: 3
|
|
>
|
|
positive_count: 1.0
|
|
positive_count: 2.0
|
|
positive_count: -1.0
|
|
positive_count: -1.0
|
|
>
|
|
timestamp_ms: 1234568
|
|
>
|
|
|
|
`,
|
|
`name: "test_histogram2"
|
|
help: "Similar histogram as before but now without sparse buckets."
|
|
type: HISTOGRAM
|
|
metric: <
|
|
histogram: <
|
|
sample_count: 175
|
|
sample_sum: 0.000828
|
|
bucket: <
|
|
cumulative_count: 2
|
|
upper_bound: -0.00048
|
|
>
|
|
bucket: <
|
|
cumulative_count: 4
|
|
upper_bound: -0.00038
|
|
exemplar: <
|
|
label: <
|
|
name: "dummyID"
|
|
value: "59727"
|
|
>
|
|
value: -0.00038
|
|
timestamp: <
|
|
seconds: 1625851153
|
|
nanos: 146848499
|
|
>
|
|
>
|
|
>
|
|
bucket: <
|
|
cumulative_count: 16
|
|
upper_bound: 1
|
|
exemplar: <
|
|
label: <
|
|
name: "dummyID"
|
|
value: "5617"
|
|
>
|
|
value: -0.000295
|
|
>
|
|
>
|
|
schema: 0
|
|
zero_threshold: 0
|
|
>
|
|
>
|
|
|
|
`,
|
|
`name: "rpc_durations_seconds"
|
|
help: "RPC latency distributions."
|
|
type: SUMMARY
|
|
metric: <
|
|
label: <
|
|
name: "service"
|
|
value: "exponential"
|
|
>
|
|
summary: <
|
|
sample_count: 262
|
|
sample_sum: 0.00025551262820703587
|
|
quantile: <
|
|
quantile: 0.5
|
|
value: 6.442786329648548e-07
|
|
>
|
|
quantile: <
|
|
quantile: 0.9
|
|
value: 1.9435742936658396e-06
|
|
>
|
|
quantile: <
|
|
quantile: 0.99
|
|
value: 4.0471608667037015e-06
|
|
>
|
|
>
|
|
>
|
|
`,
|
|
`name: "without_quantiles"
|
|
help: "A summary without quantiles."
|
|
type: SUMMARY
|
|
metric: <
|
|
summary: <
|
|
sample_count: 42
|
|
sample_sum: 1.234
|
|
>
|
|
>
|
|
`,
|
|
}
|
|
|
|
varintBuf := make([]byte, binary.MaxVarintLen32)
|
|
inputBuf := &bytes.Buffer{}
|
|
|
|
for _, tmf := range textMetricFamilies {
|
|
pb := &dto.MetricFamily{}
|
|
// From text to proto message.
|
|
require.NoError(t, proto.UnmarshalText(tmf, pb))
|
|
// From proto message to binary protobuf.
|
|
protoBuf, err := proto.Marshal(pb)
|
|
require.NoError(t, err)
|
|
|
|
// Write first length, then binary protobuf.
|
|
varintLength := binary.PutUvarint(varintBuf, uint64(len(protoBuf)))
|
|
inputBuf.Write(varintBuf[:varintLength])
|
|
inputBuf.Write(protoBuf)
|
|
}
|
|
|
|
exp := []struct {
|
|
lset labels.Labels
|
|
m string
|
|
t int64
|
|
v float64
|
|
typ MetricType
|
|
help string
|
|
unit string
|
|
comment string
|
|
shs *histogram.Histogram
|
|
fhs *histogram.FloatHistogram
|
|
e []exemplar.Exemplar
|
|
}{
|
|
{
|
|
m: "go_build_info",
|
|
help: "Build information about the main Go module.",
|
|
},
|
|
{
|
|
m: "go_build_info",
|
|
typ: MetricTypeGauge,
|
|
},
|
|
{
|
|
m: "go_build_info\xFFchecksum\xFF\xFFpath\xFFgithub.com/prometheus/client_golang\xFFversion\xFF(devel)",
|
|
v: 1,
|
|
lset: labels.FromStrings(
|
|
"__name__", "go_build_info",
|
|
"checksum", "",
|
|
"path", "github.com/prometheus/client_golang",
|
|
"version", "(devel)",
|
|
),
|
|
},
|
|
{
|
|
m: "go_memstats_alloc_bytes_total",
|
|
help: "Total number of bytes allocated, even if freed.",
|
|
},
|
|
{
|
|
m: "go_memstats_alloc_bytes_total",
|
|
typ: MetricTypeCounter,
|
|
},
|
|
{
|
|
m: "go_memstats_alloc_bytes_total",
|
|
v: 1.546544e+06,
|
|
lset: labels.FromStrings(
|
|
"__name__", "go_memstats_alloc_bytes_total",
|
|
),
|
|
e: []exemplar.Exemplar{
|
|
{Labels: labels.FromStrings("dummyID", "42"), Value: 12, HasTs: true, Ts: 1625851151233},
|
|
},
|
|
},
|
|
{
|
|
m: "something_untyped",
|
|
help: "Just to test the untyped type.",
|
|
},
|
|
{
|
|
m: "something_untyped",
|
|
typ: MetricTypeUnknown,
|
|
},
|
|
{
|
|
m: "something_untyped",
|
|
t: 1234567,
|
|
v: 42,
|
|
lset: labels.FromStrings(
|
|
"__name__", "something_untyped",
|
|
),
|
|
},
|
|
{
|
|
m: "test_histogram",
|
|
help: "Test histogram with many buckets removed to keep it manageable in size.",
|
|
},
|
|
{
|
|
m: "test_histogram",
|
|
typ: MetricTypeHistogram,
|
|
},
|
|
{
|
|
m: "test_histogram",
|
|
t: 1234568,
|
|
shs: &histogram.Histogram{
|
|
Count: 175,
|
|
ZeroCount: 2,
|
|
Sum: 0.0008280461746287094,
|
|
ZeroThreshold: 2.938735877055719e-39,
|
|
Schema: 3,
|
|
PositiveSpans: []histogram.Span{
|
|
{Offset: -161, Length: 1},
|
|
{Offset: 8, Length: 3},
|
|
},
|
|
NegativeSpans: []histogram.Span{
|
|
{Offset: -162, Length: 1},
|
|
{Offset: 23, Length: 4},
|
|
},
|
|
PositiveBuckets: []int64{1, 2, -1, -1},
|
|
NegativeBuckets: []int64{1, 3, -2, -1, 1},
|
|
},
|
|
lset: labels.FromStrings(
|
|
"__name__", "test_histogram",
|
|
),
|
|
e: []exemplar.Exemplar{
|
|
{Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00039, HasTs: true, Ts: 1625851155146},
|
|
{Labels: labels.FromStrings("dummyID", "5617"), Value: -0.00029, HasTs: false},
|
|
},
|
|
},
|
|
{
|
|
m: "test_float_histogram",
|
|
help: "Test float histogram with many buckets removed to keep it manageable in size.",
|
|
},
|
|
{
|
|
m: "test_float_histogram",
|
|
typ: MetricTypeHistogram,
|
|
},
|
|
{
|
|
m: "test_float_histogram",
|
|
t: 1234568,
|
|
fhs: &histogram.FloatHistogram{
|
|
Count: 175.0,
|
|
ZeroCount: 2.0,
|
|
Sum: 0.0008280461746287094,
|
|
ZeroThreshold: 2.938735877055719e-39,
|
|
Schema: 3,
|
|
PositiveSpans: []histogram.Span{
|
|
{Offset: -161, Length: 1},
|
|
{Offset: 8, Length: 3},
|
|
},
|
|
NegativeSpans: []histogram.Span{
|
|
{Offset: -162, Length: 1},
|
|
{Offset: 23, Length: 4},
|
|
},
|
|
PositiveBuckets: []float64{1.0, 2.0, -1.0, -1.0},
|
|
NegativeBuckets: []float64{1.0, 3.0, -2.0, -1.0, 1.0},
|
|
},
|
|
lset: labels.FromStrings(
|
|
"__name__", "test_float_histogram",
|
|
),
|
|
e: []exemplar.Exemplar{
|
|
{Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00039, HasTs: true, Ts: 1625851155146},
|
|
{Labels: labels.FromStrings("dummyID", "5617"), Value: -0.00029, HasTs: false},
|
|
},
|
|
},
|
|
{
|
|
m: "test_histogram2",
|
|
help: "Similar histogram as before but now without sparse buckets.",
|
|
},
|
|
{
|
|
m: "test_histogram2",
|
|
typ: MetricTypeHistogram,
|
|
},
|
|
{
|
|
m: "test_histogram2_count",
|
|
v: 175,
|
|
lset: labels.FromStrings(
|
|
"__name__", "test_histogram2_count",
|
|
),
|
|
},
|
|
{
|
|
m: "test_histogram2_sum",
|
|
v: 0.000828,
|
|
lset: labels.FromStrings(
|
|
"__name__", "test_histogram2_sum",
|
|
),
|
|
},
|
|
{
|
|
m: "test_histogram2_bucket\xffle\xff-0.00048",
|
|
v: 2,
|
|
lset: labels.FromStrings(
|
|
"__name__", "test_histogram2_bucket",
|
|
"le", "-0.00048",
|
|
),
|
|
},
|
|
{
|
|
m: "test_histogram2_bucket\xffle\xff-0.00038",
|
|
v: 4,
|
|
lset: labels.FromStrings(
|
|
"__name__", "test_histogram2_bucket",
|
|
"le", "-0.00038",
|
|
),
|
|
e: []exemplar.Exemplar{
|
|
{Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00038, HasTs: true, Ts: 1625851153146},
|
|
},
|
|
},
|
|
{
|
|
m: "test_histogram2_bucket\xffle\xff1.0",
|
|
v: 16,
|
|
lset: labels.FromStrings(
|
|
"__name__", "test_histogram2_bucket",
|
|
"le", "1.0",
|
|
),
|
|
e: []exemplar.Exemplar{
|
|
{Labels: labels.FromStrings("dummyID", "5617"), Value: -0.000295, HasTs: false},
|
|
},
|
|
},
|
|
{
|
|
m: "test_histogram2_bucket\xffle\xff+Inf",
|
|
v: 175,
|
|
lset: labels.FromStrings(
|
|
"__name__", "test_histogram2_bucket",
|
|
"le", "+Inf",
|
|
),
|
|
},
|
|
{
|
|
m: "rpc_durations_seconds",
|
|
help: "RPC latency distributions.",
|
|
},
|
|
{
|
|
m: "rpc_durations_seconds",
|
|
typ: MetricTypeSummary,
|
|
},
|
|
{
|
|
m: "rpc_durations_seconds_count\xffservice\xffexponential",
|
|
v: 262,
|
|
lset: labels.FromStrings(
|
|
"__name__", "rpc_durations_seconds_count",
|
|
"service", "exponential",
|
|
),
|
|
},
|
|
{
|
|
m: "rpc_durations_seconds_sum\xffservice\xffexponential",
|
|
v: 0.00025551262820703587,
|
|
lset: labels.FromStrings(
|
|
"__name__", "rpc_durations_seconds_sum",
|
|
"service", "exponential",
|
|
),
|
|
},
|
|
{
|
|
m: "rpc_durations_seconds\xffservice\xffexponential\xffquantile\xff0.5",
|
|
v: 6.442786329648548e-07,
|
|
lset: labels.FromStrings(
|
|
"__name__", "rpc_durations_seconds",
|
|
"quantile", "0.5",
|
|
"service", "exponential",
|
|
),
|
|
},
|
|
{
|
|
m: "rpc_durations_seconds\xffservice\xffexponential\xffquantile\xff0.9",
|
|
v: 1.9435742936658396e-06,
|
|
lset: labels.FromStrings(
|
|
"__name__", "rpc_durations_seconds",
|
|
"quantile", "0.9",
|
|
"service", "exponential",
|
|
),
|
|
},
|
|
{
|
|
m: "rpc_durations_seconds\xffservice\xffexponential\xffquantile\xff0.99",
|
|
v: 4.0471608667037015e-06,
|
|
lset: labels.FromStrings(
|
|
"__name__", "rpc_durations_seconds",
|
|
"quantile", "0.99",
|
|
"service", "exponential",
|
|
),
|
|
},
|
|
{
|
|
m: "without_quantiles",
|
|
help: "A summary without quantiles.",
|
|
},
|
|
{
|
|
m: "without_quantiles",
|
|
typ: MetricTypeSummary,
|
|
},
|
|
{
|
|
m: "without_quantiles_count",
|
|
v: 42,
|
|
lset: labels.FromStrings(
|
|
"__name__", "without_quantiles_count",
|
|
),
|
|
},
|
|
{
|
|
m: "without_quantiles_sum",
|
|
v: 1.234,
|
|
lset: labels.FromStrings(
|
|
"__name__", "without_quantiles_sum",
|
|
),
|
|
},
|
|
}
|
|
|
|
p := NewProtobufParser(inputBuf.Bytes())
|
|
i := 0
|
|
|
|
var res labels.Labels
|
|
|
|
for {
|
|
et, err := p.Next()
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
require.NoError(t, err)
|
|
|
|
switch et {
|
|
case EntrySeries:
|
|
m, ts, v := p.Series()
|
|
|
|
var e exemplar.Exemplar
|
|
p.Metric(&res)
|
|
found := p.Exemplar(&e)
|
|
require.Equal(t, exp[i].m, string(m))
|
|
if ts != nil {
|
|
require.Equal(t, exp[i].t, *ts)
|
|
} else {
|
|
require.Equal(t, exp[i].t, int64(0))
|
|
}
|
|
require.Equal(t, exp[i].v, v)
|
|
require.Equal(t, exp[i].lset, res)
|
|
if len(exp[i].e) == 0 {
|
|
require.Equal(t, false, found)
|
|
} else {
|
|
require.Equal(t, true, found)
|
|
require.Equal(t, exp[i].e[0], e)
|
|
}
|
|
res = res[:0]
|
|
|
|
case EntryHistogram:
|
|
m, ts, shs, fhs := p.Histogram()
|
|
p.Metric(&res)
|
|
require.Equal(t, exp[i].m, string(m))
|
|
if ts != nil {
|
|
require.Equal(t, exp[i].t, *ts)
|
|
} else {
|
|
require.Equal(t, exp[i].t, int64(0))
|
|
}
|
|
require.Equal(t, exp[i].lset, res)
|
|
res = res[:0]
|
|
require.Equal(t, exp[i].m, string(m))
|
|
if shs != nil {
|
|
require.Equal(t, exp[i].shs, shs)
|
|
} else {
|
|
require.Equal(t, exp[i].fhs, fhs)
|
|
}
|
|
j := 0
|
|
for e := (exemplar.Exemplar{}); p.Exemplar(&e); j++ {
|
|
require.Equal(t, exp[i].e[j], e)
|
|
e = exemplar.Exemplar{}
|
|
}
|
|
require.Equal(t, len(exp[i].e), j, "not enough exemplars found")
|
|
|
|
case EntryType:
|
|
m, typ := p.Type()
|
|
require.Equal(t, exp[i].m, string(m))
|
|
require.Equal(t, exp[i].typ, typ)
|
|
|
|
case EntryHelp:
|
|
m, h := p.Help()
|
|
require.Equal(t, exp[i].m, string(m))
|
|
require.Equal(t, exp[i].help, string(h))
|
|
|
|
case EntryUnit:
|
|
m, u := p.Unit()
|
|
require.Equal(t, exp[i].m, string(m))
|
|
require.Equal(t, exp[i].unit, string(u))
|
|
|
|
case EntryComment:
|
|
require.Equal(t, exp[i].comment, string(p.Comment()))
|
|
}
|
|
|
|
i++
|
|
}
|
|
require.Equal(t, len(exp), i)
|
|
}
|