// Copyright 2021 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package textparse import ( "bytes" "encoding/binary" "io" "testing" "github.com/gogo/protobuf/proto" "github.com/stretchr/testify/require" "github.com/prometheus/prometheus/model/exemplar" "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" dto "github.com/prometheus/prometheus/prompb/io/prometheus/client" ) func TestProtobufParse(t *testing.T) { textMetricFamilies := []string{ `name: "go_build_info" help: "Build information about the main Go module." type: GAUGE metric: < label: < name: "checksum" value: "" > label: < name: "path" value: "github.com/prometheus/client_golang" > label: < name: "version" value: "(devel)" > gauge: < value: 1 > > `, `name: "go_memstats_alloc_bytes_total" help: "Total number of bytes allocated, even if freed." type: COUNTER metric: < counter: < value: 1.546544e+06 exemplar: < label: < name: "dummyID" value: "42" > value: 12 timestamp: < seconds: 1625851151 nanos: 233181499 > > > > `, `name: "something_untyped" help: "Just to test the untyped type." type: UNTYPED metric: < untyped: < value: 42 > timestamp_ms: 1234567 > `, `name: "test_histogram" help: "Test histogram with many buckets removed to keep it manageable in size." type: HISTOGRAM metric: < histogram: < sample_count: 175 sample_sum: 0.0008280461746287094 bucket: < cumulative_count: 2 upper_bound: -0.0004899999999999998 > bucket: < cumulative_count: 4 upper_bound: -0.0003899999999999998 exemplar: < label: < name: "dummyID" value: "59727" > value: -0.00039 timestamp: < seconds: 1625851155 nanos: 146848499 > > > bucket: < cumulative_count: 16 upper_bound: -0.0002899999999999998 exemplar: < label: < name: "dummyID" value: "5617" > value: -0.00029 > > schema: 3 zero_threshold: 2.938735877055719e-39 zero_count: 2 negative_span: < offset: -162 length: 1 > negative_span: < offset: 23 length: 4 > negative_delta: 1 negative_delta: 3 negative_delta: -2 negative_delta: -1 negative_delta: 1 positive_span: < offset: -161 length: 1 > positive_span: < offset: 8 length: 3 > positive_delta: 1 positive_delta: 2 positive_delta: -1 positive_delta: -1 > timestamp_ms: 1234568 > `, `name: "test_float_histogram" help: "Test float histogram with many buckets removed to keep it manageable in size." type: HISTOGRAM metric: < histogram: < sample_count: 175 sample_count_float: 175.0 sample_sum: 0.0008280461746287094 bucket: < cumulative_count_float: 2.0 upper_bound: -0.0004899999999999998 > bucket: < cumulative_count_float: 4.0 upper_bound: -0.0003899999999999998 exemplar: < label: < name: "dummyID" value: "59727" > value: -0.00039 timestamp: < seconds: 1625851155 nanos: 146848499 > > > bucket: < cumulative_count_float: 16 upper_bound: -0.0002899999999999998 exemplar: < label: < name: "dummyID" value: "5617" > value: -0.00029 > > schema: 3 zero_threshold: 2.938735877055719e-39 zero_count_float: 2.0 negative_span: < offset: -162 length: 1 > negative_span: < offset: 23 length: 4 > negative_count: 1.0 negative_count: 3.0 negative_count: -2.0 negative_count: -1.0 negative_count: 1.0 positive_span: < offset: -161 length: 1 > positive_span: < offset: 8 length: 3 > positive_count: 1.0 positive_count: 2.0 positive_count: -1.0 positive_count: -1.0 > timestamp_ms: 1234568 > `, `name: "test_histogram2" help: "Similar histogram as before but now without sparse buckets." type: HISTOGRAM metric: < histogram: < sample_count: 175 sample_sum: 0.000828 bucket: < cumulative_count: 2 upper_bound: -0.00048 > bucket: < cumulative_count: 4 upper_bound: -0.00038 exemplar: < label: < name: "dummyID" value: "59727" > value: -0.00038 timestamp: < seconds: 1625851153 nanos: 146848499 > > > bucket: < cumulative_count: 16 upper_bound: 1 exemplar: < label: < name: "dummyID" value: "5617" > value: -0.000295 > > schema: 0 zero_threshold: 0 > > `, `name: "rpc_durations_seconds" help: "RPC latency distributions." type: SUMMARY metric: < label: < name: "service" value: "exponential" > summary: < sample_count: 262 sample_sum: 0.00025551262820703587 quantile: < quantile: 0.5 value: 6.442786329648548e-07 > quantile: < quantile: 0.9 value: 1.9435742936658396e-06 > quantile: < quantile: 0.99 value: 4.0471608667037015e-06 > > > `, `name: "without_quantiles" help: "A summary without quantiles." type: SUMMARY metric: < summary: < sample_count: 42 sample_sum: 1.234 > > `, } varintBuf := make([]byte, binary.MaxVarintLen32) inputBuf := &bytes.Buffer{} for _, tmf := range textMetricFamilies { pb := &dto.MetricFamily{} // From text to proto message. require.NoError(t, proto.UnmarshalText(tmf, pb)) // From proto message to binary protobuf. protoBuf, err := proto.Marshal(pb) require.NoError(t, err) // Write first length, then binary protobuf. varintLength := binary.PutUvarint(varintBuf, uint64(len(protoBuf))) inputBuf.Write(varintBuf[:varintLength]) inputBuf.Write(protoBuf) } exp := []struct { lset labels.Labels m string t int64 v float64 typ MetricType help string unit string comment string shs *histogram.Histogram fhs *histogram.FloatHistogram e []exemplar.Exemplar }{ { m: "go_build_info", help: "Build information about the main Go module.", }, { m: "go_build_info", typ: MetricTypeGauge, }, { m: "go_build_info\xFFchecksum\xFF\xFFpath\xFFgithub.com/prometheus/client_golang\xFFversion\xFF(devel)", v: 1, lset: labels.FromStrings( "__name__", "go_build_info", "checksum", "", "path", "github.com/prometheus/client_golang", "version", "(devel)", ), }, { m: "go_memstats_alloc_bytes_total", help: "Total number of bytes allocated, even if freed.", }, { m: "go_memstats_alloc_bytes_total", typ: MetricTypeCounter, }, { m: "go_memstats_alloc_bytes_total", v: 1.546544e+06, lset: labels.FromStrings( "__name__", "go_memstats_alloc_bytes_total", ), e: []exemplar.Exemplar{ {Labels: labels.FromStrings("dummyID", "42"), Value: 12, HasTs: true, Ts: 1625851151233}, }, }, { m: "something_untyped", help: "Just to test the untyped type.", }, { m: "something_untyped", typ: MetricTypeUnknown, }, { m: "something_untyped", t: 1234567, v: 42, lset: labels.FromStrings( "__name__", "something_untyped", ), }, { m: "test_histogram", help: "Test histogram with many buckets removed to keep it manageable in size.", }, { m: "test_histogram", typ: MetricTypeHistogram, }, { m: "test_histogram", t: 1234568, shs: &histogram.Histogram{ Count: 175, ZeroCount: 2, Sum: 0.0008280461746287094, ZeroThreshold: 2.938735877055719e-39, Schema: 3, PositiveSpans: []histogram.Span{ {Offset: -161, Length: 1}, {Offset: 8, Length: 3}, }, NegativeSpans: []histogram.Span{ {Offset: -162, Length: 1}, {Offset: 23, Length: 4}, }, PositiveBuckets: []int64{1, 2, -1, -1}, NegativeBuckets: []int64{1, 3, -2, -1, 1}, }, lset: labels.FromStrings( "__name__", "test_histogram", ), e: []exemplar.Exemplar{ {Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00039, HasTs: true, Ts: 1625851155146}, {Labels: labels.FromStrings("dummyID", "5617"), Value: -0.00029, HasTs: false}, }, }, { m: "test_float_histogram", help: "Test float histogram with many buckets removed to keep it manageable in size.", }, { m: "test_float_histogram", typ: MetricTypeHistogram, }, { m: "test_float_histogram", t: 1234568, fhs: &histogram.FloatHistogram{ Count: 175.0, ZeroCount: 2.0, Sum: 0.0008280461746287094, ZeroThreshold: 2.938735877055719e-39, Schema: 3, PositiveSpans: []histogram.Span{ {Offset: -161, Length: 1}, {Offset: 8, Length: 3}, }, NegativeSpans: []histogram.Span{ {Offset: -162, Length: 1}, {Offset: 23, Length: 4}, }, PositiveBuckets: []float64{1.0, 2.0, -1.0, -1.0}, NegativeBuckets: []float64{1.0, 3.0, -2.0, -1.0, 1.0}, }, lset: labels.FromStrings( "__name__", "test_float_histogram", ), e: []exemplar.Exemplar{ {Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00039, HasTs: true, Ts: 1625851155146}, {Labels: labels.FromStrings("dummyID", "5617"), Value: -0.00029, HasTs: false}, }, }, { m: "test_histogram2", help: "Similar histogram as before but now without sparse buckets.", }, { m: "test_histogram2", typ: MetricTypeHistogram, }, { m: "test_histogram2_count", v: 175, lset: labels.FromStrings( "__name__", "test_histogram2_count", ), }, { m: "test_histogram2_sum", v: 0.000828, lset: labels.FromStrings( "__name__", "test_histogram2_sum", ), }, { m: "test_histogram2_bucket\xffle\xff-0.00048", v: 2, lset: labels.FromStrings( "__name__", "test_histogram2_bucket", "le", "-0.00048", ), }, { m: "test_histogram2_bucket\xffle\xff-0.00038", v: 4, lset: labels.FromStrings( "__name__", "test_histogram2_bucket", "le", "-0.00038", ), e: []exemplar.Exemplar{ {Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00038, HasTs: true, Ts: 1625851153146}, }, }, { m: "test_histogram2_bucket\xffle\xff1.0", v: 16, lset: labels.FromStrings( "__name__", "test_histogram2_bucket", "le", "1.0", ), e: []exemplar.Exemplar{ {Labels: labels.FromStrings("dummyID", "5617"), Value: -0.000295, HasTs: false}, }, }, { m: "test_histogram2_bucket\xffle\xff+Inf", v: 175, lset: labels.FromStrings( "__name__", "test_histogram2_bucket", "le", "+Inf", ), }, { m: "rpc_durations_seconds", help: "RPC latency distributions.", }, { m: "rpc_durations_seconds", typ: MetricTypeSummary, }, { m: "rpc_durations_seconds_count\xffservice\xffexponential", v: 262, lset: labels.FromStrings( "__name__", "rpc_durations_seconds_count", "service", "exponential", ), }, { m: "rpc_durations_seconds_sum\xffservice\xffexponential", v: 0.00025551262820703587, lset: labels.FromStrings( "__name__", "rpc_durations_seconds_sum", "service", "exponential", ), }, { m: "rpc_durations_seconds\xffservice\xffexponential\xffquantile\xff0.5", v: 6.442786329648548e-07, lset: labels.FromStrings( "__name__", "rpc_durations_seconds", "quantile", "0.5", "service", "exponential", ), }, { m: "rpc_durations_seconds\xffservice\xffexponential\xffquantile\xff0.9", v: 1.9435742936658396e-06, lset: labels.FromStrings( "__name__", "rpc_durations_seconds", "quantile", "0.9", "service", "exponential", ), }, { m: "rpc_durations_seconds\xffservice\xffexponential\xffquantile\xff0.99", v: 4.0471608667037015e-06, lset: labels.FromStrings( "__name__", "rpc_durations_seconds", "quantile", "0.99", "service", "exponential", ), }, { m: "without_quantiles", help: "A summary without quantiles.", }, { m: "without_quantiles", typ: MetricTypeSummary, }, { m: "without_quantiles_count", v: 42, lset: labels.FromStrings( "__name__", "without_quantiles_count", ), }, { m: "without_quantiles_sum", v: 1.234, lset: labels.FromStrings( "__name__", "without_quantiles_sum", ), }, } p := NewProtobufParser(inputBuf.Bytes()) i := 0 var res labels.Labels for { et, err := p.Next() if err == io.EOF { break } require.NoError(t, err) switch et { case EntrySeries: m, ts, v := p.Series() var e exemplar.Exemplar p.Metric(&res) found := p.Exemplar(&e) require.Equal(t, exp[i].m, string(m)) if ts != nil { require.Equal(t, exp[i].t, *ts) } else { require.Equal(t, exp[i].t, int64(0)) } require.Equal(t, exp[i].v, v) require.Equal(t, exp[i].lset, res) if len(exp[i].e) == 0 { require.Equal(t, false, found) } else { require.Equal(t, true, found) require.Equal(t, exp[i].e[0], e) } case EntryHistogram: m, ts, shs, fhs := p.Histogram() p.Metric(&res) require.Equal(t, exp[i].m, string(m)) if ts != nil { require.Equal(t, exp[i].t, *ts) } else { require.Equal(t, exp[i].t, int64(0)) } require.Equal(t, exp[i].lset, res) require.Equal(t, exp[i].m, string(m)) if shs != nil { require.Equal(t, exp[i].shs, shs) } else { require.Equal(t, exp[i].fhs, fhs) } j := 0 for e := (exemplar.Exemplar{}); p.Exemplar(&e); j++ { require.Equal(t, exp[i].e[j], e) e = exemplar.Exemplar{} } require.Equal(t, len(exp[i].e), j, "not enough exemplars found") case EntryType: m, typ := p.Type() require.Equal(t, exp[i].m, string(m)) require.Equal(t, exp[i].typ, typ) case EntryHelp: m, h := p.Help() require.Equal(t, exp[i].m, string(m)) require.Equal(t, exp[i].help, string(h)) case EntryUnit: m, u := p.Unit() require.Equal(t, exp[i].m, string(m)) require.Equal(t, exp[i].unit, string(u)) case EntryComment: require.Equal(t, exp[i].comment, string(p.Comment())) } i++ } require.Equal(t, len(exp), i) }