From 4529a84e4a31b3c1754e2189ff98918a738dd1ca Mon Sep 17 00:00:00 2001 From: bwplotka Date: Thu, 3 Oct 2024 10:34:36 +0100 Subject: [PATCH] Kill sinks, tested, nothing is inlined. Signed-off-by: bwplotka --- model/textparse/benchmark_test.go | 220 ++++++++++++++++-------------- 1 file changed, 115 insertions(+), 105 deletions(-) diff --git a/model/textparse/benchmark_test.go b/model/textparse/benchmark_test.go index 4787edc3ce..ce29fb8c4d 100644 --- a/model/textparse/benchmark_test.go +++ b/model/textparse/benchmark_test.go @@ -1,3 +1,16 @@ +// Copyright 2024 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package textparse import ( @@ -9,37 +22,32 @@ import ( "path/filepath" "testing" - "github.com/prometheus/common/expfmt" - "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/exemplar" "github.com/prometheus/prometheus/model/labels" - "github.com/stretchr/testify/require" -) -const ( - promtestdataSampleCount = 410 + "github.com/prometheus/common/expfmt" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" ) type newParser func([]byte, *labels.SymbolTable) Parser -var ( - newTestParserFns = map[string]newParser{ - "promtext": NewPromParser, - "promproto": func(b []byte, st *labels.SymbolTable) Parser { - return NewProtobufParser(b, true, st) - }, - "omtext": func(b []byte, st *labels.SymbolTable) Parser { - return NewOpenMetricsParser(b, st, WithOMParserCTSeriesSkipped()) - }, - } -) +var newTestParserFns = map[string]newParser{ + "promtext": NewPromParser, + "promproto": func(b []byte, st *labels.SymbolTable) Parser { + return NewProtobufParser(b, true, st) + }, + "omtext": func(b []byte, st *labels.SymbolTable) Parser { + return NewOpenMetricsParser(b, st, WithOMParserCTSeriesSkipped()) + }, +} // BenchmarkParse benchmarks parsing, mimicking how scrape/scrape.go#append use it. // Typically used as follows: /* - export bench=parse && go test ./model/textparse/... \ + export bench=v1 && go test ./model/textparse/... \ -run '^$' -bench '^BenchmarkParse' \ - -benchtime 5s -count 6 -cpu 2 -benchmem -timeout 999m \ + -benchtime 2s -count 6 -cpu 2 -benchmem -timeout 999m \ | tee ${bench}.txt */ // For profiles, add -memprofile=${bench}.mem.pprof -cpuprofile=${bench}.cpu.pprof @@ -50,22 +58,30 @@ var ( // good to know if you are working on a certain optimization, but it does not // make sense to persist such cases for everybody (e.g. for CI one day). // For local iteration, feel free to adjust cases/comment out code etc. +// +// NOTE(bwplotka): Do not try to conclude "what parser (OM, proto, prom) is the fastest" +// as the testdata has different amount and type of metrics and features (e.g. exemplars). func BenchmarkParse(b *testing.B) { for _, bcase := range []struct { dataFile string // localized to ./testdata dataProto []byte - - parser string + parser string compareToExpfmtFormat expfmt.FormatType }{ - // TODO(bwplotka): Consider having the same (semantically) data across all file types. - // Currently that's not the case, this does not answer "what parser (OM, proto, prom) is the fastest" - // However, we can compare efficiency across commits for each parser. {dataFile: "promtestdata.txt", parser: "promtext", compareToExpfmtFormat: expfmt.TypeTextPlain}, {dataFile: "promtestdata.nometa.txt", parser: "promtext", compareToExpfmtFormat: expfmt.TypeTextPlain}, - {dataProto: createTestProtoBuf(b).Bytes(), parser: "promproto", compareToExpfmtFormat: expfmt.TypeProtoDelim}, - {dataFile: "omtestdata.txt", parser: "omtext", compareToExpfmtFormat: expfmt.TypeOpenMetrics}, + + // We don't pass compareToExpfmtFormat: expfmt.TypeProtoDelim as expfmt does not support GAUGE_HISTOGRAM: + // "expfmt.extractSamples: unknown metric family type GAUGE_HISTOGRAM" + // TODO(bwplotka): Fix expfmt? + {dataProto: createTestProtoBuf(b).Bytes(), parser: "promproto"}, + + // We don't pass compareToExpfmtFormat: expfmt.TypeOpenMetrics as expfmt does not support OM exemplars: + // text format parsing error in line 19: expected integer as timestamp, got "#" + // TODO(bwplotka): Fix expfmt? + {dataFile: "omtestdata.txt", parser: "omtext"}, + {dataFile: "promtestdata.txt", parser: "omtext"}, // Compare how omtext parser deals with Prometheus text format vs promtext. } { var buf []byte dataCase := bcase.dataFile @@ -81,93 +97,87 @@ func BenchmarkParse(b *testing.B) { buf, err = io.ReadAll(f) require.NoError(b, err) } - b.Run(fmt.Sprintf("data=%v", dataCase), func(b *testing.B) { - b.Run(fmt.Sprintf("parser=%v", bcase.parser), func(b *testing.B) { - newParserFn := newTestParserFns[bcase.parser] - var ( - res labels.Labels - e exemplar.Exemplar - series []byte - ) + b.Run(fmt.Sprintf("data=%v/parser=%v", dataCase, bcase.parser), func(b *testing.B) { + newParserFn := newTestParserFns[bcase.parser] + var ( + res labels.Labels + e exemplar.Exemplar + ) - b.SetBytes(int64(len(buf))) - b.ReportAllocs() - b.ResetTimer() + b.SetBytes(int64(len(buf))) + b.ReportAllocs() + b.ResetTimer() - st := labels.NewSymbolTable() - for i := 0; i < b.N; i++ { - p := newParserFn(buf, st) + st := labels.NewSymbolTable() + for i := 0; i < b.N; i++ { + p := newParserFn(buf, st) - Inner: - for { - t, err := p.Next() - switch t { - case EntryInvalid: - if errors.Is(err, io.EOF) { - break Inner - } - b.Fatal(err) - case EntryType: - _, _ = p.Type() - continue - case EntryHelp: - _, _ = p.Help() - continue - case EntryUnit: - _, _ = p.Unit() - continue - case EntryComment: - continue - case EntryHistogram: - series, _, _, _ = p.Histogram() - case EntrySeries: - series, _, _ = p.Series() - default: - b.Fatal("not implemented entry", t) + Inner: + for { + t, err := p.Next() + switch t { + case EntryInvalid: + if errors.Is(err, io.EOF) { + break Inner } - - _ = p.Metric(&res) - _ = p.CreatedTimestamp() - for hasExemplar := p.Exemplar(&e); hasExemplar; hasExemplar = p.Exemplar(&e) { - } - } - } - _ = series - }) - - // Compare with expfmt, opt-in, no need to benchmark external code. - b.Run("parser=expfmt", func(b *testing.B) { - //b.Skip("Not needed for commit-commit comparisons, skipped by default") - if bcase.compareToExpfmtFormat == expfmt.TypeUnknown { - b.Skip("compareToExpfmtFormat not set") - } - - b.SetBytes(int64(len(buf))) - b.ReportAllocs() - b.ResetTimer() - - var decSamples model.Vector - for i := 0; i < b.N; i++ { - decSamples = make(model.Vector, 0, 50) - sdec := expfmt.SampleDecoder{ - Dec: expfmt.NewDecoder(bytes.NewReader(buf), expfmt.NewFormat(bcase.compareToExpfmtFormat)), - Opts: &expfmt.DecodeOptions{ - Timestamp: model.TimeFromUnixNano(0), - }, + b.Fatal(err) + case EntryType: + _, _ = p.Type() + continue + case EntryHelp: + _, _ = p.Help() + continue + case EntryUnit: + _, _ = p.Unit() + continue + case EntryComment: + continue + case EntryHistogram: + _, _, _, _ = p.Histogram() + case EntrySeries: + _, _, _ = p.Series() + default: + b.Fatal("not implemented entry", t) } - for { - if err := sdec.Decode(&decSamples); err != nil { - if errors.Is(err, io.EOF) { - break - } - b.Fatal(err) - } - decSamples = decSamples[:0] + _ = p.Metric(&res) + _ = p.CreatedTimestamp() + for hasExemplar := p.Exemplar(&e); hasExemplar; hasExemplar = p.Exemplar(&e) { } } - _ = decSamples - }) + } + }) + + // Compare with expfmt, opt-in, no need to benchmark external code. + b.Run(fmt.Sprintf("data=%v/parser=xpfmt", dataCase), func(b *testing.B) { + // b.Skip("Not needed for commit-commit comparisons, skipped by default") + if bcase.compareToExpfmtFormat == expfmt.TypeUnknown { + b.Skip("compareToExpfmtFormat not set") + } + + b.SetBytes(int64(len(buf))) + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + decSamples := make(model.Vector, 0, 50) + sdec := expfmt.SampleDecoder{ + Dec: expfmt.NewDecoder(bytes.NewReader(buf), expfmt.NewFormat(bcase.compareToExpfmtFormat)), + Opts: &expfmt.DecodeOptions{ + Timestamp: model.TimeFromUnixNano(0), + }, + } + + for { + if err := sdec.Decode(&decSamples); err != nil { + if errors.Is(err, io.EOF) { + break + } + b.Fatal(err) + } + decSamples = decSamples[:0] + } + } }) } }