From f01718262a83751a0ff286ca3d8504937a321326 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 25 Aug 2023 23:35:42 +0200 Subject: [PATCH] Unit tests for native histograms (#12668) promql: Extend testing framework to support native histograms This includes both the internal testing framework as well as the rules unit test feature of promtool. This also adds a bunch of basic tests. Many of the code level tests can now be converted to tests within the framework, and more tests can be added easily. --------- Signed-off-by: Harold Dost Signed-off-by: Gregor Zeitlinger Signed-off-by: Stephen Lang Co-authored-by: Harold Dost Co-authored-by: Stephen Lang Co-authored-by: Gregor Zeitlinger --- cmd/promtool/testdata/unittest.yml | 46 + cmd/promtool/unittest.go | 38 +- docs/configuration/unit_testing_rules.md | 43 +- model/histogram/float_histogram.go | 50 + model/histogram/float_histogram_test.go | 49 +- promql/engine_test.go | 12 +- promql/parser/generated_parser.y | 198 ++- promql/parser/generated_parser.y.go | 1272 ++++++++++------- promql/parser/lex.go | 156 +- promql/parser/lex_test.go | 68 +- promql/parser/parse.go | 169 ++- promql/parser/parse_test.go | 311 ++++ promql/test.go | 100 +- promql/testdata/native_histograms.test | 226 +++ .../prometheusremotewrite/helper.go | 9 +- tsdb/head_test.go | 24 +- 16 files changed, 2171 insertions(+), 600 deletions(-) create mode 100644 promql/testdata/native_histograms.test diff --git a/cmd/promtool/testdata/unittest.yml b/cmd/promtool/testdata/unittest.yml index e6745aadf..ff511729b 100644 --- a/cmd/promtool/testdata/unittest.yml +++ b/cmd/promtool/testdata/unittest.yml @@ -10,6 +10,21 @@ tests: - series: test_full values: "0 0" + - series: test_repeat + values: "1x2" + + - series: test_increase + values: "1+1x2" + + - series: test_histogram + values: "{{schema:1 sum:-0.3 count:32.1 z_bucket:7.1 z_bucket_w:0.05 buckets:[5.1 10 7] offset:-3 n_buckets:[4.1 5] n_offset:-5}}" + + - series: test_histogram_repeat + values: "{{sum:3 count:2 buckets:[2]}}x2" + + - series: test_histogram_increase + values: "{{sum:3 count:2 buckets:[2]}}+{{sum:1.3 count:1 buckets:[1]}}x2" + - series: test_stale values: "0 stale" @@ -31,6 +46,37 @@ tests: exp_samples: - value: 60 + # Repeat & increase + - expr: test_repeat + eval_time: 2m + exp_samples: + - value: 1 + labels: "test_repeat" + - expr: test_increase + eval_time: 2m + exp_samples: + - value: 3 + labels: "test_increase" + + # Histograms + - expr: test_histogram + eval_time: 1m + exp_samples: + - labels: "test_histogram" + histogram: "{{schema:1 sum:-0.3 count:32.1 z_bucket:7.1 z_bucket_w:0.05 buckets:[5.1 10 7] offset:-3 n_buckets:[4.1 5] n_offset:-5}}" + + - expr: test_histogram_repeat + eval_time: 2m + exp_samples: + - labels: "test_histogram_repeat" + histogram: "{{count:2 sum:3 buckets:[2]}}" + + - expr: test_histogram_increase + eval_time: 2m + exp_samples: + - labels: "test_histogram_increase" + histogram: "{{count:4 sum:5.6 buckets:[4]}}" + # Ensure a value is stale as soon as it is marked as such. - expr: test_stale eval_time: 59s diff --git a/cmd/promtool/unittest.go b/cmd/promtool/unittest.go index e934f37c8..575480b03 100644 --- a/cmd/promtool/unittest.go +++ b/cmd/promtool/unittest.go @@ -29,6 +29,7 @@ import ( "github.com/prometheus/common/model" "gopkg.in/yaml.v2" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/promql/parser" @@ -346,14 +347,29 @@ Outer: var gotSamples []parsedSample for _, s := range got { gotSamples = append(gotSamples, parsedSample{ - Labels: s.Metric.Copy(), - Value: s.F, + Labels: s.Metric.Copy(), + Value: s.F, + Histogram: promql.HistogramTestExpression(s.H), }) } var expSamples []parsedSample for _, s := range testCase.ExpSamples { lb, err := parser.ParseMetric(s.Labels) + var hist *histogram.FloatHistogram + if err == nil && s.Histogram != "" { + _, values, parseErr := parser.ParseSeriesDesc("{} " + s.Histogram) + switch { + case parseErr != nil: + err = parseErr + case len(values) != 1: + err = fmt.Errorf("expected 1 value, got %d", len(values)) + case values[0].Histogram == nil: + err = fmt.Errorf("expected histogram, got %v", values[0]) + default: + hist = values[0].Histogram + } + } if err != nil { err = fmt.Errorf("labels %q: %w", s.Labels, err) errs = append(errs, fmt.Errorf(" expr: %q, time: %s, err: %w", testCase.Expr, @@ -361,8 +377,9 @@ Outer: continue Outer } expSamples = append(expSamples, parsedSample{ - Labels: lb, - Value: s.Value, + Labels: lb, + Value: s.Value, + Histogram: promql.HistogramTestExpression(hist), }) } @@ -530,14 +547,16 @@ type promqlTestCase struct { } type sample struct { - Labels string `yaml:"labels"` - Value float64 `yaml:"value"` + Labels string `yaml:"labels"` + Value float64 `yaml:"value"` + Histogram string `yaml:"histogram"` // A non-empty string means Value is ignored. } // parsedSample is a sample with parsed Labels. type parsedSample struct { - Labels labels.Labels - Value float64 + Labels labels.Labels + Value float64 + Histogram string // TestExpression() of histogram.FloatHistogram } func parsedSamplesString(pss []parsedSample) string { @@ -552,5 +571,8 @@ func parsedSamplesString(pss []parsedSample) string { } func (ps *parsedSample) String() string { + if ps.Histogram != "" { + return ps.Labels.String() + " " + ps.Histogram + } return ps.Labels.String() + " " + strconv.FormatFloat(ps.Value, 'E', -1, 64) } diff --git a/docs/configuration/unit_testing_rules.md b/docs/configuration/unit_testing_rules.md index efd168b35..73d8ddd38 100644 --- a/docs/configuration/unit_testing_rules.md +++ b/docs/configuration/unit_testing_rules.md @@ -76,18 +76,49 @@ series: # This uses expanding notation. # Expanding notation: -# 'a+bxc' becomes 'a a+b a+(2*b) a+(3*b) … a+(c*b)' -# Read this as series starts at a, then c further samples incrementing by b. -# 'a-bxc' becomes 'a a-b a-(2*b) a-(3*b) … a-(c*b)' -# Read this as series starts at a, then c further samples decrementing by b (or incrementing by negative b). +# 'a+bxn' becomes 'a a+b a+(2*b) a+(3*b) … a+(n*b)' +# Read this as series starts at a, then n further samples incrementing by b. +# 'a-bxn' becomes 'a a-b a-(2*b) a-(3*b) … a-(n*b)' +# Read this as series starts at a, then n further samples decrementing by b (or incrementing by negative b). +# 'axn' becomes 'a a a … a' (n times) - it's a shorthand for 'a+0xn' # There are special values to indicate missing and stale samples: -# '_' represents a missing sample from scrape -# 'stale' indicates a stale sample +# '_' represents a missing sample from scrape +# 'stale' indicates a stale sample # Examples: # 1. '-2+4x3' becomes '-2 2 6 10' - series starts at -2, then 3 further samples incrementing by 4. # 2. ' 1-2x4' becomes '1 -1 -3 -5 -7' - series starts at 1, then 4 further samples decrementing by 2. # 3. ' 1x4' becomes '1 1 1 1 1' - shorthand for '1+0x4', series starts at 1, then 4 further samples incrementing by 0. # 4. ' 1 _x3 stale' becomes '1 _ _ _ stale' - the missing sample cannot increment, so 3 missing samples are produced by the '_x3' expression. +# +# Native histogram notation: +# Native histograms can be used instead of floating point numbers using the following notation: +# {{schema:1 sum:-0.3 count:3.1 z_bucket:7.1 z_bucket_w:0.05 buckets:[5.1 10 7] offset:-3 n_buckets:[4.1 5] n_offset:-5}} +# Native histograms support the same expanding notation as floating point numbers, i.e. 'axn', 'a+bxn' and 'a-bxn'. +# All properties are optional and default to 0. The order is not important. The following properties are supported: +# - schema (int): +# Currently valid schema numbers are -4 <= n <= 8. They are all for +# base-2 bucket schemas, where 1 is a bucket boundary in each case, and +# then each power of two is divided into 2^n logarithmic buckets. Or +# in other words, each bucket boundary is the previous boundary times +# 2^(2^-n). +# - sum (float): +# The sum of all observations, including the zero bucket. +# - count (non-negative float): +# The number of observations, including those that are NaN and including the zero bucket. +# - z_bucket (non-negative float): +# The sum of all observations in the zero bucket. +# - z_bucket_w (non-negative float): +# The width of the zero bucket. +# If z_bucket_w > 0, the zero bucket contains all observations -z_bucket_w <= x <= z_bucket_w. +# Otherwise, the zero bucket only contains observations that are exactly 0. +# - buckets (list of non-negative floats): +# Observation counts in positive buckets. Each represents an absolute count. +# - offset (int): +# The starting index of the first entry in the positive buckets. +# - n_buckets (list of non-negative floats): +# Observation counts in negative buckets. Each represents an absolute count. +# - n_offset (int): +# The starting index of the first entry in the negative buckets. values: ``` diff --git a/model/histogram/float_histogram.go b/model/histogram/float_histogram.go index f8766f7a8..41873278c 100644 --- a/model/histogram/float_histogram.go +++ b/model/histogram/float_histogram.go @@ -15,6 +15,7 @@ package histogram import ( "fmt" + "math" "strings" ) @@ -130,6 +131,55 @@ func (h *FloatHistogram) String() string { return sb.String() } +// TestExpression returns the string representation of this histogram as it is used in the internal PromQL testing +// framework as well as in promtool rules unit tests. +// The syntax is described in https://prometheus.io/docs/prometheus/latest/configuration/unit_testing_rules/#series +func (h *FloatHistogram) TestExpression() string { + var res []string + m := h.Copy() + + m.Compact(math.MaxInt) // Compact to reduce the number of positive and negative spans to 1. + + if m.Schema != 0 { + res = append(res, fmt.Sprintf("schema:%d", m.Schema)) + } + if m.Count != 0 { + res = append(res, fmt.Sprintf("count:%g", m.Count)) + } + if m.Sum != 0 { + res = append(res, fmt.Sprintf("sum:%g", m.Sum)) + } + if m.ZeroCount != 0 { + res = append(res, fmt.Sprintf("z_bucket:%g", m.ZeroCount)) + } + if m.ZeroThreshold != 0 { + res = append(res, fmt.Sprintf("z_bucket_w:%g", m.ZeroThreshold)) + } + + addBuckets := func(kind, bucketsKey, offsetKey string, buckets []float64, spans []Span) []string { + if len(spans) > 1 { + panic(fmt.Sprintf("histogram with multiple %s spans not supported", kind)) + } + for _, span := range spans { + if span.Offset != 0 { + res = append(res, fmt.Sprintf("%s:%d", offsetKey, span.Offset)) + } + } + + var bucketStr []string + for _, bucket := range buckets { + bucketStr = append(bucketStr, fmt.Sprintf("%g", bucket)) + } + if len(bucketStr) > 0 { + res = append(res, fmt.Sprintf("%s:[%s]", bucketsKey, strings.Join(bucketStr, " "))) + } + return res + } + res = addBuckets("positive", "buckets", "offset", m.PositiveBuckets, m.PositiveSpans) + res = addBuckets("negative", "n_buckets", "n_offset", m.NegativeBuckets, m.NegativeSpans) + return "{{" + strings.Join(res, " ") + "}}" +} + // ZeroBucket returns the zero bucket. func (h *FloatHistogram) ZeroBucket() Bucket[float64] { return Bucket[float64]{ diff --git a/model/histogram/float_histogram_test.go b/model/histogram/float_histogram_test.go index dd3e30427..0b712be43 100644 --- a/model/histogram/float_histogram_test.go +++ b/model/histogram/float_histogram_test.go @@ -938,6 +938,21 @@ func TestFloatHistogramCompact(t *testing.T) { NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 3, 4}, }, }, + { + "cut empty buckets in the middle", + &FloatHistogram{ + PositiveSpans: []Span{{5, 4}}, + PositiveBuckets: []float64{1, 3, 0, 2}, + }, + 0, + &FloatHistogram{ + PositiveSpans: []Span{ + {Offset: 5, Length: 2}, + {Offset: 1, Length: 1}, + }, + PositiveBuckets: []float64{1, 3, 2}, + }, + }, { "cut empty buckets at start or end of spans, even in the middle", &FloatHistogram{ @@ -955,7 +970,7 @@ func TestFloatHistogramCompact(t *testing.T) { }, }, { - "cut empty buckets at start or end but merge spans due to maxEmptyBuckets", + "cut empty buckets at start and end - also merge spans due to maxEmptyBuckets", &FloatHistogram{ PositiveSpans: []Span{{-4, 4}, {5, 3}}, PositiveBuckets: []float64{0, 0, 1, 3.3, 4.2, 0.1, 3.3}, @@ -998,18 +1013,42 @@ func TestFloatHistogramCompact(t *testing.T) { PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3}, }, }, + { + "cut empty buckets from the middle of a span, avoiding none due to maxEmptyBuckets", + &FloatHistogram{ + PositiveSpans: []Span{{-2, 4}}, + PositiveBuckets: []float64{1, 0, 0, 3.3}, + }, + 1, + &FloatHistogram{ + PositiveSpans: []Span{{-2, 1}, {2, 1}}, + PositiveBuckets: []float64{1, 3.3}, + }, + }, + { + "cut empty buckets and merge spans due to maxEmptyBuckets", + &FloatHistogram{ + PositiveSpans: []Span{{-2, 4}, {3, 1}}, + PositiveBuckets: []float64{1, 0, 0, 3.3, 4.2}, + }, + 1, + &FloatHistogram{ + PositiveSpans: []Span{{-2, 1}, {2, 1}, {3, 1}}, + PositiveBuckets: []float64{1, 3.3, 4.2}, + }, + }, { "cut empty buckets from the middle of a span, avoiding some due to maxEmptyBuckets", &FloatHistogram{ - PositiveSpans: []Span{{-4, 6}, {3, 3}}, - PositiveBuckets: []float64{0, 0, 1, 0, 0, 3.3, 4.2, 0.1, 3.3}, + PositiveSpans: []Span{{-4, 6}, {3, 3}, {10, 2}}, + PositiveBuckets: []float64{0, 0, 1, 0, 0, 3.3, 4.2, 0.1, 3.3, 2, 3}, NegativeSpans: []Span{{0, 2}, {3, 5}}, NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 0, 3, 4}, }, 1, &FloatHistogram{ - PositiveSpans: []Span{{-2, 1}, {2, 1}, {3, 3}}, - PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3}, + PositiveSpans: []Span{{-2, 1}, {2, 1}, {3, 3}, {10, 2}}, + PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3, 2, 3}, NegativeSpans: []Span{{0, 2}, {3, 5}}, NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 0, 3, 4}, }, diff --git a/promql/engine_test.go b/promql/engine_test.go index 1ded05858..82e44bcbc 100644 --- a/promql/engine_test.go +++ b/promql/engine_test.go @@ -4547,6 +4547,16 @@ func TestNativeHistogram_SubOperator(t *testing.T) { vector, err := res.Vector() require.NoError(t, err) + if len(vector) == len(exp) { + for i, e := range exp { + got := vector[i].H + if got != e.H { + // Error messages are better if we compare structs, not pointers. + require.Equal(t, *e.H, *got) + } + } + } + require.Equal(t, exp, vector) } @@ -4557,8 +4567,8 @@ func TestNativeHistogram_SubOperator(t *testing.T) { } queryAndCheck(queryString, []Sample{{T: ts, H: &c.expected, Metric: labels.EmptyLabels()}}) }) - idx0++ } + idx0++ } } diff --git a/promql/parser/generated_parser.y b/promql/parser/generated_parser.y index b28e9d544..f7951db2b 100644 --- a/promql/parser/generated_parser.y +++ b/promql/parser/generated_parser.y @@ -21,23 +21,28 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/value" + "github.com/prometheus/prometheus/model/histogram" ) %} %union { - node Node - item Item - matchers []*labels.Matcher - matcher *labels.Matcher - label labels.Label - labels labels.Labels - lblList []labels.Label - strings []string - series []SequenceValue - uint uint64 - float float64 - duration time.Duration + node Node + item Item + matchers []*labels.Matcher + matcher *labels.Matcher + label labels.Label + labels labels.Labels + lblList []labels.Label + strings []string + series []SequenceValue + histogram *histogram.FloatHistogram + descriptors map[string]interface{} + bucket_set []float64 + int int64 + uint uint64 + float float64 + duration time.Duration } @@ -54,6 +59,8 @@ IDENTIFIER LEFT_BRACE LEFT_BRACKET LEFT_PAREN +OPEN_HIST +CLOSE_HIST METRIC_IDENTIFIER NUMBER RIGHT_BRACE @@ -64,6 +71,20 @@ SPACE STRING TIMES +// Histogram Descriptors. +%token histogramDescStart +%token +SUM_DESC +COUNT_DESC +SCHEMA_DESC +OFFSET_DESC +NEGATIVE_OFFSET_DESC +BUCKETS_DESC +NEGATIVE_BUCKETS_DESC +ZERO_BUCKET_DESC +ZERO_BUCKET_WIDTH_DESC +%token histogramDescEnd + // Operators. %token operatorsStart %token @@ -145,6 +166,10 @@ START_METRIC_SELECTOR %type