diff --git a/docs/querying/functions.md b/docs/querying/functions.md index 1da70c6034..6b3a77e973 100644 --- a/docs/querying/functions.md +++ b/docs/querying/functions.md @@ -145,7 +145,7 @@ delta(cpu_temp_celsius{host="zeus"}[2h]) ``` `delta` acts on native histograms by calculating a new histogram where each -compononent (sum and count of observations, buckets) is the difference between +component (sum and count of observations, buckets) is the difference between the respective component in the first and last native histogram in `v`. However, each element in `v` that contains a mix of float and native histogram samples within the range, will be missing from the result vector. @@ -323,6 +323,19 @@ a histogram. You can use `histogram_quantile(1, v instant-vector)` to get the estimated maximum value stored in a histogram. +## `histogram_stddev()` and `histogram_stdvar()` + +_Both functions only act on native histograms, which are an experimental +feature. The behavior of these functions may change in future versions of +Prometheus, including their removal from PromQL._ + +`histogram_stddev(v instant-vector)` returns the estimated standard deviation +of observations in a native histogram, based on the geometric mean of the buckets +where the observations lie. Samples that are not native histograms are ignored and +do not show up in the returned vector. + +Similarly, `histogram_stdvar(v instant-vector)` returns the estimated standard +variance of observations in a native histogram. ## `holt_winters()` @@ -495,7 +508,7 @@ rate(http_requests_total{job="api-server"}[5m]) ``` `rate` acts on native histograms by calculating a new histogram where each -compononent (sum and count of observations, buckets) is the rate of increase +component (sum and count of observations, buckets) is the rate of increase between the respective component in the first and last native histogram in `v`. However, each element in `v` that contains a mix of float and native histogram samples within the range, will be missing from the result vector. diff --git a/model/histogram/float_histogram.go b/model/histogram/float_histogram.go index bfb3c3d19f..f8766f7a8b 100644 --- a/model/histogram/float_histogram.go +++ b/model/histogram/float_histogram.go @@ -159,7 +159,7 @@ func (h *FloatHistogram) Mul(factor float64) *FloatHistogram { return h } -// Div works like Scale but divides instead of multiplies. +// Div works like Mul but divides instead of multiplies. // When dividing by 0, everything will be set to Inf. func (h *FloatHistogram) Div(scalar float64) *FloatHistogram { h.ZeroCount /= scalar diff --git a/promql/engine_test.go b/promql/engine_test.go index 154a45514e..1ded05858b 100644 --- a/promql/engine_test.go +++ b/promql/engine_test.go @@ -3312,6 +3312,165 @@ func TestNativeHistogram_HistogramCountAndSum(t *testing.T) { } } +func TestNativeHistogram_HistogramStdDevVar(t *testing.T) { + // TODO(codesome): Integrate histograms into the PromQL testing framework + // and write more tests there. + testCases := []struct { + name string + h *histogram.Histogram + stdVar float64 + }{ + { + name: "1, 2, 3, 4 low-res", + h: &histogram.Histogram{ + Count: 4, + Sum: 10, + Schema: 2, + PositiveSpans: []histogram.Span{ + {Offset: 0, Length: 1}, + {Offset: 3, Length: 1}, + {Offset: 2, Length: 2}, + }, + PositiveBuckets: []int64{1, 0, 0, 0}, + }, + stdVar: 1.163807968526718, // actual variance: 1.25 + }, + { + name: "1, 2, 3, 4 hi-res", + h: &histogram.Histogram{ + Count: 4, + Sum: 10, + Schema: 8, + PositiveSpans: []histogram.Span{ + {Offset: 0, Length: 1}, + {Offset: 255, Length: 1}, + {Offset: 149, Length: 1}, + {Offset: 105, Length: 1}, + }, + PositiveBuckets: []int64{1, 0, 0, 0}, + }, + stdVar: 1.2471347737158793, // actual variance: 1.25 + }, + { + name: "-50, -8, 0, 3, 8, 9, 100", + h: &histogram.Histogram{ + Count: 7, + ZeroCount: 1, + Sum: 62, + Schema: 3, + PositiveSpans: []histogram.Span{ + {Offset: 13, Length: 1}, + {Offset: 10, Length: 1}, + {Offset: 1, Length: 1}, + {Offset: 27, Length: 1}, + }, + PositiveBuckets: []int64{1, 0, 0, 0}, + NegativeSpans: []histogram.Span{ + {Offset: 24, Length: 1}, + {Offset: 21, Length: 1}, + }, + NegativeBuckets: []int64{1, 0}, + }, + stdVar: 1544.8582535368798, // actual variance: 1738.4082 + }, + { + name: "-50, -8, 0, 3, 8, 9, 100, NaN", + h: &histogram.Histogram{ + Count: 8, + ZeroCount: 1, + Sum: math.NaN(), + Schema: 3, + PositiveSpans: []histogram.Span{ + {Offset: 13, Length: 1}, + {Offset: 10, Length: 1}, + {Offset: 1, Length: 1}, + {Offset: 27, Length: 1}, + }, + PositiveBuckets: []int64{1, 0, 0, 0}, + NegativeSpans: []histogram.Span{ + {Offset: 24, Length: 1}, + {Offset: 21, Length: 1}, + }, + NegativeBuckets: []int64{1, 0}, + }, + stdVar: math.NaN(), + }, + { + name: "-50, -8, 0, 3, 8, 9, 100, +Inf", + h: &histogram.Histogram{ + Count: 8, + ZeroCount: 1, + Sum: math.Inf(1), + Schema: 3, + PositiveSpans: []histogram.Span{ + {Offset: 13, Length: 1}, + {Offset: 10, Length: 1}, + {Offset: 1, Length: 1}, + {Offset: 27, Length: 1}, + }, + PositiveBuckets: []int64{1, 0, 0, 0}, + NegativeSpans: []histogram.Span{ + {Offset: 24, Length: 1}, + {Offset: 21, Length: 1}, + }, + NegativeBuckets: []int64{1, 0}, + }, + stdVar: math.NaN(), + }, + } + for _, tc := range testCases { + for _, floatHisto := range []bool{true, false} { + t.Run(fmt.Sprintf("%s floatHistogram=%t", tc.name, floatHisto), func(t *testing.T) { + engine := newTestEngine() + storage := teststorage.New(t) + t.Cleanup(func() { storage.Close() }) + + seriesName := "sparse_histogram_series" + lbls := labels.FromStrings("__name__", seriesName) + + ts := int64(10 * time.Minute / time.Millisecond) + app := storage.Appender(context.Background()) + var err error + if floatHisto { + _, err = app.AppendHistogram(0, lbls, ts, nil, tc.h.ToFloat()) + } else { + _, err = app.AppendHistogram(0, lbls, ts, tc.h, nil) + } + require.NoError(t, err) + require.NoError(t, app.Commit()) + + queryString := fmt.Sprintf("histogram_stdvar(%s)", seriesName) + qry, err := engine.NewInstantQuery(context.Background(), storage, nil, queryString, timestamp.Time(ts)) + require.NoError(t, err) + + res := qry.Exec(context.Background()) + require.NoError(t, res.Err) + + vector, err := res.Vector() + require.NoError(t, err) + + require.Len(t, vector, 1) + require.Nil(t, vector[0].H) + require.InEpsilon(t, tc.stdVar, vector[0].F, 1e-12) + + queryString = fmt.Sprintf("histogram_stddev(%s)", seriesName) + qry, err = engine.NewInstantQuery(context.Background(), storage, nil, queryString, timestamp.Time(ts)) + require.NoError(t, err) + + res = qry.Exec(context.Background()) + require.NoError(t, res.Err) + + vector, err = res.Vector() + require.NoError(t, err) + + require.Len(t, vector, 1) + require.Nil(t, vector[0].H) + require.InEpsilon(t, math.Sqrt(tc.stdVar), vector[0].F, 1e-12) + }) + } + } +} + func TestNativeHistogram_HistogramQuantile(t *testing.T) { // TODO(codesome): Integrate histograms into the PromQL testing framework // and write more tests there. diff --git a/promql/functions.go b/promql/functions.go index 96bffab96d..5c39d6bd8a 100644 --- a/promql/functions.go +++ b/promql/functions.go @@ -996,6 +996,72 @@ func funcHistogramSum(vals []parser.Value, args parser.Expressions, enh *EvalNod return enh.Out } +// === histogram_stddev(Vector parser.ValueTypeVector) Vector === +func funcHistogramStdDev(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { + inVec := vals[0].(Vector) + + for _, sample := range inVec { + // Skip non-histogram samples. + if sample.H == nil { + continue + } + mean := sample.H.Sum / sample.H.Count + var variance, cVariance float64 + it := sample.H.AllBucketIterator() + for it.Next() { + bucket := it.At() + var val float64 + if bucket.Lower <= 0 && 0 <= bucket.Upper { + val = 0 + } else { + val = math.Sqrt(bucket.Upper * bucket.Lower) + } + delta := val - mean + variance, cVariance = kahanSumInc(bucket.Count*delta*delta, variance, cVariance) + } + variance += cVariance + variance /= sample.H.Count + enh.Out = append(enh.Out, Sample{ + Metric: enh.DropMetricName(sample.Metric), + F: math.Sqrt(variance), + }) + } + return enh.Out +} + +// === histogram_stdvar(Vector parser.ValueTypeVector) Vector === +func funcHistogramStdVar(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { + inVec := vals[0].(Vector) + + for _, sample := range inVec { + // Skip non-histogram samples. + if sample.H == nil { + continue + } + mean := sample.H.Sum / sample.H.Count + var variance, cVariance float64 + it := sample.H.AllBucketIterator() + for it.Next() { + bucket := it.At() + var val float64 + if bucket.Lower <= 0 && 0 <= bucket.Upper { + val = 0 + } else { + val = math.Sqrt(bucket.Upper * bucket.Lower) + } + delta := val - mean + variance, cVariance = kahanSumInc(bucket.Count*delta*delta, variance, cVariance) + } + variance += cVariance + variance /= sample.H.Count + enh.Out = append(enh.Out, Sample{ + Metric: enh.DropMetricName(sample.Metric), + F: variance, + }) + } + return enh.Out +} + // === histogram_fraction(lower, upper parser.ValueTypeScalar, Vector parser.ValueTypeVector) Vector === func funcHistogramFraction(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { lower := vals[0].(Vector)[0].F @@ -1377,6 +1443,8 @@ var FunctionCalls = map[string]FunctionCall{ "histogram_fraction": funcHistogramFraction, "histogram_quantile": funcHistogramQuantile, "histogram_sum": funcHistogramSum, + "histogram_stddev": funcHistogramStdDev, + "histogram_stdvar": funcHistogramStdVar, "holt_winters": funcHoltWinters, "hour": funcHour, "idelta": funcIdelta, diff --git a/promql/parser/functions.go b/promql/parser/functions.go index 479c7f635d..45a30219e6 100644 --- a/promql/parser/functions.go +++ b/promql/parser/functions.go @@ -173,6 +173,16 @@ var Functions = map[string]*Function{ ArgTypes: []ValueType{ValueTypeVector}, ReturnType: ValueTypeVector, }, + "histogram_stddev": { + Name: "histogram_stddev", + ArgTypes: []ValueType{ValueTypeVector}, + ReturnType: ValueTypeVector, + }, + "histogram_stdvar": { + Name: "histogram_stdvar", + ArgTypes: []ValueType{ValueTypeVector}, + ReturnType: ValueTypeVector, + }, "histogram_fraction": { Name: "histogram_fraction", ArgTypes: []ValueType{ValueTypeScalar, ValueTypeScalar, ValueTypeVector}, diff --git a/web/ui/module/codemirror-promql/src/complete/promql.terms.ts b/web/ui/module/codemirror-promql/src/complete/promql.terms.ts index c97385bad3..5df81fe103 100644 --- a/web/ui/module/codemirror-promql/src/complete/promql.terms.ts +++ b/web/ui/module/codemirror-promql/src/complete/promql.terms.ts @@ -239,6 +239,18 @@ export const functionIdentifierTerms = [ info: 'Return the sum of observations from a native histogram (experimental feature)', type: 'function', }, + { + label: 'histogram_stddev', + detail: 'function', + info: 'Estimate the standard deviation of observations from a native histogram (experimental feature)', + type: 'function', + }, + { + label: 'histogram_stdvar', + detail: 'function', + info: 'Estimate the standard variance of observations from a native histogram (experimental feature)', + type: 'function', + }, { label: 'holt_winters', detail: 'function', @@ -430,7 +442,7 @@ export const functionIdentifierTerms = [ { label: 'stdvar_over_time', detail: 'function', - info: 'Calculate the standard variation within input series over time', + info: 'Calculate the standard variance within input series over time', type: 'function', }, { diff --git a/web/ui/module/codemirror-promql/src/parser/parser.test.ts b/web/ui/module/codemirror-promql/src/parser/parser.test.ts index b2140c0752..5ef9c1f907 100644 --- a/web/ui/module/codemirror-promql/src/parser/parser.test.ts +++ b/web/ui/module/codemirror-promql/src/parser/parser.test.ts @@ -752,6 +752,30 @@ describe('promql operations', () => { expectedValueType: ValueType.vector, expectedDiag: [], }, + { + expr: + 'histogram_stddev( # Root of the query, final result, returns the standard deviation of observations.\n' + + ' sum by(method, path) ( # Argument to histogram_stddev(), an aggregated histogram.\n' + + ' rate( # Argument to sum(), the per-second increase of a histogram over 5m.\n' + + ' demo_api_request_duration_seconds{job="demo"}[5m] # Argument to rate(), a vector of sparse histogram series over the last 5m.\n' + + ' )\n' + + ' )\n' + + ')', + expectedValueType: ValueType.vector, + expectedDiag: [], + }, + { + expr: + 'histogram_stdvar( # Root of the query, final result, returns the standard variance of observations.\n' + + ' sum by(method, path) ( # Argument to histogram_stdvar(), an aggregated histogram.\n' + + ' rate( # Argument to sum(), the per-second increase of a histogram over 5m.\n' + + ' demo_api_request_duration_seconds{job="demo"}[5m] # Argument to rate(), a vector of sparse histogram series over the last 5m.\n' + + ' )\n' + + ' )\n' + + ')', + expectedValueType: ValueType.vector, + expectedDiag: [], + }, { expr: '1 @ start()', expectedValueType: ValueType.scalar, diff --git a/web/ui/module/codemirror-promql/src/types/function.ts b/web/ui/module/codemirror-promql/src/types/function.ts index 746524b6f8..649cbad337 100644 --- a/web/ui/module/codemirror-promql/src/types/function.ts +++ b/web/ui/module/codemirror-promql/src/types/function.ts @@ -42,6 +42,8 @@ import { HistogramCount, HistogramFraction, HistogramQuantile, + HistogramStdDev, + HistogramStdVar, HistogramSum, HoltWinters, Hour, @@ -282,6 +284,18 @@ const promqlFunctions: { [key: number]: PromQLFunction } = { variadic: 0, returnType: ValueType.vector, }, + [HistogramStdDev]: { + name: 'histogram_stddev', + argTypes: [ValueType.vector], + variadic: 0, + returnType: ValueType.vector, + }, + [HistogramStdVar]: { + name: 'histogram_stdvar', + argTypes: [ValueType.vector], + variadic: 0, + returnType: ValueType.vector, + }, [HistogramSum]: { name: 'histogram_sum', argTypes: [ValueType.vector], diff --git a/web/ui/module/lezer-promql/src/promql.grammar b/web/ui/module/lezer-promql/src/promql.grammar index 3973f019ab..37f9a39cd8 100644 --- a/web/ui/module/lezer-promql/src/promql.grammar +++ b/web/ui/module/lezer-promql/src/promql.grammar @@ -135,6 +135,8 @@ FunctionIdentifier { HistogramCount | HistogramFraction | HistogramQuantile | + HistogramStdDev | + HistogramStdVar | HistogramSum | HoltWinters | Hour | @@ -362,6 +364,8 @@ NumberLiteral { HistogramCount { condFn<"histogram_count"> } HistogramFraction { condFn<"histogram_fraction"> } HistogramQuantile { condFn<"histogram_quantile"> } + HistogramStdDev { condFn<"histogram_stddev"> } + HistogramStdVar { condFn<"histogram_stdvar"> } HistogramSum { condFn<"histogram_sum"> } HoltWinters { condFn<"holt_winters"> } Hour { condFn<"hour"> }