mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2024-12-25 21:54:10 -08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity
prometheus/model/histogram/histogram_test.go
Filip Petkovski 583f3e587c
Optimize histogram iterators (#13340) 
Optimize histogram iterators

Histogram iterators allocate new objects in the AtHistogram and
AtFloatHistogram methods, which makes calculating rates over long
ranges expensive.

In #13215 we allowed an existing object to be reused
when converting an integer histogram to a float histogram. This commit follows
the same idea and allows injecting an existing object in the AtHistogram and
AtFloatHistogram methods. When the injected value is nil, iterators allocate
new histograms, otherwise they populate and return the injected object.

The commit also adds a CopyTo method to Histogram and FloatHistogram which
is used in the BufferedIterator to overwrite items in the ring instead of making
new copies.

Note that a specialized HPoint pool is needed for all of this to work 
(`matrixSelectorHPool`).

---------

Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
Co-authored-by: George Krajcsovits <krajorama@users.noreply.github.com>
2024-01-23 17:02:14 +01:00

1185 lines
38 KiB
Go
Raw Blame History

// Copyright 2021 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package histogram
import (
"fmt"
"math"
"testing"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/model/value"
)
func TestHistogramString(t *testing.T) {
cases := []struct {
histogram Histogram
expectedString string
}{
{
histogram: Histogram{
Schema: 0,
},
expectedString: "{count:0, sum:0}",
},
{
histogram: Histogram{
Schema: 0,
Count: 9,
Sum: -3.1415,
ZeroCount: 12,
ZeroThreshold: 0.001,
NegativeSpans: []Span{
{Offset: 0, Length: 5},
{Offset: 1, Length: 1},
},
NegativeBuckets: []int64{1, 2, -2, 1, -1, 0},
},
expectedString: "{count:9, sum:-3.1415, [-64,-32):1, [-16,-8):1, [-8,-4):2, [-4,-2):1, [-2,-1):3, [-1,-0.5):1, [-0.001,0.001]:12}",
},
{
histogram: Histogram{
Schema: 0,
Count: 19,
Sum: 2.7,
PositiveSpans: []Span{
{Offset: 0, Length: 4},
{Offset: 0, Length: 0},
{Offset: 0, Length: 3},
},
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 0},
NegativeSpans: []Span{
{Offset: 0, Length: 5},
{Offset: 1, Length: 0},
{Offset: 0, Length: 1},
},
NegativeBuckets: []int64{1, 2, -2, 1, -1, 0},
},
expectedString: "{count:19, sum:2.7, [-64,-32):1, [-16,-8):1, [-8,-4):2, [-4,-2):1, [-2,-1):3, [-1,-0.5):1, (0.5,1]:1, (1,2]:3, (2,4]:1, (4,8]:2, (8,16]:1, (16,32]:1, (32,64]:1}",
},
}
for i, c := range cases {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
actualString := c.histogram.String()
require.Equal(t, c.expectedString, actualString)
})
}
}
func TestCumulativeBucketIterator(t *testing.T) {
cases := []struct {
histogram Histogram
expectedBuckets []Bucket[uint64]
}{
{
histogram: Histogram{
Schema: 0,
PositiveSpans: []Span{
{Offset: 0, Length: 2},
{Offset: 1, Length: 2},
},
PositiveBuckets: []int64{1, 1, -1, 0},
},
expectedBuckets: []Bucket[uint64]{
{Lower: math.Inf(-1), Upper: 1, Count: 1, LowerInclusive: true, UpperInclusive: true, Index: 0},
{Lower: math.Inf(-1), Upper: 2, Count: 3, LowerInclusive: true, UpperInclusive: true, Index: 1},
{Lower: math.Inf(-1), Upper: 4, Count: 3, LowerInclusive: true, UpperInclusive: true, Index: 2},
{Lower: math.Inf(-1), Upper: 8, Count: 4, LowerInclusive: true, UpperInclusive: true, Index: 3},
{Lower: math.Inf(-1), Upper: 16, Count: 5, LowerInclusive: true, UpperInclusive: true, Index: 4},
},
},
{
histogram: Histogram{
Schema: 0,
PositiveSpans: []Span{
{Offset: 0, Length: 5},
{Offset: 1, Length: 1},
},
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0},
},
expectedBuckets: []Bucket[uint64]{
{Lower: math.Inf(-1), Upper: 1, Count: 1, LowerInclusive: true, UpperInclusive: true, Index: 0},
{Lower: math.Inf(-1), Upper: 2, Count: 4, LowerInclusive: true, UpperInclusive: true, Index: 1},
{Lower: math.Inf(-1), Upper: 4, Count: 5, LowerInclusive: true, UpperInclusive: true, Index: 2},
{Lower: math.Inf(-1), Upper: 8, Count: 7, LowerInclusive: true, UpperInclusive: true, Index: 3},
{Lower: math.Inf(-1), Upper: 16, Count: 8, LowerInclusive: true, UpperInclusive: true, Index: 4},
{Lower: math.Inf(-1), Upper: 32, Count: 8, LowerInclusive: true, UpperInclusive: true, Index: 5},
{Lower: math.Inf(-1), Upper: 64, Count: 9, LowerInclusive: true, UpperInclusive: true, Index: 6},
},
},
{
histogram: Histogram{
Schema: 0,
PositiveSpans: []Span{
{Offset: 0, Length: 7},
},
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 0},
},
expectedBuckets: []Bucket[uint64]{
{Lower: math.Inf(-1), Upper: 1, Count: 1, LowerInclusive: true, UpperInclusive: true, Index: 0},
{Lower: math.Inf(-1), Upper: 2, Count: 4, LowerInclusive: true, UpperInclusive: true, Index: 1},
{Lower: math.Inf(-1), Upper: 4, Count: 5, LowerInclusive: true, UpperInclusive: true, Index: 2},
{Lower: math.Inf(-1), Upper: 8, Count: 7, LowerInclusive: true, UpperInclusive: true, Index: 3},
{Lower: math.Inf(-1), Upper: 16, Count: 8, LowerInclusive: true, UpperInclusive: true, Index: 4},
{Lower: math.Inf(-1), Upper: 32, Count: 9, LowerInclusive: true, UpperInclusive: true, Index: 5},
{Lower: math.Inf(-1), Upper: 64, Count: 10, LowerInclusive: true, UpperInclusive: true, Index: 6},
},
},
{
histogram: Histogram{
Schema: 3,
PositiveSpans: []Span{
{Offset: -5, Length: 2}, // -5 -4
{Offset: 2, Length: 3}, // -1 0 1
{Offset: 2, Length: 2}, // 4 5
},
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 3},
},
expectedBuckets: []Bucket[uint64]{
{Lower: math.Inf(-1), Upper: 0.6484197773255048, Count: 1, LowerInclusive: true, UpperInclusive: true, Index: -5},
{Lower: math.Inf(-1), Upper: 0.7071067811865475, Count: 4, LowerInclusive: true, UpperInclusive: true, Index: -4},
{Lower: math.Inf(-1), Upper: 0.7711054127039704, Count: 4, LowerInclusive: true, UpperInclusive: true, Index: -3},
{Lower: math.Inf(-1), Upper: 0.8408964152537144, Count: 4, LowerInclusive: true, UpperInclusive: true, Index: -2},
{Lower: math.Inf(-1), Upper: 0.9170040432046711, Count: 5, LowerInclusive: true, UpperInclusive: true, Index: -1},
{Lower: math.Inf(-1), Upper: 1, Count: 7, LowerInclusive: true, UpperInclusive: true, Index: 0},
{Lower: math.Inf(-1), Upper: 1.0905077326652577, Count: 8, LowerInclusive: true, UpperInclusive: true, Index: 1},
{Lower: math.Inf(-1), Upper: 1.189207115002721, Count: 8, LowerInclusive: true, UpperInclusive: true, Index: 2},
{Lower: math.Inf(-1), Upper: 1.2968395546510096, Count: 8, LowerInclusive: true, UpperInclusive: true, Index: 3},
{Lower: math.Inf(-1), Upper: 1.414213562373095, Count: 9, LowerInclusive: true, UpperInclusive: true, Index: 4},
{Lower: math.Inf(-1), Upper: 1.5422108254079407, Count: 13, LowerInclusive: true, UpperInclusive: true, Index: 5},
},
},
{
histogram: Histogram{
Schema: -2,
PositiveSpans: []Span{
{Offset: -2, Length: 4}, // -2 -1 0 1
{Offset: 2, Length: 2}, // 4 5
},
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0},
},
expectedBuckets: []Bucket[uint64]{
{Lower: math.Inf(-1), Upper: 0.00390625, Count: 1, LowerInclusive: true, UpperInclusive: true, Index: -2},
{Lower: math.Inf(-1), Upper: 0.0625, Count: 4, LowerInclusive: true, UpperInclusive: true, Index: -1},
{Lower: math.Inf(-1), Upper: 1, Count: 5, LowerInclusive: true, UpperInclusive: true, Index: 0},
{Lower: math.Inf(-1), Upper: 16, Count: 7, LowerInclusive: true, UpperInclusive: true, Index: 1},
{Lower: math.Inf(-1), Upper: 256, Count: 7, LowerInclusive: true, UpperInclusive: true, Index: 2},
{Lower: math.Inf(-1), Upper: 4096, Count: 7, LowerInclusive: true, UpperInclusive: true, Index: 3},
{Lower: math.Inf(-1), Upper: 65536, Count: 8, LowerInclusive: true, UpperInclusive: true, Index: 4},
{Lower: math.Inf(-1), Upper: 1048576, Count: 9, LowerInclusive: true, UpperInclusive: true, Index: 5},
},
},
{
histogram: Histogram{
Schema: -1,
PositiveSpans: []Span{
{Offset: -2, Length: 5}, // -2 -1 0 1 2
},
PositiveBuckets: []int64{1, 2, -2, 1, -1},
},
expectedBuckets: []Bucket[uint64]{
{Lower: math.Inf(-1), Upper: 0.0625, Count: 1, LowerInclusive: true, UpperInclusive: true, Index: -2},
{Lower: math.Inf(-1), Upper: 0.25, Count: 4, LowerInclusive: true, UpperInclusive: true, Index: -1},
{Lower: math.Inf(-1), Upper: 1, Count: 5, LowerInclusive: true, UpperInclusive: true, Index: 0},
{Lower: math.Inf(-1), Upper: 4, Count: 7, LowerInclusive: true, UpperInclusive: true, Index: 1},
{Lower: math.Inf(-1), Upper: 16, Count: 8, LowerInclusive: true, UpperInclusive: true, Index: 2},
},
},
}
for i, c := range cases {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
it := c.histogram.CumulativeBucketIterator()
actualBuckets := make([]Bucket[uint64], 0, len(c.expectedBuckets))
for it.Next() {
actualBuckets = append(actualBuckets, it.At())
}
require.Equal(t, c.expectedBuckets, actualBuckets)
})
}
}
func TestRegularBucketIterator(t *testing.T) {
cases := []struct {
histogram Histogram
expectedPositiveBuckets []Bucket[uint64]
expectedNegativeBuckets []Bucket[uint64]
}{
{
histogram: Histogram{
Schema: 0,
},
expectedPositiveBuckets: []Bucket[uint64]{},
expectedNegativeBuckets: []Bucket[uint64]{},
},
{
histogram: Histogram{
Schema: 0,
PositiveSpans: []Span{
{Offset: 0, Length: 2},
{Offset: 1, Length: 2},
},
PositiveBuckets: []int64{1, 1, -1, 0},
},
expectedPositiveBuckets: []Bucket[uint64]{
{Lower: 0.5, Upper: 1, Count: 1, LowerInclusive: false, UpperInclusive: true, Index: 0},
{Lower: 1, Upper: 2, Count: 2, LowerInclusive: false, UpperInclusive: true, Index: 1},
{Lower: 4, Upper: 8, Count: 1, LowerInclusive: false, UpperInclusive: true, Index: 3},
{Lower: 8, Upper: 16, Count: 1, LowerInclusive: false, UpperInclusive: true, Index: 4},
},
expectedNegativeBuckets: []Bucket[uint64]{},
},
{
histogram: Histogram{
Schema: 0,
NegativeSpans: []Span{
{Offset: 0, Length: 5},
{Offset: 1, Length: 1},
},
NegativeBuckets: []int64{1, 2, -2, 1, -1, 0},
},
expectedPositiveBuckets: []Bucket[uint64]{},
expectedNegativeBuckets: []Bucket[uint64]{
{Lower: -1, Upper: -0.5, Count: 1, LowerInclusive: true, UpperInclusive: false, Index: 0},
{Lower: -2, Upper: -1, Count: 3, LowerInclusive: true, UpperInclusive: false, Index: 1},
{Lower: -4, Upper: -2, Count: 1, LowerInclusive: true, UpperInclusive: false, Index: 2},
{Lower: -8, Upper: -4, Count: 2, LowerInclusive: true, UpperInclusive: false, Index: 3},
{Lower: -16, Upper: -8, Count: 1, LowerInclusive: true, UpperInclusive: false, Index: 4},
{Lower: -64, Upper: -32, Count: 1, LowerInclusive: true, UpperInclusive: false, Index: 6},
},
},
{
histogram: Histogram{
Schema: 0,
PositiveSpans: []Span{
{Offset: 0, Length: 4},
{Offset: 0, Length: 0},
{Offset: 0, Length: 3},
},
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 0},
NegativeSpans: []Span{
{Offset: 0, Length: 5},
{Offset: 1, Length: 0},
{Offset: 0, Length: 1},
},
NegativeBuckets: []int64{1, 2, -2, 1, -1, 0},
},
expectedPositiveBuckets: []Bucket[uint64]{
{Lower: 0.5, Upper: 1, Count: 1, LowerInclusive: false, UpperInclusive: true, Index: 0},
{Lower: 1, Upper: 2, Count: 3, LowerInclusive: false, UpperInclusive: true, Index: 1},
{Lower: 2, Upper: 4, Count: 1, LowerInclusive: false, UpperInclusive: true, Index: 2},
{Lower: 4, Upper: 8, Count: 2, LowerInclusive: false, UpperInclusive: true, Index: 3},
{Lower: 8, Upper: 16, Count: 1, LowerInclusive: false, UpperInclusive: true, Index: 4},
{Lower: 16, Upper: 32, Count: 1, LowerInclusive: false, UpperInclusive: true, Index: 5},
{Lower: 32, Upper: 64, Count: 1, LowerInclusive: false, UpperInclusive: true, Index: 6},
},
expectedNegativeBuckets: []Bucket[uint64]{
{Lower: -1, Upper: -0.5, Count: 1, LowerInclusive: true, UpperInclusive: false, Index: 0},
{Lower: -2, Upper: -1, Count: 3, LowerInclusive: true, UpperInclusive: false, Index: 1},
{Lower: -4, Upper: -2, Count: 1, LowerInclusive: true, UpperInclusive: false, Index: 2},
{Lower: -8, Upper: -4, Count: 2, LowerInclusive: true, UpperInclusive: false, Index: 3},
{Lower: -16, Upper: -8, Count: 1, LowerInclusive: true, UpperInclusive: false, Index: 4},
{Lower: -64, Upper: -32, Count: 1, LowerInclusive: true, UpperInclusive: false, Index: 6},
},
},
{
histogram: Histogram{
Schema: 3,
PositiveSpans: []Span{
{Offset: -5, Length: 2}, // -5 -4
{Offset: 2, Length: 3}, // -1 0 1
{Offset: 2, Length: 2}, // 4 5
},
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 3},
},
expectedPositiveBuckets: []Bucket[uint64]{
{Lower: 0.5946035575013605, Upper: 0.6484197773255048, Count: 1, LowerInclusive: false, UpperInclusive: true, Index: -5},
{Lower: 0.6484197773255048, Upper: 0.7071067811865475, Count: 3, LowerInclusive: false, UpperInclusive: true, Index: -4},
{Lower: 0.8408964152537144, Upper: 0.9170040432046711, Count: 1, LowerInclusive: false, UpperInclusive: true, Index: -1},
{Lower: 0.9170040432046711, Upper: 1, Count: 2, LowerInclusive: false, UpperInclusive: true, Index: 0},
{Lower: 1, Upper: 1.0905077326652577, Count: 1, LowerInclusive: false, UpperInclusive: true, Index: 1},
{Lower: 1.2968395546510096, Upper: 1.414213562373095, Count: 1, LowerInclusive: false, UpperInclusive: true, Index: 4},
{Lower: 1.414213562373095, Upper: 1.5422108254079407, Count: 4, LowerInclusive: false, UpperInclusive: true, Index: 5},
},
expectedNegativeBuckets: []Bucket[uint64]{},
},
{
histogram: Histogram{
Schema: -2,
PositiveSpans: []Span{
{Offset: -2, Length: 4}, // -2 -1 0 1
{Offset: 2, Length: 2}, // 4 5
},
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0},
},
expectedPositiveBuckets: []Bucket[uint64]{
{Lower: 0.000244140625, Upper: 0.00390625, Count: 1, LowerInclusive: false, UpperInclusive: true, Index: -2},
{Lower: 0.00390625, Upper: 0.0625, Count: 3, LowerInclusive: false, UpperInclusive: true, Index: -1},
{Lower: 0.0625, Upper: 1, Count: 1, LowerInclusive: false, UpperInclusive: true, Index: 0},
{Lower: 1, Upper: 16, Count: 2, LowerInclusive: false, UpperInclusive: true, Index: 1},
{Lower: 4096, Upper: 65536, Count: 1, LowerInclusive: false, UpperInclusive: true, Index: 4},
{Lower: 65536, Upper: 1048576, Count: 1, LowerInclusive: false, UpperInclusive: true, Index: 5},
},
expectedNegativeBuckets: []Bucket[uint64]{},
},
{
histogram: Histogram{
Schema: -1,
PositiveSpans: []Span{
{Offset: -2, Length: 5}, // -2 -1 0 1 2
},
PositiveBuckets: []int64{1, 2, -2, 1, -1},
},
expectedPositiveBuckets: []Bucket[uint64]{
{Lower: 0.015625, Upper: 0.0625, Count: 1, LowerInclusive: false, UpperInclusive: true, Index: -2},
{Lower: 0.0625, Upper: 0.25, Count: 3, LowerInclusive: false, UpperInclusive: true, Index: -1},
{Lower: 0.25, Upper: 1, Count: 1, LowerInclusive: false, UpperInclusive: true, Index: 0},
{Lower: 1, Upper: 4, Count: 2, LowerInclusive: false, UpperInclusive: true, Index: 1},
{Lower: 4, Upper: 16, Count: 1, LowerInclusive: false, UpperInclusive: true, Index: 2},
},
expectedNegativeBuckets: []Bucket[uint64]{},
},
}
for i, c := range cases {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
it := c.histogram.PositiveBucketIterator()
actualPositiveBuckets := make([]Bucket[uint64], 0, len(c.expectedPositiveBuckets))
for it.Next() {
actualPositiveBuckets = append(actualPositiveBuckets, it.At())
}
require.Equal(t, c.expectedPositiveBuckets, actualPositiveBuckets)
it = c.histogram.NegativeBucketIterator()
actualNegativeBuckets := make([]Bucket[uint64], 0, len(c.expectedNegativeBuckets))
for it.Next() {
actualNegativeBuckets = append(actualNegativeBuckets, it.At())
}
require.Equal(t, c.expectedNegativeBuckets, actualNegativeBuckets)
})
}
}
func TestHistogramToFloat(t *testing.T) {
h := Histogram{
Schema: 3,
Count: 61,
Sum: 2.7,
ZeroThreshold: 0.1,
ZeroCount: 42,
PositiveSpans: []Span{
{Offset: 0, Length: 4},
{Offset: 0, Length: 0},
{Offset: 0, Length: 3},
},
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 0},
NegativeSpans: []Span{
{Offset: 0, Length: 5},
{Offset: 1, Length: 0},
{Offset: 0, Length: 1},
},
NegativeBuckets: []int64{1, 2, -2, 1, -1, 0},
}
cases := []struct {
name string
fh *FloatHistogram
}{
{name: "without prior float histogram"},
{name: "prior float histogram with more buckets", fh: &FloatHistogram{
Schema: 2,
Count: 3,
Sum: 5,
ZeroThreshold: 4,
ZeroCount: 1,
PositiveSpans: []Span{
{Offset: 1, Length: 2},
{Offset: 1, Length: 2},
{Offset: 1, Length: 2},
},
PositiveBuckets: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9},
NegativeSpans: []Span{
{Offset: 20, Length: 6},
{Offset: 12, Length: 7},
{Offset: 33, Length: 10},
},
NegativeBuckets: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9},
}},
{name: "prior float histogram with fewer buckets", fh: &FloatHistogram{
Schema: 2,
Count: 3,
Sum: 5,
ZeroThreshold: 4,
ZeroCount: 1,
PositiveSpans: []Span{
{Offset: 1, Length: 2},
{Offset: 1, Length: 2},
{Offset: 1, Length: 2},
},
PositiveBuckets: []float64{1, 2},
NegativeSpans: []Span{
{Offset: 20, Length: 6},
{Offset: 12, Length: 7},
{Offset: 33, Length: 10},
},
NegativeBuckets: []float64{1, 2},
}},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
fh := h.ToFloat(c.fh)
require.Equal(t, h.String(), fh.String())
})
}
}
// TestHistogramEquals tests both Histogram and FloatHistogram.
func TestHistogramEquals(t *testing.T) {
h1 := Histogram{
Schema: 3,
Count: 61,
Sum: 2.7,
ZeroThreshold: 0.1,
ZeroCount: 42,
PositiveSpans: []Span{
{Offset: 0, Length: 4},
{Offset: 10, Length: 3},
},
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 0},
NegativeSpans: []Span{
{Offset: 0, Length: 4},
{Offset: 10, Length: 3},
},
NegativeBuckets: []int64{1, 2, -2, 1, -1, 0, 0},
}
equals := func(h1, h2 Histogram) {
require.True(t, h1.Equals(&h2))
require.True(t, h2.Equals(&h1))
h1f, h2f := h1.ToFloat(nil), h2.ToFloat(nil)
require.True(t, h1f.Equals(h2f))
require.True(t, h2f.Equals(h1f))
}
notEquals := func(h1, h2 Histogram) {
require.False(t, h1.Equals(&h2))
require.False(t, h2.Equals(&h1))
h1f, h2f := h1.ToFloat(nil), h2.ToFloat(nil)
require.False(t, h1f.Equals(h2f))
require.False(t, h2f.Equals(h1f))
}
h2 := h1.Copy()
equals(h1, *h2)
// Changed spans but same layout.
h2.PositiveSpans = append(h2.PositiveSpans, Span{Offset: 5})
h2.NegativeSpans = append(h2.NegativeSpans, Span{Offset: 2})
equals(h1, *h2)
// Adding empty spans in between.
h2.PositiveSpans[1].Offset = 6
h2.PositiveSpans = []Span{
h2.PositiveSpans[0],
{Offset: 1},
{Offset: 3},
h2.PositiveSpans[1],
h2.PositiveSpans[2],
}
h2.NegativeSpans[1].Offset = 5
h2.NegativeSpans = []Span{
h2.NegativeSpans[0],
{Offset: 2},
{Offset: 3},
h2.NegativeSpans[1],
h2.NegativeSpans[2],
}
equals(h1, *h2)
// All mismatches.
notEquals(h1, Histogram{})
h2.Schema = 1
notEquals(h1, *h2)
h2 = h1.Copy()
h2.Count++
notEquals(h1, *h2)
h2 = h1.Copy()
h2.Sum++
notEquals(h1, *h2)
h2 = h1.Copy()
h2.ZeroThreshold++
notEquals(h1, *h2)
h2 = h1.Copy()
h2.ZeroCount++
notEquals(h1, *h2)
// Changing value of buckets.
h2 = h1.Copy()
h2.PositiveBuckets[len(h2.PositiveBuckets)-1]++
notEquals(h1, *h2)
h2 = h1.Copy()
h2.NegativeBuckets[len(h2.NegativeBuckets)-1]++
notEquals(h1, *h2)
// Changing bucket layout.
h2 = h1.Copy()
h2.PositiveSpans[1].Offset++
notEquals(h1, *h2)
h2 = h1.Copy()
h2.NegativeSpans[1].Offset++
notEquals(h1, *h2)
// Adding an empty bucket.
h2 = h1.Copy()
h2.PositiveSpans[0].Offset--
h2.PositiveSpans[0].Length++
h2.PositiveBuckets = append([]int64{0}, h2.PositiveBuckets...)
notEquals(h1, *h2)
h2 = h1.Copy()
h2.NegativeSpans[0].Offset--
h2.NegativeSpans[0].Length++
h2.NegativeBuckets = append([]int64{0}, h2.NegativeBuckets...)
notEquals(h1, *h2)
// Adding new bucket.
h2 = h1.Copy()
h2.PositiveSpans = append(h2.PositiveSpans, Span{
Offset: 1,
Length: 1,
})
h2.PositiveBuckets = append(h2.PositiveBuckets, 1)
notEquals(h1, *h2)
h2 = h1.Copy()
h2.NegativeSpans = append(h2.NegativeSpans, Span{
Offset: 1,
Length: 1,
})
h2.NegativeBuckets = append(h2.NegativeBuckets, 1)
notEquals(h1, *h2)
// Sum is StaleNaN.
hStale := h1.Copy()
hStale.Sum = math.Float64frombits(value.StaleNaN)
notEquals(h1, *hStale)
equals(*hStale, *hStale)
// Sum is NaN (but not a StaleNaN).
hNaN := h1.Copy()
hNaN.Sum = math.NaN()
notEquals(h1, *hNaN)
equals(*hNaN, *hNaN)
// Sum StaleNaN vs regular NaN.
notEquals(*hStale, *hNaN)
}
func TestHistogramCopy(t *testing.T) {
cases := []struct {
name string
orig *Histogram
expected *Histogram
}{
{
name: "without buckets",
orig: &Histogram{},
expected: &Histogram{},
},
{
name: "with buckets",
orig: &Histogram{
PositiveSpans: []Span{{-2, 1}},
PositiveBuckets: []int64{1, 3, -3, 42},
NegativeSpans: []Span{{3, 2}},
NegativeBuckets: []int64{5, 3, 1.234e5, 1000},
},
expected: &Histogram{
PositiveSpans: []Span{{-2, 1}},
PositiveBuckets: []int64{1, 3, -3, 42},
NegativeSpans: []Span{{3, 2}},
NegativeBuckets: []int64{5, 3, 1.234e5, 1000},
},
},
{
name: "with empty buckets and non empty capacity",
orig: &Histogram{
PositiveSpans: make([]Span, 0, 1),
PositiveBuckets: make([]int64, 0, 1),
NegativeSpans: make([]Span, 0, 1),
NegativeBuckets: make([]int64, 0, 1),
},
expected: &Histogram{},
},
}
for _, tcase := range cases {
t.Run(tcase.name, func(t *testing.T) {
hCopy := tcase.orig.Copy()
// Modify a primitive value in the original histogram.
tcase.orig.Sum++
require.Equal(t, tcase.expected, hCopy)
assertDeepCopyHSpans(t, tcase.orig, hCopy, tcase.expected)
})
}
}
func TestHistogramCopyTo(t *testing.T) {
cases := []struct {
name string
orig *Histogram
expected *Histogram
}{
{
name: "without buckets",
orig: &Histogram{},
expected: &Histogram{},
},
{
name: "with buckets",
orig: &Histogram{
PositiveSpans: []Span{{-2, 1}},
PositiveBuckets: []int64{1, 3, -3, 42},
NegativeSpans: []Span{{3, 2}},
NegativeBuckets: []int64{5, 3, 1.234e5, 1000},
},
expected: &Histogram{
PositiveSpans: []Span{{-2, 1}},
PositiveBuckets: []int64{1, 3, -3, 42},
NegativeSpans: []Span{{3, 2}},
NegativeBuckets: []int64{5, 3, 1.234e5, 1000},
},
},
{
name: "with empty buckets and non empty capacity",
orig: &Histogram{
PositiveSpans: make([]Span, 0, 1),
PositiveBuckets: make([]int64, 0, 1),
NegativeSpans: make([]Span, 0, 1),
NegativeBuckets: make([]int64, 0, 1),
},
expected: &Histogram{},
},
}
for _, tcase := range cases {
t.Run(tcase.name, func(t *testing.T) {
hCopy := &Histogram{}
tcase.orig.CopyTo(hCopy)
// Modify a primitive value in the original histogram.
tcase.orig.Sum++
require.Equal(t, tcase.expected, hCopy)
assertDeepCopyHSpans(t, tcase.orig, hCopy, tcase.expected)
})
}
}
func assertDeepCopyHSpans(t *testing.T, orig, hCopy, expected *Histogram) {
// Do an in-place expansion of an original spans slice.
orig.PositiveSpans = expandSpans(orig.PositiveSpans)
orig.PositiveSpans[len(orig.PositiveSpans)-1] = Span{1, 2}
hCopy.PositiveSpans = expandSpans(hCopy.PositiveSpans)
expected.PositiveSpans = expandSpans(expected.PositiveSpans)
// Expand the copy spans and assert that modifying the original has not affected the copy.
require.Equal(t, expected, hCopy)
}
func expandSpans(spans []Span) []Span {
n := len(spans)
if cap(spans) > n {
spans = spans[:n+1]
} else {
spans = append(spans, Span{})
}
return spans
}
func TestHistogramCompact(t *testing.T) {
cases := []struct {
name string
in *Histogram
maxEmptyBuckets int
expected *Histogram
}{
{
"empty histogram",
&Histogram{},
0,
&Histogram{},
},
{
"nothing should happen",
&Histogram{
PositiveSpans: []Span{{-2, 1}, {2, 3}},
PositiveBuckets: []int64{1, 3, -3, 42},
NegativeSpans: []Span{{3, 2}, {3, 2}},
NegativeBuckets: []int64{5, 3, 1.234e5, 1000},
},
0,
&Histogram{
PositiveSpans: []Span{{-2, 1}, {2, 3}},
PositiveBuckets: []int64{1, 3, -3, 42},
NegativeSpans: []Span{{3, 2}, {3, 2}},
NegativeBuckets: []int64{5, 3, 1.234e5, 1000},
},
},
{
"eliminate zero offsets",
&Histogram{
PositiveSpans: []Span{{-2, 1}, {0, 3}, {0, 1}},
PositiveBuckets: []int64{1, 3, -3, 42, 3},
NegativeSpans: []Span{{0, 2}, {0, 2}, {2, 1}, {0, 1}},
NegativeBuckets: []int64{5, 3, 1.234e5, 1000, 3, 4},
},
0,
&Histogram{
PositiveSpans: []Span{{-2, 5}},
PositiveBuckets: []int64{1, 3, -3, 42, 3},
NegativeSpans: []Span{{0, 4}, {2, 2}},
NegativeBuckets: []int64{5, 3, 1.234e5, 1000, 3, 4},
},
},
{
"eliminate zero length",
&Histogram{
PositiveSpans: []Span{{-2, 2}, {2, 0}, {3, 3}},
PositiveBuckets: []int64{1, 3, -3, 42, 3},
NegativeSpans: []Span{{0, 2}, {0, 0}, {2, 0}, {1, 4}},
NegativeBuckets: []int64{5, 3, 1.234e5, 1000, 3, 4},
},
0,
&Histogram{
PositiveSpans: []Span{{-2, 2}, {5, 3}},
PositiveBuckets: []int64{1, 3, -3, 42, 3},
NegativeSpans: []Span{{0, 2}, {3, 4}},
NegativeBuckets: []int64{5, 3, 1.234e5, 1000, 3, 4},
},
},
{
"eliminate multiple zero length spans",
&Histogram{
PositiveSpans: []Span{{-2, 2}, {2, 0}, {2, 0}, {2, 0}, {3, 3}},
PositiveBuckets: []int64{1, 3, -3, 42, 3},
},
0,
&Histogram{
PositiveSpans: []Span{{-2, 2}, {9, 3}},
PositiveBuckets: []int64{1, 3, -3, 42, 3},
},
},
{
"cut empty buckets at start or end",
&Histogram{
PositiveSpans: []Span{{-4, 4}, {5, 3}},
PositiveBuckets: []int64{0, 0, 1, 3, -3, 42, 3},
NegativeSpans: []Span{{0, 2}, {3, 5}},
NegativeBuckets: []int64{5, 3, -4, -2, 3, 4, -9},
},
0,
&Histogram{
PositiveSpans: []Span{{-2, 2}, {5, 3}},
PositiveBuckets: []int64{1, 3, -3, 42, 3},
NegativeSpans: []Span{{0, 2}, {3, 4}},
NegativeBuckets: []int64{5, 3, -4, -2, 3, 4},
},
},
{
"cut empty buckets at start and end",
&Histogram{
PositiveSpans: []Span{{-4, 4}, {5, 6}},
PositiveBuckets: []int64{0, 0, 1, 3, -3, 42, 3, -46, 0, 0},
NegativeSpans: []Span{{-2, 4}, {3, 5}},
NegativeBuckets: []int64{0, 0, 5, 3, -4, -2, 3, 4, -9},
},
0,
&Histogram{
PositiveSpans: []Span{{-2, 2}, {5, 3}},
PositiveBuckets: []int64{1, 3, -3, 42, 3},
NegativeSpans: []Span{{0, 2}, {3, 4}},
NegativeBuckets: []int64{5, 3, -4, -2, 3, 4},
},
},
{
"cut empty buckets at start or end of spans, even in the middle",
&Histogram{
PositiveSpans: []Span{{-4, 6}, {3, 6}},
PositiveBuckets: []int64{0, 0, 1, 3, -4, 0, 1, 42, 3, -46, 0, 0},
NegativeSpans: []Span{{0, 2}, {2, 6}},
NegativeBuckets: []int64{5, 3, -8, 4, -2, 3, 4, -9},
},
0,
&Histogram{
PositiveSpans: []Span{{-2, 2}, {5, 3}},
PositiveBuckets: []int64{1, 3, -3, 42, 3},
NegativeSpans: []Span{{0, 2}, {3, 4}},
NegativeBuckets: []int64{5, 3, -4, -2, 3, 4},
},
},
{
"cut empty buckets at start or end but merge spans due to maxEmptyBuckets",
&Histogram{
PositiveSpans: []Span{{-4, 4}, {5, 3}},
PositiveBuckets: []int64{0, 0, 1, 3, -3, 42, 3},
NegativeSpans: []Span{{0, 2}, {3, 5}},
NegativeBuckets: []int64{5, 3, -4, -2, 3, 4, -9},
},
10,
&Histogram{
PositiveSpans: []Span{{-2, 10}},
PositiveBuckets: []int64{1, 3, -4, 0, 0, 0, 0, 1, 42, 3},
NegativeSpans: []Span{{0, 9}},
NegativeBuckets: []int64{5, 3, -8, 0, 0, 4, -2, 3, 4},
},
},
{
"cut empty buckets from the middle of a span",
&Histogram{
PositiveSpans: []Span{{-4, 6}, {3, 3}},
PositiveBuckets: []int64{0, 0, 1, -1, 0, 3, -2, 42, 3},
NegativeSpans: []Span{{0, 2}, {3, 5}},
NegativeBuckets: []int64{5, 3, -4, -2, -2, 3, 4},
},
0,
&Histogram{
PositiveSpans: []Span{{-2, 1}, {2, 1}, {3, 3}},
PositiveBuckets: []int64{1, 2, -2, 42, 3},
NegativeSpans: []Span{{0, 2}, {3, 2}, {1, 2}},
NegativeBuckets: []int64{5, 3, -4, -2, 1, 4},
},
},
{
"cut out a span containing only empty buckets",
&Histogram{
PositiveSpans: []Span{{-4, 3}, {2, 2}, {3, 4}},
PositiveBuckets: []int64{0, 0, 1, -1, 0, 3, -2, 42, 3},
},
0,
&Histogram{
PositiveSpans: []Span{{-2, 1}, {7, 4}},
PositiveBuckets: []int64{1, 2, -2, 42, 3},
},
},
{
"cut empty buckets from the middle of a span, avoiding some due to maxEmptyBuckets",
&Histogram{
PositiveSpans: []Span{{-4, 6}, {3, 3}},
PositiveBuckets: []int64{0, 0, 1, -1, 0, 3, -2, 42, 3},
NegativeSpans: []Span{{0, 2}, {3, 5}},
NegativeBuckets: []int64{5, 3, -4, -2, -2, 3, 4},
},
1,
&Histogram{
PositiveSpans: []Span{{-2, 1}, {2, 1}, {3, 3}},
PositiveBuckets: []int64{1, 2, -2, 42, 3},
NegativeSpans: []Span{{0, 2}, {3, 5}},
NegativeBuckets: []int64{5, 3, -4, -2, -2, 3, 4},
},
},
{
"avoiding all cutting of empty buckets from the middle of a chunk due to maxEmptyBuckets",
&Histogram{
PositiveSpans: []Span{{-4, 6}, {3, 3}},
PositiveBuckets: []int64{0, 0, 1, -1, 0, 3, -2, 42, 3},
NegativeSpans: []Span{{0, 2}, {3, 5}},
NegativeBuckets: []int64{5, 3, -4, -2, -2, 3, 4},
},
2,
&Histogram{
PositiveSpans: []Span{{-2, 4}, {3, 3}},
PositiveBuckets: []int64{1, -1, 0, 3, -2, 42, 3},
NegativeSpans: []Span{{0, 2}, {3, 5}},
NegativeBuckets: []int64{5, 3, -4, -2, -2, 3, 4},
},
},
{
"everything merged into one span due to maxEmptyBuckets",
&Histogram{
PositiveSpans: []Span{{-4, 6}, {3, 3}},
PositiveBuckets: []int64{0, 0, 1, -1, 0, 3, -2, 42, 3},
NegativeSpans: []Span{{0, 2}, {3, 5}},
NegativeBuckets: []int64{5, 3, -4, -2, -2, 3, 4},
},
3,
&Histogram{
PositiveSpans: []Span{{-2, 10}},
PositiveBuckets: []int64{1, -1, 0, 3, -3, 0, 0, 1, 42, 3},
NegativeSpans: []Span{{0, 10}},
NegativeBuckets: []int64{5, 3, -8, 0, 0, 4, -2, -2, 3, 4},
},
},
{
"only empty buckets and maxEmptyBuckets greater zero",
&Histogram{
PositiveSpans: []Span{{-4, 6}, {3, 3}},
PositiveBuckets: []int64{0, 0, 0, 0, 0, 0, 0, 0, 0},
NegativeSpans: []Span{{0, 7}},
NegativeBuckets: []int64{0, 0, 0, 0, 0, 0, 0},
},
3,
&Histogram{
PositiveSpans: []Span{},
PositiveBuckets: []int64{},
NegativeSpans: []Span{},
NegativeBuckets: []int64{},
},
},
{
"multiple spans of only empty buckets",
&Histogram{
PositiveSpans: []Span{{-10, 2}, {2, 1}, {3, 3}},
PositiveBuckets: []int64{0, 0, 0, 0, 2, 3},
NegativeSpans: []Span{{-10, 2}, {2, 1}, {3, 3}},
NegativeBuckets: []int64{2, 3, -5, 0, 0, 0},
},
0,
&Histogram{
PositiveSpans: []Span{{-1, 2}},
PositiveBuckets: []int64{2, 3},
NegativeSpans: []Span{{-10, 2}},
NegativeBuckets: []int64{2, 3},
},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
require.Equal(t, c.expected, c.in.Compact(c.maxEmptyBuckets))
// Compact has happened in-place, too.
require.Equal(t, c.expected, c.in)
})
}
}
func TestHistogramValidation(t *testing.T) {
tests := map[string]struct {
h *Histogram
errMsg string
skipFloat bool
}{
"valid histogram": {
h: &Histogram{
Count: 12,
ZeroCount: 2,
ZeroThreshold: 0.001,
Sum: 19.4,
Schema: 1,
PositiveSpans: []Span{
{Offset: 0, Length: 2},
{Offset: 1, Length: 2},
},
PositiveBuckets: []int64{1, 1, -1, 0},
NegativeSpans: []Span{
{Offset: 0, Length: 2},
{Offset: 1, Length: 2},
},
NegativeBuckets: []int64{1, 1, -1, 0},
},
},
"valid histogram with NaN observations that has its Count (4) higher than the actual total of buckets (2 + 1)": {
// This case is possible if NaN values (which do not fall into any bucket) are observed.
h: &Histogram{
ZeroCount: 2,
Count: 4,
Sum: math.NaN(),
PositiveSpans: []Span{{Offset: 0, Length: 1}},
PositiveBuckets: []int64{1},
},
},
"rejects histogram without NaN observations that has its Count (4) higher than the actual total of buckets (2 + 1)": {
h: &Histogram{
ZeroCount: 2,
Count: 4,
Sum: 333,
PositiveSpans: []Span{{Offset: 0, Length: 1}},
PositiveBuckets: []int64{1},
},
errMsg: `3 observations found in buckets, but the Count field is 4: histogram's observation count should equal the number of observations found in the buckets (in absence of NaN)`,
skipFloat: true,
},
"rejects histogram that has too few negative buckets": {
h: &Histogram{
NegativeSpans: []Span{{Offset: 0, Length: 1}},
NegativeBuckets: []int64{},
},
errMsg: `negative side: spans need 1 buckets, have 0 buckets: histogram spans specify different number of buckets than provided`,
},
"rejects histogram that has too few positive buckets": {
h: &Histogram{
PositiveSpans: []Span{{Offset: 0, Length: 1}},
PositiveBuckets: []int64{},
},
errMsg: `positive side: spans need 1 buckets, have 0 buckets: histogram spans specify different number of buckets than provided`,
},
"rejects histogram that has too many negative buckets": {
h: &Histogram{
NegativeSpans: []Span{{Offset: 0, Length: 1}},
NegativeBuckets: []int64{1, 2},
},
errMsg: `negative side: spans need 1 buckets, have 2 buckets: histogram spans specify different number of buckets than provided`,
},
"rejects histogram that has too many positive buckets": {
h: &Histogram{
PositiveSpans: []Span{{Offset: 0, Length: 1}},
PositiveBuckets: []int64{1, 2},
},
errMsg: `positive side: spans need 1 buckets, have 2 buckets: histogram spans specify different number of buckets than provided`,
},
"rejects a histogram that has a negative span with a negative offset": {
h: &Histogram{
NegativeSpans: []Span{{Offset: -1, Length: 1}, {Offset: -1, Length: 1}},
NegativeBuckets: []int64{1, 2},
},
errMsg: `negative side: span number 2 with offset -1: histogram has a span whose offset is negative`,
},
"rejects a histogram which has a positive span with a negative offset": {
h: &Histogram{
PositiveSpans: []Span{{Offset: -1, Length: 1}, {Offset: -1, Length: 1}},
PositiveBuckets: []int64{1, 2},
},
errMsg: `positive side: span number 2 with offset -1: histogram has a span whose offset is negative`,
},
"rejects a histogram that has a negative bucket with a negative count": {
h: &Histogram{
NegativeSpans: []Span{{Offset: -1, Length: 1}},
NegativeBuckets: []int64{-1},
},
errMsg: `negative side: bucket number 1 has observation count of -1: histogram has a bucket whose observation count is negative`,
},
"rejects a histogram that has a positive bucket with a negative count": {
h: &Histogram{
PositiveSpans: []Span{{Offset: -1, Length: 1}},
PositiveBuckets: []int64{-1},
},
errMsg: `positive side: bucket number 1 has observation count of -1: histogram has a bucket whose observation count is negative`,
},
"rejects a histogram that has a lower count than count in buckets": {
h: &Histogram{
Count: 0,
NegativeSpans: []Span{{Offset: -1, Length: 1}},
PositiveSpans: []Span{{Offset: -1, Length: 1}},
NegativeBuckets: []int64{1},
PositiveBuckets: []int64{1},
},
errMsg: `2 observations found in buckets, but the Count field is 0: histogram's observation count should equal the number of observations found in the buckets (in absence of NaN)`,
skipFloat: true,
},
"rejects a histogram that doesn't count the zero bucket in its count": {
h: &Histogram{
Count: 2,
ZeroCount: 1,
NegativeSpans: []Span{{Offset: -1, Length: 1}},
PositiveSpans: []Span{{Offset: -1, Length: 1}},
NegativeBuckets: []int64{1},
PositiveBuckets: []int64{1},
},
errMsg: `3 observations found in buckets, but the Count field is 2: histogram's observation count should equal the number of observations found in the buckets (in absence of NaN)`,
skipFloat: true,
},
}
for testName, tc := range tests {
t.Run(testName, func(t *testing.T) {
if err := tc.h.Validate(); tc.errMsg != "" {
require.EqualError(t, err, tc.errMsg)
} else {
require.NoError(t, err)
}
if tc.skipFloat {
return
}
fh := tc.h.ToFloat(nil)
if err := fh.Validate(); tc.errMsg != "" {
require.EqualError(t, err, tc.errMsg)
} else {
require.NoError(t, err)
}
})
}
}
func BenchmarkHistogramValidation(b *testing.B) {
histograms := GenerateBigTestHistograms(b.N, 500)
b.ResetTimer()
for _, h := range histograms {
require.NoError(b, h.Validate())
}
}
func TestHistogramReduceResolution(t *testing.T) {
tcs := map[string]struct {
origin *Histogram
target *Histogram
}{
"valid histogram": {
origin: &Histogram{
Schema: 0,
PositiveSpans: []Span{
{Offset: 0, Length: 4},
{Offset: 0, Length: 0},
{Offset: 3, Length: 2},
},
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0},
NegativeSpans: []Span{
{Offset: 0, Length: 4},
{Offset: 0, Length: 0},
{Offset: 3, Length: 2},
},
NegativeBuckets: []int64{1, 2, -2, 1, -1, 0},
},
target: &Histogram{
Schema: -1,
PositiveSpans: []Span{
{Offset: 0, Length: 3},
{Offset: 1, Length: 1},
},
PositiveBuckets: []int64{1, 3, -2, 0},
NegativeSpans: []Span{
{Offset: 0, Length: 3},
{Offset: 1, Length: 1},
},
NegativeBuckets: []int64{1, 3, -2, 0},
},
},
}
for _, tc := range tcs {
target := tc.origin.ReduceResolution(tc.target.Schema)
require.Equal(t, tc.target, target)
// Check that receiver histogram was mutated:
require.Equal(t, tc.target, tc.origin)
}
}
Reference in a new issue View git blame Copy permalink