mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2024-11-09 23:24:05 -08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Add custom buckets to native histogram chunks encoding (#13706)

Browse source 
* add custom bounds to chunks encoding
* change custom buckets schema number
* rename custom bounds to custom values

Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
This commit is contained in:
zenador 2024-03-22 21:36:39 +08:00 committed by GitHub
parent 5d0a0a7542
commit 4acbb7dea6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 953 additions and 331 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

52
model/histogram/float_histogram.go
View file

@ -34,8 +34,8 @@ type FloatHistogram struct {
// They are all for base-2 bucket schemas, where 1 is a bucket boundary in
// each case, and then each power of two is divided into 2^n logarithmic buckets.
// Or in other words, each bucket boundary is the previous boundary times
// 2^(2^-n). Another valid schema number is 127 for custom buckets, defined by
// the CustomBounds field.
// 2^(2^-n). Another valid schema number is -53 for custom buckets, defined by
// the CustomValues field.
Schema int32
// Width of the zero bucket.
ZeroThreshold float64
@ -53,9 +53,9 @@ type FloatHistogram struct {
// Holds the custom (usually upper) bounds for bucket definitions, otherwise nil.
// This slice is interned, to be treated as immutable and copied by reference.
// These numbers should be strictly increasing. This field is only used when the
// schema is 127, and the ZeroThreshold, ZeroCount, NegativeSpans and NegativeBuckets
// fields are not used.
CustomBounds []float64
// schema is for custom buckets, and the ZeroThreshold, ZeroCount, NegativeSpans
// and NegativeBuckets fields are not used.
CustomValues []float64
}
func (h *FloatHistogram) UsesCustomBuckets() bool {
@ -72,7 +72,10 @@ func (h *FloatHistogram) Copy() *FloatHistogram {
}
if h.UsesCustomBuckets() {
c.CustomBounds = h.CustomBounds
if len(h.CustomValues) != 0 {
c.CustomValues = make([]float64, len(h.CustomValues))
copy(c.CustomValues, h.CustomValues)
}
} else {
c.ZeroThreshold = h.ZeroThreshold
c.ZeroCount = h.ZeroCount
@ -114,7 +117,8 @@ func (h *FloatHistogram) CopyTo(to *FloatHistogram) {
to.NegativeSpans = clearIfNotNil(to.NegativeSpans)
to.NegativeBuckets = clearIfNotNil(to.NegativeBuckets)
to.CustomBounds = h.CustomBounds
to.CustomValues = resize(to.CustomValues, len(h.CustomValues))
copy(to.CustomValues, h.CustomValues)
} else {
to.ZeroThreshold = h.ZeroThreshold
to.ZeroCount = h.ZeroCount
@ -125,7 +129,7 @@ func (h *FloatHistogram) CopyTo(to *FloatHistogram) {
to.NegativeBuckets = resize(to.NegativeBuckets, len(h.NegativeBuckets))
copy(to.NegativeBuckets, h.NegativeBuckets)
to.CustomBounds = clearIfNotNil(to.CustomBounds)
to.CustomValues = clearIfNotNil(to.CustomValues)
}
to.PositiveSpans = resize(to.PositiveSpans, len(h.PositiveSpans))
@ -311,7 +315,7 @@ func (h *FloatHistogram) Add(other *FloatHistogram) (*FloatHistogram, error) {
if h.UsesCustomBuckets() != other.UsesCustomBuckets() {
return nil, ErrHistogramsIncompatibleSchema
}
if h.UsesCustomBuckets() && !floatBucketsMatch(h.CustomBounds, other.CustomBounds) {
if h.UsesCustomBuckets() && !FloatBucketsMatch(h.CustomValues, other.CustomValues) {
return nil, ErrHistogramsIncompatibleBounds
}
@ -387,7 +391,7 @@ func (h *FloatHistogram) Sub(other *FloatHistogram) (*FloatHistogram, error) {
if h.UsesCustomBuckets() != other.UsesCustomBuckets() {
return nil, ErrHistogramsIncompatibleSchema
}
if h.UsesCustomBuckets() && !floatBucketsMatch(h.CustomBounds, other.CustomBounds) {
if h.UsesCustomBuckets() && !FloatBucketsMatch(h.CustomValues, other.CustomValues) {
return nil, ErrHistogramsIncompatibleBounds
}
@ -454,7 +458,7 @@ func (h *FloatHistogram) Equals(h2 *FloatHistogram) bool {
}
if h.UsesCustomBuckets() {
if !floatBucketsMatch(h.CustomBounds, h2.CustomBounds) {
if !FloatBucketsMatch(h.CustomValues, h2.CustomValues) {
return false
}
}
@ -467,14 +471,14 @@ func (h *FloatHistogram) Equals(h2 *FloatHistogram) bool {
if !spansMatch(h.NegativeSpans, h2.NegativeSpans) {
return false
}
if !floatBucketsMatch(h.NegativeBuckets, h2.NegativeBuckets) {
if !FloatBucketsMatch(h.NegativeBuckets, h2.NegativeBuckets) {
return false
}
if !spansMatch(h.PositiveSpans, h2.PositiveSpans) {
return false
}
if !floatBucketsMatch(h.PositiveBuckets, h2.PositiveBuckets) {
if !FloatBucketsMatch(h.PositiveBuckets, h2.PositiveBuckets) {
return false
}
@ -490,7 +494,7 @@ func (h *FloatHistogram) Size() int {
negSpanSize := len(h.NegativeSpans) * 8 // 8 bytes (int32 + uint32).
posBucketSize := len(h.PositiveBuckets) * 8 // 8 bytes (float64).
negBucketSize := len(h.NegativeBuckets) * 8 // 8 bytes (float64).
customBoundSize := len(h.CustomBounds) * 8 // 8 bytes (float64).
customBoundSize := len(h.CustomValues) * 8 // 8 bytes (float64).
// Total size of the struct.
@ -505,7 +509,7 @@ func (h *FloatHistogram) Size() int {
// fh.NegativeSpans is 24 bytes.
// fh.PositiveBuckets is 24 bytes.
// fh.NegativeBuckets is 24 bytes.
// fh.CustomBounds is 24 bytes.
// fh.CustomValues is 24 bytes.
structSize := 168
return structSize + posSpanSize + negSpanSize + posBucketSize + negBucketSize + customBoundSize
@ -593,7 +597,7 @@ func (h *FloatHistogram) DetectReset(previous *FloatHistogram) bool {
if h.Count < previous.Count {
return true
}
if h.UsesCustomBuckets() != previous.UsesCustomBuckets() || (h.UsesCustomBuckets() && !floatBucketsMatch(h.CustomBounds, previous.CustomBounds)) {
if h.UsesCustomBuckets() != previous.UsesCustomBuckets() || (h.UsesCustomBuckets() && !FloatBucketsMatch(h.CustomValues, previous.CustomValues)) {
// Mark that something has changed or that the application has been restarted. However, this does
// not matter so much since the change in schema will be handled directly in the chunks and PromQL
// functions.
@ -704,7 +708,7 @@ func (h *FloatHistogram) NegativeBucketIterator() BucketIterator[float64] {
// positive buckets in descending order (starting at the highest bucket and
// going down towards the zero bucket).
func (h *FloatHistogram) PositiveReverseBucketIterator() BucketIterator[float64] {
it := newReverseFloatBucketIterator(h.PositiveSpans, h.PositiveBuckets, h.Schema, true, h.CustomBounds)
it := newReverseFloatBucketIterator(h.PositiveSpans, h.PositiveBuckets, h.Schema, true, h.CustomValues)
return &it
}
@ -738,7 +742,7 @@ func (h *FloatHistogram) AllBucketIterator() BucketIterator[float64] {
func (h *FloatHistogram) AllReverseBucketIterator() BucketIterator[float64] {
return &allFloatBucketIterator{
h: h,
leftIter: newReverseFloatBucketIterator(h.PositiveSpans, h.PositiveBuckets, h.Schema, true, h.CustomBounds),
leftIter: newReverseFloatBucketIterator(h.PositiveSpans, h.PositiveBuckets, h.Schema, true, h.CustomValues),
rightIter: h.floatBucketIterator(false, 0, h.Schema),
state: -1,
}
@ -753,7 +757,7 @@ func (h *FloatHistogram) AllReverseBucketIterator() BucketIterator[float64] {
func (h *FloatHistogram) Validate() error {
var nCount, pCount float64
if h.UsesCustomBuckets() {
if err := checkHistogramCustomBounds(h.CustomBounds, h.PositiveSpans, len(h.PositiveBuckets)); err != nil {
if err := checkHistogramCustomBounds(h.CustomValues, h.PositiveSpans, len(h.PositiveBuckets)); err != nil {
return fmt.Errorf("custom buckets: %w", err)
}
if h.ZeroCount != 0 {
@ -779,7 +783,7 @@ func (h *FloatHistogram) Validate() error {
if err != nil {
return fmt.Errorf("negative side: %w", err)
}
if h.CustomBounds != nil {
if h.CustomValues != nil {
return fmt.Errorf("histogram with exponential schema must not have custom bounds")
}
}
@ -940,7 +944,7 @@ func (h *FloatHistogram) floatBucketIterator(
if positive {
i.spans = h.PositiveSpans
i.buckets = h.PositiveBuckets
i.customBounds = h.CustomBounds
i.customValues = h.CustomValues
} else {
i.spans = h.NegativeSpans
i.buckets = h.NegativeBuckets
@ -950,7 +954,7 @@ func (h *FloatHistogram) floatBucketIterator(
// reverseFloatBucketIterator is a low-level constructor for reverse bucket iterators.
func newReverseFloatBucketIterator(
spans []Span, buckets []float64, schema int32, positive bool, customBounds []float64,
spans []Span, buckets []float64, schema int32, positive bool, customValues []float64,
) reverseFloatBucketIterator {
r := reverseFloatBucketIterator{
baseBucketIterator: baseBucketIterator[float64, float64]{
@ -958,7 +962,7 @@ func newReverseFloatBucketIterator(
spans: spans,
buckets: buckets,
positive: positive,
customBounds: customBounds,
customValues: customValues,
},
}
@ -1296,7 +1300,7 @@ func addBuckets(
return spansA, bucketsA
}
func floatBucketsMatch(b1, b2 []float64) bool {
func FloatBucketsMatch(b1, b2 []float64) bool {
if len(b1) != len(b2) {
return false
}

144
model/histogram/float_histogram_test.go
View file

@ -139,7 +139,7 @@ func TestFloatHistogramMul(t *testing.T) {
Sum: 2349209.324,
PositiveSpans: []Span{{0, 1}, {2, 3}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1},
CustomBounds: []float64{1, 2, 3},
CustomValues: []float64{1, 2, 3},
},
1,
&FloatHistogram{
@ -148,7 +148,7 @@ func TestFloatHistogramMul(t *testing.T) {
Sum: 2349209.324,
PositiveSpans: []Span{{0, 1}, {2, 3}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1},
CustomBounds: []float64{1, 2, 3},
CustomValues: []float64{1, 2, 3},
},
},
{
@ -159,7 +159,7 @@ func TestFloatHistogramMul(t *testing.T) {
Sum: 23,
PositiveSpans: []Span{{0, 2}, {1, 3}},
PositiveBuckets: []float64{1, 0, 3, 4, 7},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
3,
&FloatHistogram{
@ -168,7 +168,7 @@ func TestFloatHistogramMul(t *testing.T) {
Sum: 69,
PositiveSpans: []Span{{0, 2}, {1, 3}},
PositiveBuckets: []float64{3, 0, 9, 12, 21},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
},
}
@ -224,13 +224,13 @@ func TestFloatHistogramCopy(t *testing.T) {
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{0, 1}},
PositiveBuckets: []float64{1, 3, -3, 42},
CustomBounds: []float64{1, 2, 3},
CustomValues: []float64{1, 2, 3},
},
expected: &FloatHistogram{
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{0, 1}},
PositiveBuckets: []float64{1, 3, -3, 42},
CustomBounds: []float64{1, 2, 3},
CustomValues: []float64{1, 2, 3},
},
},
}
@ -289,13 +289,13 @@ func TestFloatHistogramCopyTo(t *testing.T) {
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{0, 1}},
PositiveBuckets: []float64{1, 3, -3, 42},
CustomBounds: []float64{1, 2, 3},
CustomValues: []float64{1, 2, 3},
},
expected: &FloatHistogram{
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{0, 1}},
PositiveBuckets: []float64{1, 3, -3, 42},
CustomBounds: []float64{1, 2, 3},
CustomValues: []float64{1, 2, 3},
},
},
}
@ -417,7 +417,7 @@ func TestFloatHistogramDiv(t *testing.T) {
Sum: 2349209.324,
PositiveSpans: []Span{{0, 1}, {2, 3}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1},
CustomBounds: []float64{1, 2, 3},
CustomValues: []float64{1, 2, 3},
},
1,
&FloatHistogram{
@ -426,7 +426,7 @@ func TestFloatHistogramDiv(t *testing.T) {
Sum: 2349209.324,
PositiveSpans: []Span{{0, 1}, {2, 3}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1},
CustomBounds: []float64{1, 2, 3},
CustomValues: []float64{1, 2, 3},
},
},
{
@ -437,7 +437,7 @@ func TestFloatHistogramDiv(t *testing.T) {
Sum: 23,
PositiveSpans: []Span{{0, 2}, {1, 3}},
PositiveBuckets: []float64{1, 0, 3, 4, 7},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
2,
&FloatHistogram{
@ -446,7 +446,7 @@ func TestFloatHistogramDiv(t *testing.T) {
Sum: 11.5,
PositiveSpans: []Span{{0, 2}, {1, 3}},
PositiveBuckets: []float64{0.5, 0, 1.5, 2, 3.5},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
},
}
@ -1051,7 +1051,7 @@ func TestFloatHistogramDetectReset(t *testing.T) {
"no buckets to some buckets with custom bounds",
&FloatHistogram{
Schema: CustomBucketsSchema,
CustomBounds: []float64{1, 2, 3},
CustomValues: []float64{1, 2, 3},
},
&FloatHistogram{
Schema: CustomBucketsSchema,
@ -1059,7 +1059,7 @@ func TestFloatHistogramDetectReset(t *testing.T) {
Sum: 2349209.324,
PositiveSpans: []Span{{0, 1}, {2, 3}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1},
CustomBounds: []float64{1, 2, 3},
CustomValues: []float64{1, 2, 3},
},
false,
},
@ -1071,11 +1071,11 @@ func TestFloatHistogramDetectReset(t *testing.T) {
Sum: 2349209.324,
PositiveSpans: []Span{{0, 1}, {2, 3}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1},
CustomBounds: []float64{1, 2, 3},
CustomValues: []float64{1, 2, 3},
},
&FloatHistogram{
Schema: CustomBucketsSchema,
CustomBounds: []float64{1, 2, 3},
CustomValues: []float64{1, 2, 3},
},
true,
},
@ -1087,7 +1087,7 @@ func TestFloatHistogramDetectReset(t *testing.T) {
Sum: 2349209.324,
PositiveSpans: []Span{{0, 1}, {2, 3}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
&FloatHistogram{
Schema: CustomBucketsSchema,
@ -1095,7 +1095,7 @@ func TestFloatHistogramDetectReset(t *testing.T) {
Sum: 2349209.324,
PositiveSpans: []Span{{0, 2}, {1, 3}},
PositiveBuckets: []float64{1, 1.23, 3.3, 4.2, 0.1},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
false,
},
@ -1107,7 +1107,7 @@ func TestFloatHistogramDetectReset(t *testing.T) {
Sum: 2349209.324,
PositiveSpans: []Span{{0, 2}, {1, 3}},
PositiveBuckets: []float64{1, 1.23, 3.3, 4.2, 0.1},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
&FloatHistogram{
Schema: CustomBucketsSchema,
@ -1115,7 +1115,7 @@ func TestFloatHistogramDetectReset(t *testing.T) {
Sum: 2349209.324,
PositiveSpans: []Span{{0, 1}, {2, 3}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
true,
},
@ -1127,7 +1127,7 @@ func TestFloatHistogramDetectReset(t *testing.T) {
Sum: 2349209.324,
PositiveSpans: []Span{{0, 2}, {1, 3}},
PositiveBuckets: []float64{1, 0, 3.3, 4.2, 0.1},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
&FloatHistogram{
Schema: CustomBucketsSchema,
@ -1135,7 +1135,7 @@ func TestFloatHistogramDetectReset(t *testing.T) {
Sum: 2349209.324,
PositiveSpans: []Span{{0, 1}, {2, 3}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
false,
},
@ -1147,7 +1147,7 @@ func TestFloatHistogramDetectReset(t *testing.T) {
Sum: 2349209.324,
PositiveSpans: []Span{{0, 2}, {1, 3}},
PositiveBuckets: []float64{1, 0, 3.3, 4.2, 0.1},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
&FloatHistogram{
Schema: CustomBucketsSchema,
@ -1155,7 +1155,7 @@ func TestFloatHistogramDetectReset(t *testing.T) {
Sum: 2349209.324,
PositiveSpans: []Span{{0, 2}, {1, 3}},
PositiveBuckets: []float64{1, 0, 3.3, 4.3, 0.1},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
false,
},
@ -1167,7 +1167,7 @@ func TestFloatHistogramDetectReset(t *testing.T) {
Sum: 2349209.324,
PositiveSpans: []Span{{0, 2}, {1, 3}},
PositiveBuckets: []float64{1, 0, 3.3, 4.2, 0.1},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
&FloatHistogram{
Schema: CustomBucketsSchema,
@ -1175,7 +1175,7 @@ func TestFloatHistogramDetectReset(t *testing.T) {
Sum: 2349209.324,
PositiveSpans: []Span{{0, 2}, {1, 3}},
PositiveBuckets: []float64{1, 0, 3.3, 4.1, 0.1},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
true,
},
@ -1478,14 +1478,14 @@ func TestFloatHistogramCompact(t *testing.T) {
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{0, 1}, {2, 3}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1},
CustomBounds: []float64{1, 2, 3},
CustomValues: []float64{1, 2, 3},
},
0,
&FloatHistogram{
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{0, 1}, {2, 3}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1},
CustomBounds: []float64{1, 2, 3},
CustomValues: []float64{1, 2, 3},
},
},
{
@ -1494,14 +1494,14 @@ func TestFloatHistogramCompact(t *testing.T) {
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{0, 1}, {0, 3}, {0, 1}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
0,
&FloatHistogram{
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{0, 5}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
},
{
@ -1510,14 +1510,14 @@ func TestFloatHistogramCompact(t *testing.T) {
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{0, 2}, {2, 0}, {2, 0}, {2, 0}, {3, 3}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
0,
&FloatHistogram{
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{0, 2}, {9, 3}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
},
{
@ -1526,14 +1526,14 @@ func TestFloatHistogramCompact(t *testing.T) {
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{0, 4}, {5, 6}},
PositiveBuckets: []float64{0, 0, 1, 3.3, 4.2, 0.1, 3.3, 0, 0, 0},
CustomBounds: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9},
CustomValues: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9},
},
0,
&FloatHistogram{
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{2, 2}, {5, 3}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3},
CustomBounds: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9},
CustomValues: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9},
},
},
}
@ -2009,7 +2009,7 @@ func TestFloatHistogramAdd(t *testing.T) {
Sum: 2.345,
PositiveSpans: []Span{{0, 2}, {1, 3}},
PositiveBuckets: []float64{1, 0, 3, 4, 7},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
&FloatHistogram{
Schema: CustomBucketsSchema,
@ -2017,7 +2017,7 @@ func TestFloatHistogramAdd(t *testing.T) {
Sum: 1.234,
PositiveSpans: []Span{{0, 2}, {1, 3}},
PositiveBuckets: []float64{0, 0, 2, 3, 6},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
&FloatHistogram{
Schema: CustomBucketsSchema,
@ -2025,7 +2025,7 @@ func TestFloatHistogramAdd(t *testing.T) {
Sum: 3.579,
PositiveSpans: []Span{{0, 2}, {1, 3}},
PositiveBuckets: []float64{1, 0, 5, 7, 13},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
"",
},
@ -2037,7 +2037,7 @@ func TestFloatHistogramAdd(t *testing.T) {
Sum: 2.345,
PositiveSpans: []Span{{0, 2}, {1, 1}, {0, 2}},
PositiveBuckets: []float64{1, 0, 3, 4, 7},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
&FloatHistogram{
Schema: CustomBucketsSchema,
@ -2045,7 +2045,7 @@ func TestFloatHistogramAdd(t *testing.T) {
Sum: 1.234,
PositiveSpans: []Span{{0, 2}, {1, 2}, {0, 1}},
PositiveBuckets: []float64{0, 0, 2, 3, 6},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
&FloatHistogram{
Schema: CustomBucketsSchema,
@ -2053,7 +2053,7 @@ func TestFloatHistogramAdd(t *testing.T) {
Sum: 3.579,
PositiveSpans: []Span{{0, 2}, {1, 1}, {0, 2}},
PositiveBuckets: []float64{1, 0, 5, 7, 13},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
"",
},
@ -2065,7 +2065,7 @@ func TestFloatHistogramAdd(t *testing.T) {
Sum: 2.345,
PositiveSpans: []Span{{0, 2}, {2, 3}},
PositiveBuckets: []float64{1, 0, 3, 4, 7},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
&FloatHistogram{
Schema: CustomBucketsSchema,
@ -2073,7 +2073,7 @@ func TestFloatHistogramAdd(t *testing.T) {
Sum: 1.234,
PositiveSpans: []Span{{2, 2}, {3, 3}},
PositiveBuckets: []float64{5, 4, 2, 3, 6},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
&FloatHistogram{
Schema: CustomBucketsSchema,
@ -2081,7 +2081,7 @@ func TestFloatHistogramAdd(t *testing.T) {
Sum: 3.579,
PositiveSpans: []Span{{0, 4}, {0, 6}},
PositiveBuckets: []float64{1, 0, 5, 4, 3, 4, 7, 2, 3, 6},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
"",
},
@ -2093,7 +2093,7 @@ func TestFloatHistogramAdd(t *testing.T) {
Sum: 1.234,
PositiveSpans: []Span{{2, 2}, {3, 3}},
PositiveBuckets: []float64{5, 4, 2, 3, 6},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
&FloatHistogram{
Schema: CustomBucketsSchema,
@ -2101,7 +2101,7 @@ func TestFloatHistogramAdd(t *testing.T) {
Sum: 2.345,
PositiveSpans: []Span{{0, 2}, {2, 3}},
PositiveBuckets: []float64{1, 0, 3, 4, 7},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
&FloatHistogram{
Schema: CustomBucketsSchema,
@ -2109,7 +2109,7 @@ func TestFloatHistogramAdd(t *testing.T) {
Sum: 3.579,
PositiveSpans: []Span{{0, 4}, {0, 6}},
PositiveBuckets: []float64{1, 0, 5, 4, 3, 4, 7, 2, 3, 6},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
"",
},
@ -2121,7 +2121,7 @@ func TestFloatHistogramAdd(t *testing.T) {
Sum: 2.345,
PositiveSpans: []Span{{0, 2}, {2, 3}},
PositiveBuckets: []float64{1, 0, 3, 4, 7},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
&FloatHistogram{
Schema: CustomBucketsSchema,
@ -2129,7 +2129,7 @@ func TestFloatHistogramAdd(t *testing.T) {
Sum: 1.234,
PositiveSpans: []Span{{1, 4}, {0, 3}},
PositiveBuckets: []float64{5, 4, 2, 3, 6, 2, 5},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
&FloatHistogram{
Schema: CustomBucketsSchema,
@ -2137,7 +2137,7 @@ func TestFloatHistogramAdd(t *testing.T) {
Sum: 3.579,
PositiveSpans: []Span{{0, 4}, {0, 4}},
PositiveBuckets: []float64{1, 5, 4, 2, 6, 10, 9, 5},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
"",
},
@ -2149,7 +2149,7 @@ func TestFloatHistogramAdd(t *testing.T) {
Sum: 1.234,
PositiveSpans: []Span{{1, 4}, {0, 3}},
PositiveBuckets: []float64{5, 4, 2, 3, 6, 2, 5},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
&FloatHistogram{
Schema: CustomBucketsSchema,
@ -2157,7 +2157,7 @@ func TestFloatHistogramAdd(t *testing.T) {
Sum: 2.345,
PositiveSpans: []Span{{0, 2}, {2, 3}},
PositiveBuckets: []float64{1, 0, 3, 4, 7},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
&FloatHistogram{
Schema: CustomBucketsSchema,
@ -2165,7 +2165,7 @@ func TestFloatHistogramAdd(t *testing.T) {
Sum: 3.579,
PositiveSpans: []Span{{0, 4}, {0, 4}},
PositiveBuckets: []float64{1, 5, 4, 2, 6, 10, 9, 5},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
"",
},
@ -2177,7 +2177,7 @@ func TestFloatHistogramAdd(t *testing.T) {
Sum: 2.345,
PositiveSpans: []Span{{0, 2}, {1, 3}},
PositiveBuckets: []float64{1, 0, 3, 4, 7},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
&FloatHistogram{
Schema: CustomBucketsSchema,
@ -2185,7 +2185,7 @@ func TestFloatHistogramAdd(t *testing.T) {
Sum: 1.234,
PositiveSpans: []Span{{0, 2}, {1, 3}},
PositiveBuckets: []float64{0, 0, 2, 3, 6},
CustomBounds: []float64{1, 2, 3, 4, 5},
CustomValues: []float64{1, 2, 3, 4, 5},
},
nil,
"cannot apply this operation on custom buckets histograms with different custom bounds",
@ -2209,7 +2209,7 @@ func TestFloatHistogramAdd(t *testing.T) {
Sum: 12,
PositiveSpans: []Span{{0, 2}, {1, 3}},
PositiveBuckets: []float64{0, 0, 2, 3, 6},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
nil,
"cannot apply this operation on histograms with a mix of exponential and custom bucket schemas",
@ -2342,7 +2342,7 @@ func TestFloatHistogramSub(t *testing.T) {
Sum: 23,
PositiveSpans: []Span{{0, 2}, {1, 3}},
PositiveBuckets: []float64{1, 0, 3, 4, 7},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
&FloatHistogram{
Schema: CustomBucketsSchema,
@ -2350,7 +2350,7 @@ func TestFloatHistogramSub(t *testing.T) {
Sum: 12,
PositiveSpans: []Span{{0, 2}, {1, 3}},
PositiveBuckets: []float64{0, 0, 2, 3, 6},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
&FloatHistogram{
Schema: CustomBucketsSchema,
@ -2358,7 +2358,7 @@ func TestFloatHistogramSub(t *testing.T) {
Sum: 11,
PositiveSpans: []Span{{0, 2}, {1, 3}},
PositiveBuckets: []float64{1, 0, 1, 1, 1},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
"",
},
@ -2370,7 +2370,7 @@ func TestFloatHistogramSub(t *testing.T) {
Sum: 23,
PositiveSpans: []Span{{0, 2}, {1, 3}},
PositiveBuckets: []float64{1, 0, 3, 4, 7},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
&FloatHistogram{
Schema: CustomBucketsSchema,
@ -2378,7 +2378,7 @@ func TestFloatHistogramSub(t *testing.T) {
Sum: 12,
PositiveSpans: []Span{{0, 2}, {1, 3}},
PositiveBuckets: []float64{0, 0, 2, 3, 6},
CustomBounds: []float64{1, 2, 3, 4, 5},
CustomValues: []float64{1, 2, 3, 4, 5},
},
nil,
"cannot apply this operation on custom buckets histograms with different custom bounds",
@ -2402,7 +2402,7 @@ func TestFloatHistogramSub(t *testing.T) {
Sum: 12,
PositiveSpans: []Span{{0, 2}, {1, 3}},
PositiveBuckets: []float64{0, 0, 2, 3, 6},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
nil,
"cannot apply this operation on histograms with a mix of exponential and custom bucket schemas",
@ -2521,7 +2521,7 @@ func TestFloatHistogramCopyToSchema(t *testing.T) {
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{0, 3}, {5, 5}},
PositiveBuckets: []float64{1, 0, 0, 3, 2, 2, 3, 4},
CustomBounds: []float64{1, 2, 3, 4, 5, 6, 7},
CustomValues: []float64{1, 2, 3, 4, 5, 6, 7},
},
&FloatHistogram{
Count: 30,
@ -2529,7 +2529,7 @@ func TestFloatHistogramCopyToSchema(t *testing.T) {
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{0, 3}, {5, 5}},
PositiveBuckets: []float64{1, 0, 0, 3, 2, 2, 3, 4},
CustomBounds: []float64{1, 2, 3, 4, 5, 6, 7},
CustomValues: []float64{1, 2, 3, 4, 5, 6, 7},
},
},
}
@ -3120,7 +3120,7 @@ func TestFloatCustomBucketsIterators(t *testing.T) {
{Offset: 1, Length: 1},
},
PositiveBuckets: []float64{100, 344, 123, 55},
CustomBounds: []float64{10, 25, 50, 100, 500},
CustomValues: []float64{10, 25, 50, 100, 500},
},
expPositiveBuckets: []Bucket[float64]{
{Lower: math.Inf(-1), Upper: 10, LowerInclusive: true, UpperInclusive: true, Count: 100, Index: 0},
@ -3140,7 +3140,7 @@ func TestFloatCustomBucketsIterators(t *testing.T) {
{Offset: 1, Length: 1},
},
PositiveBuckets: []float64{100, 344, 123, 55},
CustomBounds: []float64{-10, -5, 0, 10, 25},
CustomValues: []float64{-10, -5, 0, 10, 25},
},
expPositiveBuckets: []Bucket[float64]{
{Lower: math.Inf(-1), Upper: -10, LowerInclusive: true, UpperInclusive: true, Count: 100, Index: 0},
@ -3259,7 +3259,7 @@ func TestFloatHistogramEquals(t *testing.T) {
// Custom bounds are defined for exponential schema.
hCustom := h1.Copy()
hCustom.CustomBounds = []float64{1, 2, 3}
hCustom.CustomValues = []float64{1, 2, 3}
equals(h1, *hCustom)
cbh1 := FloatHistogram{
@ -3268,7 +3268,7 @@ func TestFloatHistogramEquals(t *testing.T) {
Sum: 9.7,
PositiveSpans: []Span{{0, 1}},
PositiveBuckets: []float64{3},
CustomBounds: []float64{1, 2, 3},
CustomValues: []float64{1, 2, 3},
}
require.NoError(t, cbh1.Validate())
@ -3278,7 +3278,7 @@ func TestFloatHistogramEquals(t *testing.T) {
// Has different custom bounds for custom buckets schema.
cbh2 = cbh1.Copy()
cbh2.CustomBounds = []float64{1, 2, 3, 4}
cbh2.CustomValues = []float64{1, 2, 3, 4}
notEquals(cbh1, *cbh2)
// Has non-empty negative spans and buckets for custom buckets schema.
@ -3313,7 +3313,7 @@ func TestFloatHistogramSize(t *testing.T) {
PositiveBuckets: nil, // 24 bytes.
NegativeSpans: nil, // 24 bytes.
NegativeBuckets: nil, // 24 bytes.
CustomBounds: nil, // 24 bytes.
CustomValues: nil, // 24 bytes.
},
8 + 4 + 4 + 8 + 8 + 8 + 8 + 24 + 24 + 24 + 24 + 24,
},
@ -3335,7 +3335,7 @@ func TestFloatHistogramSize(t *testing.T) {
{3, 2}, // 2 * 4 bytes.
{3, 2}}, // 2 * 4 bytes.
NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000}, // 24 bytes + 4 * 8 bytes.
CustomBounds: nil, // 24 bytes.
CustomValues: nil, // 24 bytes.
},
8 + 4 + 4 + 8 + 8 + 8 + 8 + (24 + 2*4 + 2*4) + (24 + 2*4 + 2*4) + (24 + 4*8) + (24 + 4*8) + 24,
},
@ -3355,7 +3355,7 @@ func TestFloatHistogramSize(t *testing.T) {
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1}, // 24 bytes + 4 * 8 bytes.
NegativeSpans: nil, // 24 bytes.
NegativeBuckets: nil, // 24 bytes.
CustomBounds: []float64{1, 2, 3}, // 24 bytes + 3 * 8 bytes.
CustomValues: []float64{1, 2, 3}, // 24 bytes + 3 * 8 bytes.
},
8 + 4 + 4 + 8 + 8 + 8 + 8 + (24 + 2*4 + 2*4) + (24 + 4*8) + 24 + 24 + (24 + 3*8),
},
@ -3406,7 +3406,7 @@ func TestFloatHistogramString(t *testing.T) {
{2, 4},
},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 5},
CustomBounds: []float64{1, 2, 5, 10, 15, 20},
CustomValues: []float64{1, 2, 5, 10, 15, 20},
},
`{count:3493.3, sum:2.349209324e+06, [-Inf,1]:1, (5,10]:3.3, (10,15]:4.2, (15,20]:0.1, (20,+Inf]:5}`,
},

18
model/histogram/generic.go
View file

@ -23,7 +23,7 @@ import (
const (
ExponentialSchemaMax int32 = 8
ExponentialSchemaMin int32 = -4
CustomBucketsSchema int32 = 127
CustomBucketsSchema int32 = -53
)
var (
@ -134,7 +134,7 @@ type baseBucketIterator[BC BucketCount, IBC InternalBucketCount] struct {
currCount IBC // Count in the current bucket.
currIdx int32 // The actual bucket index.
customBounds []float64 // Bounds (usually upper) for histograms with custom buckets.
customValues []float64 // Bounds (usually upper) for histograms with custom buckets.
}
func (b *baseBucketIterator[BC, IBC]) At() Bucket[BC] {
@ -148,11 +148,11 @@ func (b *baseBucketIterator[BC, IBC]) at(schema int32) Bucket[BC] {
Index: b.currIdx,
}
if b.positive {
bucket.Upper = getBound(b.currIdx, schema, b.customBounds)
bucket.Lower = getBound(b.currIdx-1, schema, b.customBounds)
bucket.Upper = getBound(b.currIdx, schema, b.customValues)
bucket.Lower = getBound(b.currIdx-1, schema, b.customValues)
} else {
bucket.Lower = -getBound(b.currIdx, schema, b.customBounds)
bucket.Upper = -getBound(b.currIdx-1, schema, b.customBounds)
bucket.Lower = -getBound(b.currIdx, schema, b.customValues)
bucket.Upper = -getBound(b.currIdx-1, schema, b.customValues)
}
if IsCustomBucketsSchema(schema) {
bucket.LowerInclusive = b.currIdx == 0
@ -446,9 +446,9 @@ func checkHistogramCustomBounds(bounds []float64, spans []Span, numBuckets int)
return nil
}
func getBound(idx, schema int32, customBounds []float64) float64 {
func getBound(idx, schema int32, customValues []float64) float64 {
if IsCustomBucketsSchema(schema) {
length := int32(len(customBounds))
length := int32(len(customValues))
switch {
case idx > length || idx < -1:
panic(fmt.Errorf("index %d out of bounds for custom bounds of length %d", idx, length))
@ -457,7 +457,7 @@ func getBound(idx, schema int32, customBounds []float64) float64 {
case idx == -1:
return math.Inf(-1)
default:
return customBounds[idx]
return customValues[idx]
}
}
return getBoundExponential(idx, schema)

42
model/histogram/histogram.go
View file

@ -54,8 +54,8 @@ type Histogram struct {
// They are all for base-2 bucket schemas, where 1 is a bucket boundary in
// each case, and then each power of two is divided into 2^n logarithmic buckets.
// Or in other words, each bucket boundary is the previous boundary times
// 2^(2^-n). Another valid schema number is 127 for custom buckets, defined by
// the CustomBounds field.
// 2^(2^-n). Another valid schema number is -53 for custom buckets, defined by
// the CustomValues field.
Schema int32
// Width of the zero bucket.
ZeroThreshold float64
@ -74,9 +74,9 @@ type Histogram struct {
// Holds the custom (usually upper) bounds for bucket definitions, otherwise nil.
// This slice is interned, to be treated as immutable and copied by reference.
// These numbers should be strictly increasing. This field is only used when the
// schema is 127, and the ZeroThreshold, ZeroCount, NegativeSpans and NegativeBuckets
// fields are not used.
CustomBounds []float64
// schema is for custom buckets, and the ZeroThreshold, ZeroCount, NegativeSpans
// and NegativeBuckets fields are not used.
CustomValues []float64
}
// A Span defines a continuous sequence of buckets.
@ -102,7 +102,10 @@ func (h *Histogram) Copy() *Histogram {
}
if h.UsesCustomBuckets() {
c.CustomBounds = h.CustomBounds
if len(h.CustomValues) != 0 {
c.CustomValues = make([]float64, len(h.CustomValues))
copy(c.CustomValues, h.CustomValues)
}
} else {
c.ZeroThreshold = h.ZeroThreshold
c.ZeroCount = h.ZeroCount
@ -144,7 +147,8 @@ func (h *Histogram) CopyTo(to *Histogram) {
to.NegativeSpans = clearIfNotNil(to.NegativeSpans)
to.NegativeBuckets = clearIfNotNil(to.NegativeBuckets)
to.CustomBounds = h.CustomBounds
to.CustomValues = resize(to.CustomValues, len(h.CustomValues))
copy(to.CustomValues, h.CustomValues)
} else {
to.ZeroThreshold = h.ZeroThreshold
to.ZeroCount = h.ZeroCount
@ -155,7 +159,7 @@ func (h *Histogram) CopyTo(to *Histogram) {
to.NegativeBuckets = resize(to.NegativeBuckets, len(h.NegativeBuckets))
copy(to.NegativeBuckets, h.NegativeBuckets)
to.CustomBounds = clearIfNotNil(to.CustomBounds)
to.CustomValues = clearIfNotNil(to.CustomValues)
}
to.PositiveSpans = resize(to.PositiveSpans, len(h.PositiveSpans))
@ -213,7 +217,7 @@ func (h *Histogram) ZeroBucket() Bucket[uint64] {
// PositiveBucketIterator returns a BucketIterator to iterate over all positive
// buckets in ascending order (starting next to the zero bucket and going up).
func (h *Histogram) PositiveBucketIterator() BucketIterator[uint64] {
it := newRegularBucketIterator(h.PositiveSpans, h.PositiveBuckets, h.Schema, true, h.CustomBounds)
it := newRegularBucketIterator(h.PositiveSpans, h.PositiveBuckets, h.Schema, true, h.CustomValues)
return &it
}
@ -255,7 +259,7 @@ func (h *Histogram) Equals(h2 *Histogram) bool {
}
if h.UsesCustomBuckets() {
if !floatBucketsMatch(h.CustomBounds, h2.CustomBounds) {
if !FloatBucketsMatch(h.CustomValues, h2.CustomValues) {
return false
}
}
@ -375,7 +379,9 @@ func (h *Histogram) ToFloat(fh *FloatHistogram) *FloatHistogram {
fh.ZeroCount = 0
fh.NegativeSpans = clearIfNotNil(fh.NegativeSpans)
fh.NegativeBuckets = clearIfNotNil(fh.NegativeBuckets)
fh.CustomBounds = h.CustomBounds
fh.CustomValues = resize(fh.CustomValues, len(h.CustomValues))
copy(fh.CustomValues, h.CustomValues)
} else {
fh.ZeroThreshold = h.ZeroThreshold
fh.ZeroCount = float64(h.ZeroCount)
@ -389,7 +395,7 @@ func (h *Histogram) ToFloat(fh *FloatHistogram) *FloatHistogram {
currentNegative += float64(b)
fh.NegativeBuckets[i] = currentNegative
}
fh.CustomBounds = clearIfNotNil(fh.CustomBounds)
fh.CustomValues = clearIfNotNil(fh.CustomValues)
}
fh.PositiveSpans = resize(fh.PositiveSpans, len(h.PositiveSpans))
@ -430,7 +436,7 @@ func clearIfNotNil[T any](items []T) []T {
func (h *Histogram) Validate() error {
var nCount, pCount uint64
if h.UsesCustomBuckets() {
if err := checkHistogramCustomBounds(h.CustomBounds, h.PositiveSpans, len(h.PositiveBuckets)); err != nil {
if err := checkHistogramCustomBounds(h.CustomValues, h.PositiveSpans, len(h.PositiveBuckets)); err != nil {
return fmt.Errorf("custom buckets: %w", err)
}
if h.ZeroCount != 0 {
@ -456,7 +462,7 @@ func (h *Histogram) Validate() error {
if err != nil {
return fmt.Errorf("negative side: %w", err)
}
if h.CustomBounds != nil {
if h.CustomValues != nil {
return fmt.Errorf("histogram with exponential schema must not have custom bounds")
}
}
@ -483,13 +489,13 @@ type regularBucketIterator struct {
baseBucketIterator[uint64, int64]
}
func newRegularBucketIterator(spans []Span, buckets []int64, schema int32, positive bool, customBounds []float64) regularBucketIterator {
func newRegularBucketIterator(spans []Span, buckets []int64, schema int32, positive bool, customValues []float64) regularBucketIterator {
i := baseBucketIterator[uint64, int64]{
schema: schema,
spans: spans,
buckets: buckets,
positive: positive,
customBounds: customBounds,
customValues: customValues,
}
return regularBucketIterator{i}
}
@ -563,7 +569,7 @@ func (c *cumulativeBucketIterator) Next() bool {
if c.emptyBucketCount > 0 {
// We are traversing through empty buckets at the moment.
c.currUpper = getBound(c.currIdx, c.h.Schema, c.h.CustomBounds)
c.currUpper = getBound(c.currIdx, c.h.Schema, c.h.CustomValues)
c.currIdx++
c.emptyBucketCount--
return true
@ -580,7 +586,7 @@ func (c *cumulativeBucketIterator) Next() bool {
c.currCount += c.h.PositiveBuckets[c.posBucketsIdx]
c.currCumulativeCount += uint64(c.currCount)
c.currUpper = getBound(c.currIdx, c.h.Schema, c.h.CustomBounds)
c.currUpper = getBound(c.currIdx, c.h.Schema, c.h.CustomValues)
c.posBucketsIdx++
c.idxInSpan++

62
model/histogram/histogram_test.go
View file

@ -80,7 +80,7 @@ func TestHistogramString(t *testing.T) {
{Offset: 0, Length: 3},
},
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 0},
CustomBounds: []float64{1, 2, 5, 10, 15, 20, 25, 50},
CustomValues: []float64{1, 2, 5, 10, 15, 20, 25, 50},
},
expectedString: "{count:19, sum:2.7, [-Inf,1]:1, (1,2]:3, (2,5]:1, (5,10]:2, (10,15]:1, (15,20]:1, (20,25]:1}",
},
@ -231,7 +231,7 @@ func TestCumulativeBucketIterator(t *testing.T) {
{Offset: 1, Length: 2},
},
PositiveBuckets: []int64{1, 1, -1, 0},
CustomBounds: []float64{5, 10, 20, 50},
CustomValues: []float64{5, 10, 20, 50},
},
expectedBuckets: []Bucket[uint64]{
{Lower: math.Inf(-1), Upper: 5, Count: 1, LowerInclusive: true, UpperInclusive: true, Index: 0},
@ -411,7 +411,7 @@ func TestRegularBucketIterator(t *testing.T) {
{Offset: 1, Length: 2},
},
PositiveBuckets: []int64{1, 1, -1, 0},
CustomBounds: []float64{5, 10, 20, 50},
CustomValues: []float64{5, 10, 20, 50},
},
expectedPositiveBuckets: []Bucket[uint64]{
{Lower: math.Inf(-1), Upper: 5, Count: 1, LowerInclusive: true, UpperInclusive: true, Index: 0},
@ -430,7 +430,7 @@ func TestRegularBucketIterator(t *testing.T) {
{Offset: 1, Length: 2},
},
PositiveBuckets: []int64{1, 1, -1, 0},
CustomBounds: []float64{0, 10, 20, 50},
CustomValues: []float64{0, 10, 20, 50},
},
expectedPositiveBuckets: []Bucket[uint64]{
{Lower: math.Inf(-1), Upper: 0, Count: 1, LowerInclusive: true, UpperInclusive: true, Index: 0},
@ -448,7 +448,7 @@ func TestRegularBucketIterator(t *testing.T) {
{Offset: 0, Length: 5},
},
PositiveBuckets: []int64{1, 1, 0, -1, 0},
CustomBounds: []float64{-5, 0, 20, 50},
CustomValues: []float64{-5, 0, 20, 50},
},
expectedPositiveBuckets: []Bucket[uint64]{
{Lower: math.Inf(-1), Upper: -5, Count: 1, LowerInclusive: true, UpperInclusive: true, Index: 0},
@ -563,7 +563,7 @@ func TestCustomBucketsHistogramToFloat(t *testing.T) {
{Offset: 0, Length: 3},
},
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 0},
CustomBounds: []float64{5, 10, 20, 50, 100, 500},
CustomValues: []float64{5, 10, 20, 50, 100, 500},
}
cases := []struct {
name string
@ -773,7 +773,7 @@ func TestHistogramEquals(t *testing.T) {
// Has non-empty custom bounds for exponential schema.
hCustom := h1.Copy()
hCustom.CustomBounds = []float64{1, 2, 3}
hCustom.CustomValues = []float64{1, 2, 3}
equals(h1, *hCustom)
cbh1 := Histogram{
@ -785,7 +785,7 @@ func TestHistogramEquals(t *testing.T) {
{Offset: 10, Length: 3},
},
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 0},
CustomBounds: []float64{0.1, 0.2, 0.5, 1, 2, 5, 10, 15, 20, 25, 50, 75, 100, 200, 250, 500, 1000},
CustomValues: []float64{0.1, 0.2, 0.5, 1, 2, 5, 10, 15, 20, 25, 50, 75, 100, 200, 250, 500, 1000},
}
require.NoError(t, cbh1.Validate())
@ -795,7 +795,7 @@ func TestHistogramEquals(t *testing.T) {
// Has different custom bounds for custom buckets schema.
cbh2 = cbh1.Copy()
cbh2.CustomBounds = []float64{0.1, 0.2, 0.5}
cbh2.CustomValues = []float64{0.1, 0.2, 0.5}
notEquals(cbh1, *cbh2)
// Has non-empty negative spans and buckets for custom buckets schema.
@ -853,13 +853,13 @@ func TestHistogramCopy(t *testing.T) {
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{-2, 1}},
PositiveBuckets: []int64{1, 3, -3, 42},
CustomBounds: []float64{5, 10, 15},
CustomValues: []float64{5, 10, 15},
},
expected: &Histogram{
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{-2, 1}},
PositiveBuckets: []int64{1, 3, -3, 42},
CustomBounds: []float64{5, 10, 15},
CustomValues: []float64{5, 10, 15},
},
},
}
@ -918,13 +918,13 @@ func TestHistogramCopyTo(t *testing.T) {
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{-2, 1}},
PositiveBuckets: []int64{1, 3, -3, 42},
CustomBounds: []float64{5, 10, 15},
CustomValues: []float64{5, 10, 15},
},
expected: &Histogram{
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{-2, 1}},
PositiveBuckets: []int64{1, 3, -3, 42},
CustomBounds: []float64{5, 10, 15},
CustomValues: []float64{5, 10, 15},
},
},
}
@ -1214,14 +1214,14 @@ func TestHistogramCompact(t *testing.T) {
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{-2, 1}, {2, 3}},
PositiveBuckets: []int64{1, 3, -3, 42},
CustomBounds: []float64{5, 10, 15},
CustomValues: []float64{5, 10, 15},
},
0,
&Histogram{
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{-2, 1}, {2, 3}},
PositiveBuckets: []int64{1, 3, -3, 42},
CustomBounds: []float64{5, 10, 15},
CustomValues: []float64{5, 10, 15},
},
},
{
@ -1230,14 +1230,14 @@ func TestHistogramCompact(t *testing.T) {
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{-2, 1}, {0, 3}, {0, 1}},
PositiveBuckets: []int64{1, 3, -3, 42, 3},
CustomBounds: []float64{5, 10, 15, 20},
CustomValues: []float64{5, 10, 15, 20},
},
0,
&Histogram{
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{-2, 5}},
PositiveBuckets: []int64{1, 3, -3, 42, 3},
CustomBounds: []float64{5, 10, 15, 20},
CustomValues: []float64{5, 10, 15, 20},
},
},
{
@ -1246,14 +1246,14 @@ func TestHistogramCompact(t *testing.T) {
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{-2, 2}, {2, 0}, {3, 3}},
PositiveBuckets: []int64{1, 3, -3, 42, 3},
CustomBounds: []float64{5, 10, 15, 20},
CustomValues: []float64{5, 10, 15, 20},
},
0,
&Histogram{
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{-2, 2}, {5, 3}},
PositiveBuckets: []int64{1, 3, -3, 42, 3},
CustomBounds: []float64{5, 10, 15, 20},
CustomValues: []float64{5, 10, 15, 20},
},
},
{
@ -1262,14 +1262,14 @@ func TestHistogramCompact(t *testing.T) {
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{-2, 2}, {2, 0}, {2, 0}, {2, 0}, {3, 3}},
PositiveBuckets: []int64{1, 3, -3, 42, 3},
CustomBounds: []float64{5, 10, 15, 20},
CustomValues: []float64{5, 10, 15, 20},
},
0,
&Histogram{
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{-2, 2}, {9, 3}},
PositiveBuckets: []int64{1, 3, -3, 42, 3},
CustomBounds: []float64{5, 10, 15, 20},
CustomValues: []float64{5, 10, 15, 20},
},
},
{
@ -1278,14 +1278,14 @@ func TestHistogramCompact(t *testing.T) {
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{-4, 6}, {3, 6}},
PositiveBuckets: []int64{0, 0, 1, 3, -4, 0, 1, 42, 3, -46, 0, 0},
CustomBounds: []float64{5, 10, 15, 20},
CustomValues: []float64{5, 10, 15, 20},
},
0,
&Histogram{
Schema: CustomBucketsSchema,
PositiveSpans: []Span{{-2, 2}, {5, 3}},
PositiveBuckets: []int64{1, 3, -3, 42, 3},
CustomBounds: []float64{5, 10, 15, 20},
CustomValues: []float64{5, 10, 15, 20},
},
},
}
@ -1454,7 +1454,7 @@ func TestHistogramValidation(t *testing.T) {
{Offset: 1, Length: 2},
},
PositiveBuckets: []int64{1, 1, -1, 0},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
errMsg: `histogram with exponential schema must not have custom bounds`,
skipFloat: true, // Converting to float will remove the wrong fields so only the float version will pass validation
@ -1476,7 +1476,7 @@ func TestHistogramValidation(t *testing.T) {
{Offset: 1, Length: 2},
},
NegativeBuckets: []int64{1, 1, -1, 0},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
errMsg: `custom buckets: must have zero count of 0`,
skipFloat: true, // Converting to float will remove the wrong fields so only the float version will pass validation
@ -1491,7 +1491,7 @@ func TestHistogramValidation(t *testing.T) {
{Offset: 1, Length: 2},
},
PositiveBuckets: []int64{1, 1, -1, 0},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
errMsg: `custom buckets: span number 1 with offset -1: histogram has a span whose offset is negative`,
},
@ -1505,7 +1505,7 @@ func TestHistogramValidation(t *testing.T) {
{Offset: -1, Length: 2},
},
PositiveBuckets: []int64{1, 1, -1, 0},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
errMsg: `custom buckets: span number 2 with offset -1: histogram has a span whose offset is negative`,
},
@ -1519,7 +1519,7 @@ func TestHistogramValidation(t *testing.T) {
{Offset: 1, Length: 2},
},
PositiveBuckets: []int64{1, 1, -1},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
errMsg: `custom buckets: spans need 4 buckets, have 3 buckets: histogram spans specify different number of buckets than provided`,
},
@ -1533,7 +1533,7 @@ func TestHistogramValidation(t *testing.T) {
{Offset: 1, Length: 2},
},
PositiveBuckets: []int64{1, 1, -1, 0},
CustomBounds: []float64{1, 2, 3},
CustomValues: []float64{1, 2, 3},
},
errMsg: `custom buckets: only 3 custom bounds defined which is insufficient to cover total span length of 5: histogram custom bounds are too few`,
},
@ -1547,7 +1547,7 @@ func TestHistogramValidation(t *testing.T) {
{Offset: 1, Length: 2},
},
PositiveBuckets: []int64{1, 1, -1, 0},
CustomBounds: []float64{1, 2, 3, 4},
CustomValues: []float64{1, 2, 3, 4},
},
},
"valid custom buckets histogram with extra bounds": {
@ -1560,7 +1560,7 @@ func TestHistogramValidation(t *testing.T) {
{Offset: 1, Length: 2},
},
PositiveBuckets: []int64{1, 1, -1, 0},
CustomBounds: []float64{1, 2, 3, 4, 5, 6, 7, 8},
CustomValues: []float64{1, 2, 3, 4, 5, 6, 7, 8},
},
},
}

4
scrape/target_test.go
View file

@ -481,7 +481,7 @@ func TestBucketLimitAppender(t *testing.T) {
{Offset: 0, Length: 3},
},
PositiveBuckets: []int64{3, 0, 0},
CustomBounds: []float64{1, 2, 3},
CustomValues: []float64{1, 2, 3},
}
cases := []struct {
@ -592,7 +592,7 @@ func TestMaxSchemaAppender(t *testing.T) {
{Offset: 0, Length: 3},
},
PositiveBuckets: []int64{3, 0, 0},
CustomBounds: []float64{1, 2, 3},
CustomValues: []float64{1, 2, 3},
}
cases := []struct {

16
tsdb/chunkenc/bstream.go
View file

@ -113,6 +113,16 @@ func (b *bstream) writeBits(u uint64, nbits int) {
}
}
// wrapper for the standard library's PutUvarint to make it work
// with our bstream.
func (b *bstream) putUvarint(x uint64) {
buf := make([]byte, 2)
l := binary.PutUvarint(buf, x)
for i := 0; i < l; i++ {
b.writeByte(buf[i])
}
}
type bstreamReader struct {
stream []byte
streamOffset int // The offset from which read the next byte from the stream.
@ -257,3 +267,9 @@ func (b *bstreamReader) loadNextBuffer(nbits uint8) bool {
return true
}
// wrapper for the standard library's ReadUvarint to make it work
// with our bstream.
func (b *bstreamReader) readUvarint() (uint64, error) {
return binary.ReadUvarint(b)
}

52
tsdb/chunkenc/float_histogram.go
View file

@ -72,6 +72,7 @@ func (c *FloatHistogramChunk) NumSamples() int {
func (c *FloatHistogramChunk) Layout() (
schema int32, zeroThreshold float64,
negativeSpans, positiveSpans []histogram.Span,
customValues []float64,
err error,
) {
if c.NumSamples() == 0 {
@ -129,17 +130,18 @@ func (c *FloatHistogramChunk) Appender() (Appender, error) {
a := &FloatHistogramAppender{
b: &c.b,
schema: it.schema,
zThreshold: it.zThreshold,
pSpans: it.pSpans,
nSpans: it.nSpans,
t: it.t,
tDelta: it.tDelta,
cnt: it.cnt,
zCnt: it.zCnt,
pBuckets: pBuckets,
nBuckets: nBuckets,
sum: it.sum,
schema: it.schema,
zThreshold: it.zThreshold,
pSpans: it.pSpans,
nSpans: it.nSpans,
customValues: it.customValues,
t: it.t,
tDelta: it.tDelta,
cnt: it.cnt,
zCnt: it.zCnt,
pBuckets: pBuckets,
nBuckets: nBuckets,
sum: it.sum,
}
if it.numTotal == 0 {
a.sum.leading = 0xff
@ -187,6 +189,7 @@ type FloatHistogramAppender struct {
schema int32
zThreshold float64
pSpans, nSpans []histogram.Span
customValues []float64
t, tDelta int64
sum, cnt, zCnt xorValue
@ -218,6 +221,7 @@ func (a *FloatHistogramAppender) Append(int64, float64) {
//
// The chunk is not appendable in the following cases:
// - The schema has changed.
// - The custom bounds have changed if the current schema is custom buckets.
// - The threshold for the zero bucket has changed.
// - Any buckets have disappeared.
// - There was a counter reset in the count of observations or in any bucket, including the zero bucket.
@ -259,6 +263,11 @@ func (a *FloatHistogramAppender) appendable(h *histogram.FloatHistogram) (
return
}
if histogram.IsCustomBucketsSchema(h.Schema) && !histogram.FloatBucketsMatch(h.CustomValues, a.customValues) {
counterReset = true
return
}
if h.ZeroCount < a.zCnt.value {
// There has been a counter reset since ZeroThreshold didn't change.
counterReset = true
@ -299,6 +308,7 @@ func (a *FloatHistogramAppender) appendable(h *histogram.FloatHistogram) (
//
// The chunk is not appendable in the following cases:
// - The schema has changed.
// - The custom bounds have changed if the current schema is custom buckets.
// - The threshold for the zero bucket has changed.
// - The last sample in the chunk was stale while the current sample is not stale.
func (a *FloatHistogramAppender) appendableGauge(h *histogram.FloatHistogram) (
@ -325,6 +335,10 @@ func (a *FloatHistogramAppender) appendableGauge(h *histogram.FloatHistogram) (
return
}
if histogram.IsCustomBucketsSchema(h.Schema) && !histogram.FloatBucketsMatch(h.CustomValues, a.customValues) {
return
}
positiveInserts, backwardPositiveInserts, positiveSpans = expandSpansBothWays(a.pSpans, h.PositiveSpans)
negativeInserts, backwardNegativeInserts, negativeSpans = expandSpansBothWays(a.nSpans, h.NegativeSpans)
okToAppend = true
@ -418,7 +432,7 @@ func (a *FloatHistogramAppender) appendFloatHistogram(t int64, h *histogram.Floa
if num == 0 {
// The first append gets the privilege to dictate the layout
// but it's also responsible for encoding it into the chunk!
writeHistogramChunkLayout(a.b, h.Schema, h.ZeroThreshold, h.PositiveSpans, h.NegativeSpans)
writeHistogramChunkLayout(a.b, h.Schema, h.ZeroThreshold, h.PositiveSpans, h.NegativeSpans, h.CustomValues)
a.schema = h.Schema
a.zThreshold = h.ZeroThreshold
@ -434,6 +448,12 @@ func (a *FloatHistogramAppender) appendFloatHistogram(t int64, h *histogram.Floa
} else {
a.nSpans = nil
}
if len(h.CustomValues) > 0 {
a.customValues = make([]float64, len(h.CustomValues))
copy(a.customValues, h.CustomValues)
} else {
a.customValues = nil
}
numPBuckets, numNBuckets := countSpans(h.PositiveSpans), countSpans(h.NegativeSpans)
if numPBuckets > 0 {
@ -689,6 +709,7 @@ type floatHistogramIterator struct {
schema int32
zThreshold float64
pSpans, nSpans []histogram.Span
customValues []float64
// For the fields that are tracked as deltas and ultimately dod's.
t int64
@ -749,6 +770,7 @@ func (it *floatHistogramIterator) AtFloatHistogram(fh *histogram.FloatHistogram)
NegativeSpans: it.nSpans,
PositiveBuckets: it.pBuckets,
NegativeBuckets: it.nBuckets,
CustomValues: it.customValues,
}
}
@ -771,6 +793,9 @@ func (it *floatHistogramIterator) AtFloatHistogram(fh *histogram.FloatHistogram)
fh.NegativeBuckets = resize(fh.NegativeBuckets, len(it.nBuckets))
copy(fh.NegativeBuckets, it.nBuckets)
fh.CustomValues = resize(fh.CustomValues, len(it.customValues))
copy(fh.CustomValues, it.customValues)
return it.t, fh
}
@ -815,7 +840,7 @@ func (it *floatHistogramIterator) Next() ValueType {
// The first read is responsible for reading the chunk layout
// and for initializing fields that depend on it. We give
// counter reset info at chunk level, hence we discard it here.
schema, zeroThreshold, posSpans, negSpans, err := readHistogramChunkLayout(&it.br)
schema, zeroThreshold, posSpans, negSpans, customValues, err := readHistogramChunkLayout(&it.br)
if err != nil {
it.err = err
return ValNone
@ -823,6 +848,7 @@ func (it *floatHistogramIterator) Next() ValueType {
it.schema = schema
it.zThreshold = zeroThreshold
it.pSpans, it.nSpans = posSpans, negSpans
it.customValues = customValues
numPBuckets, numNBuckets := countSpans(posSpans), countSpans(negSpans)
// Allocate bucket slices as needed, recycling existing slices
// in case this iterator was reset and already has slices of a

328
tsdb/chunkenc/float_histogram_test.go
View file

@ -280,7 +280,38 @@ func TestFloatHistogramChunkBucketChanges(t *testing.T) {
}
func TestFloatHistogramChunkAppendable(t *testing.T) {
setup := func() (Chunk, *FloatHistogramAppender, int64, *histogram.FloatHistogram) {
eh := &histogram.FloatHistogram{
Count: 5,
ZeroCount: 2,
Sum: 18.4,
ZeroThreshold: 1e-125,
Schema: 1,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 2},
{Offset: 2, Length: 1},
{Offset: 3, Length: 2},
{Offset: 3, Length: 1},
{Offset: 1, Length: 1},
},
PositiveBuckets: []float64{6, 3, 3, 2, 4, 5, 1},
}
cbh := &histogram.FloatHistogram{
Count: 24,
Sum: 18.4,
Schema: histogram.CustomBucketsSchema,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 2},
{Offset: 2, Length: 1},
{Offset: 3, Length: 2},
{Offset: 3, Length: 1},
{Offset: 1, Length: 1},
},
PositiveBuckets: []float64{6, 3, 3, 2, 4, 5, 1},
CustomValues: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20},
}
setup := func(h *histogram.FloatHistogram) (Chunk, *FloatHistogramAppender, int64, *histogram.FloatHistogram) {
c := Chunk(NewFloatHistogramChunk())
// Create fresh appender and add the first histogram.
@ -289,32 +320,17 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
require.Equal(t, 0, c.NumSamples())
ts := int64(1234567890)
h1 := &histogram.FloatHistogram{
Count: 5,
ZeroCount: 2,
Sum: 18.4,
ZeroThreshold: 1e-125,
Schema: 1,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 2},
{Offset: 2, Length: 1},
{Offset: 3, Length: 2},
{Offset: 3, Length: 1},
{Offset: 1, Length: 1},
},
PositiveBuckets: []float64{6, 3, 3, 2, 4, 5, 1},
}
chk, _, app, err := app.AppendFloatHistogram(nil, ts, h1.Copy(), false)
chk, _, app, err := app.AppendFloatHistogram(nil, ts, h.Copy(), false)
require.NoError(t, err)
require.Nil(t, chk)
require.Equal(t, 1, c.NumSamples())
require.Equal(t, UnknownCounterReset, c.(*FloatHistogramChunk).GetCounterResetHeader())
return c, app.(*FloatHistogramAppender), ts, h1
return c, app.(*FloatHistogramAppender), ts, h
}
{ // Schema change.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.Schema++
_, _, ok, _ := hApp.appendable(h2)
@ -324,7 +340,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
}
{ // Zero threshold change.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.ZeroThreshold += 0.1
_, _, ok, _ := hApp.appendable(h2)
@ -334,7 +350,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
}
{ // New histogram that has more buckets.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.PositiveSpans = []histogram.Span{
{Offset: 0, Length: 3},
@ -357,7 +373,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
}
{ // New histogram that has a bucket missing.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.PositiveSpans = []histogram.Span{
@ -379,7 +395,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
}
{ // New histogram that has a counter reset while buckets are same.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.Sum = 23
h2.PositiveBuckets = []float64{6, 2, 3, 2, 4, 5, 1}
@ -394,7 +410,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
}
{ // New histogram that has a counter reset while new buckets were added.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.PositiveSpans = []histogram.Span{
{Offset: 0, Length: 3},
@ -415,7 +431,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
}
{
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
// New histogram that has a counter reset while new buckets were
// added before the first bucket and reset on first bucket. (to
// catch the edge case where the new bucket should be forwarded
@ -442,7 +458,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
}
{ // New histogram that has an explicit counter reset.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.CounterResetHint = histogram.CounterReset
@ -450,7 +466,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
}
{ // Start new chunk explicitly, and append a new histogram that is considered appendable to the previous chunk.
_, hApp, ts, h1 := setup()
_, hApp, ts, h1 := setup(eh)
h2 := h1.Copy() // Identity is appendable.
nextChunk := NewFloatHistogramChunk()
@ -466,7 +482,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
}
{ // Start new chunk explicitly, and append a new histogram that is not considered appendable to the previous chunk.
_, hApp, ts, h1 := setup()
_, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.Count-- // Make this not appendable due to counter reset.
@ -483,7 +499,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
}
{ // Start new chunk explicitly, and append a new histogram that would need recoding if we added it to the chunk.
_, hApp, ts, h1 := setup()
_, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.PositiveSpans = []histogram.Span{
{Offset: 0, Length: 3},
@ -507,6 +523,72 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
assertSampleCount(t, nextChunk, 1, ValFloatHistogram)
require.Equal(t, NotCounterReset, nextChunk.GetCounterResetHeader())
}
{ // Custom buckets, no change.
c, hApp, ts, h1 := setup(cbh)
h2 := h1.Copy()
_, _, ok, _ := hApp.appendable(h2)
require.True(t, ok)
assertNoNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
}
{ // Custom buckets, increase in bucket counts but no change in layout.
c, hApp, ts, h1 := setup(cbh)
h2 := h1.Copy()
h2.Count++
h2.PositiveBuckets = []float64{6, 3, 3, 2, 4, 5, 2}
_, _, ok, _ := hApp.appendable(h2)
require.True(t, ok)
assertNoNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
}
{ // Custom buckets, decrease in bucket counts but no change in layout.
c, hApp, ts, h1 := setup(cbh)
h2 := h1.Copy()
h2.Count--
h2.PositiveBuckets = []float64{6, 3, 3, 2, 4, 5, 0}
_, _, ok, _ := hApp.appendable(h2)
require.False(t, ok)
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
}
{ // Custom buckets, change only in custom bounds.
c, hApp, ts, h1 := setup(cbh)
h2 := h1.Copy()
h2.CustomValues = []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21}
_, _, ok, _ := hApp.appendable(h2)
require.False(t, ok)
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
}
{ // Custom buckets, with more buckets.
c, hApp, ts, h1 := setup(cbh)
h2 := h1.Copy()
h2.PositiveSpans = []histogram.Span{
{Offset: 0, Length: 3},
{Offset: 1, Length: 1},
{Offset: 1, Length: 4},
{Offset: 3, Length: 3},
}
h2.Count += 6
h2.Sum = 30
// Existing histogram should get values converted from the above to:
// 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between)
// so the new histogram should have new counts >= these per-bucket counts, e.g.:
h2.PositiveBuckets = []float64{7, 5, 1, 3, 1, 0, 2, 5, 5, 0, 1} // (total 30)
posInterjections, negInterjections, ok, cr := hApp.appendable(h2)
require.NotEmpty(t, posInterjections)
require.Empty(t, negInterjections)
require.True(t, ok) // Only new buckets came in.
require.False(t, cr)
assertRecodedFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
}
}
func assertNewFloatHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *FloatHistogramAppender, ts int64, h *histogram.FloatHistogram, expectHeader CounterResetHeader) {
@ -526,7 +608,7 @@ func assertNewFloatHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *Fl
func assertNoNewFloatHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *FloatHistogramAppender, ts int64, h *histogram.FloatHistogram, expectHeader CounterResetHeader) {
oldChunkBytes := oldChunk.Bytes()
newChunk, recoded, newAppender, err := hApp.AppendFloatHistogram(nil, ts, h, false)
require.NotEqual(t, oldChunkBytes, oldChunk.Bytes()) // Sanity check that previous chunk is untouched.
require.Greater(t, len(oldChunk.Bytes()), len(oldChunkBytes)) // Check that current chunk is bigger than previously.
require.NoError(t, err)
require.Nil(t, newChunk)
require.False(t, recoded)
@ -715,6 +797,32 @@ func TestFloatHistogramChunkAppendableWithEmptySpan(t *testing.T) {
NegativeBuckets: []float64{1, 4, 2, 7, 5, 5, 2},
},
},
"empty span in old and new custom buckets histogram": {
h1: &histogram.FloatHistogram{
Schema: histogram.CustomBucketsSchema,
Count: 7,
Sum: 1234.5,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 4},
{Offset: 0, Length: 0},
{Offset: 0, Length: 3},
},
PositiveBuckets: []float64{1, 2, 1, 1, 1, 1, 1},
CustomValues: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
},
h2: &histogram.FloatHistogram{
Schema: histogram.CustomBucketsSchema,
Count: 10,
Sum: 2345.6,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 4},
{Offset: 0, Length: 0},
{Offset: 0, Length: 3},
},
PositiveBuckets: []float64{1, 3, 1, 2, 1, 1, 1},
CustomValues: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
},
},
}
for name, tc := range tests {
@ -741,7 +849,40 @@ func TestFloatHistogramChunkAppendableWithEmptySpan(t *testing.T) {
}
func TestFloatHistogramChunkAppendableGauge(t *testing.T) {
setup := func() (Chunk, *FloatHistogramAppender, int64, *histogram.FloatHistogram) {
eh := &histogram.FloatHistogram{
CounterResetHint: histogram.GaugeType,
Count: 5,
ZeroCount: 2,
Sum: 18.4,
ZeroThreshold: 1e-125,
Schema: 1,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 2},
{Offset: 2, Length: 1},
{Offset: 3, Length: 2},
{Offset: 3, Length: 1},
{Offset: 1, Length: 1},
},
PositiveBuckets: []float64{6, 3, 3, 2, 4, 5, 1},
}
cbh := &histogram.FloatHistogram{
CounterResetHint: histogram.GaugeType,
Count: 24,
Sum: 18.4,
Schema: histogram.CustomBucketsSchema,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 2},
{Offset: 2, Length: 1},
{Offset: 3, Length: 2},
{Offset: 3, Length: 1},
{Offset: 1, Length: 1},
},
PositiveBuckets: []float64{6, 3, 3, 2, 4, 5, 1},
CustomValues: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20},
}
setup := func(h *histogram.FloatHistogram) (Chunk, *FloatHistogramAppender, int64, *histogram.FloatHistogram) {
c := Chunk(NewFloatHistogramChunk())
// Create fresh appender and add the first histogram.
@ -750,33 +891,17 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) {
require.Equal(t, 0, c.NumSamples())
ts := int64(1234567890)
h1 := &histogram.FloatHistogram{
CounterResetHint: histogram.GaugeType,
Count: 5,
ZeroCount: 2,
Sum: 18.4,
ZeroThreshold: 1e-125,
Schema: 1,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 2},
{Offset: 2, Length: 1},
{Offset: 3, Length: 2},
{Offset: 3, Length: 1},
{Offset: 1, Length: 1},
},
PositiveBuckets: []float64{6, 3, 3, 2, 4, 5, 1},
}
chk, _, app, err := app.AppendFloatHistogram(nil, ts, h1.Copy(), false)
chk, _, app, err := app.AppendFloatHistogram(nil, ts, h.Copy(), false)
require.NoError(t, err)
require.Nil(t, chk)
require.Equal(t, 1, c.NumSamples())
require.Equal(t, GaugeType, c.(*FloatHistogramChunk).GetCounterResetHeader())
return c, app.(*FloatHistogramAppender), ts, h1
return c, app.(*FloatHistogramAppender), ts, h
}
{ // Schema change.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.Schema++
_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
@ -786,7 +911,7 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) {
}
{ // Zero threshold change.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.ZeroThreshold += 0.1
_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
@ -796,7 +921,7 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) {
}
{ // New histogram that has more buckets.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.PositiveSpans = []histogram.Span{
{Offset: 0, Length: 3},
@ -820,7 +945,7 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) {
}
{ // New histogram that has buckets missing.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.PositiveSpans = []histogram.Span{
{Offset: 0, Length: 2},
@ -844,7 +969,7 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) {
}
{ // New histogram that has a bucket missing and new buckets.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.PositiveSpans = []histogram.Span{
{Offset: 0, Length: 2},
@ -866,7 +991,7 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) {
}
{ // New histogram that has a counter reset while buckets are same.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.Sum = 23
h2.PositiveBuckets = []float64{6, 2, 3, 2, 4, 5, 1}
@ -882,7 +1007,7 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) {
}
{ // New histogram that has a counter reset while new buckets were added.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.PositiveSpans = []histogram.Span{
{Offset: 0, Length: 3},
@ -906,7 +1031,7 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) {
{
// New histogram that has a counter reset while new buckets were
// added before the first bucket and reset on first bucket.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.PositiveSpans = []histogram.Span{
{Offset: -3, Length: 2},
@ -928,6 +1053,73 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) {
assertRecodedFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
}
{ // Custom buckets, no change.
c, hApp, ts, h1 := setup(cbh)
h2 := h1.Copy()
_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
require.True(t, ok)
assertNoNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
}
{ // Custom buckets, increase in bucket counts but no change in layout.
c, hApp, ts, h1 := setup(cbh)
h2 := h1.Copy()
h2.Count++
h2.PositiveBuckets = []float64{6, 3, 3, 2, 4, 5, 2}
_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
require.True(t, ok)
assertNoNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
}
{ // Custom buckets, decrease in bucket counts but no change in layout.
c, hApp, ts, h1 := setup(cbh)
h2 := h1.Copy()
h2.Count--
h2.PositiveBuckets = []float64{6, 3, 3, 2, 4, 5, 0}
_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
require.True(t, ok)
assertNoNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
}
{ // Custom buckets, change only in custom bounds.
c, hApp, ts, h1 := setup(cbh)
h2 := h1.Copy()
h2.CustomValues = []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21}
_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
require.False(t, ok)
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
}
{ // Custom buckets, with more buckets.
c, hApp, ts, h1 := setup(cbh)
h2 := h1.Copy()
h2.PositiveSpans = []histogram.Span{
{Offset: 0, Length: 3},
{Offset: 1, Length: 1},
{Offset: 1, Length: 4},
{Offset: 3, Length: 3},
}
h2.Count += 6
h2.Sum = 30
// Existing histogram should get values converted from the above to:
// 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between)
// so the new histogram should have new counts >= these per-bucket counts, e.g.:
h2.PositiveBuckets = []float64{7, 5, 1, 3, 1, 0, 2, 5, 5, 0, 1} // (total 30)
posInterjections, negInterjections, pBackwardI, nBackwardI, _, _, ok := hApp.appendableGauge(h2)
require.NotEmpty(t, posInterjections)
require.Empty(t, negInterjections)
require.Empty(t, pBackwardI)
require.Empty(t, nBackwardI)
require.True(t, ok) // Only new buckets came in.
assertRecodedFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
}
}
func TestFloatHistogramAppendOnlyErrors(t *testing.T) {
@ -975,4 +1167,26 @@ func TestFloatHistogramAppendOnlyErrors(t *testing.T) {
require.False(t, isRecoded)
require.EqualError(t, err, "float histogram counter reset")
})
t.Run("counter reset error with custom buckets", func(t *testing.T) {
c := Chunk(NewFloatHistogramChunk())
// Create fresh appender and add the first histogram.
app, err := c.Appender()
require.NoError(t, err)
h := tsdbutil.GenerateTestCustomBucketsFloatHistogram(0)
var isRecoded bool
c, isRecoded, app, err = app.AppendFloatHistogram(nil, 1, h, true)
require.Nil(t, c)
require.False(t, isRecoded)
require.NoError(t, err)
// Add erroring histogram.
h2 := h.Copy()
h2.CustomValues = []float64{0, 1, 2, 3, 4, 5, 6, 7}
c, isRecoded, _, err = app.AppendFloatHistogram(nil, 2, h2, true)
require.Nil(t, c)
require.False(t, isRecoded)
require.EqualError(t, err, "float histogram counter reset")
})
}

34
tsdb/chunkenc/histogram.go
View file

@ -65,6 +65,7 @@ func (c *HistogramChunk) NumSamples() int {
func (c *HistogramChunk) Layout() (
schema int32, zeroThreshold float64,
negativeSpans, positiveSpans []histogram.Span,
customValues []float64,
err error,
) {
if c.NumSamples() == 0 {
@ -127,6 +128,7 @@ func (c *HistogramChunk) Appender() (Appender, error) {
zThreshold: it.zThreshold,
pSpans: it.pSpans,
nSpans: it.nSpans,
customValues: it.customValues,
t: it.t,
cnt: it.cnt,
zCnt: it.zCnt,
@ -194,6 +196,7 @@ type HistogramAppender struct {
schema int32
zThreshold float64
pSpans, nSpans []histogram.Span
customValues []float64
// Although we intend to start new chunks on counter resets, we still
// have to handle negative deltas for gauge histograms. Therefore, even
@ -237,6 +240,7 @@ func (a *HistogramAppender) Append(int64, float64) {
// The chunk is not appendable in the following cases:
//
// - The schema has changed.
// - The custom bounds have changed if the current schema is custom buckets.
// - The threshold for the zero bucket has changed.
// - Any buckets have disappeared.
// - There was a counter reset in the count of observations or in any bucket,
@ -279,6 +283,11 @@ func (a *HistogramAppender) appendable(h *histogram.Histogram) (
return
}
if histogram.IsCustomBucketsSchema(h.Schema) && !histogram.FloatBucketsMatch(h.CustomValues, a.customValues) {
counterReset = true
return
}
if h.ZeroCount < a.zCnt {
// There has been a counter reset since ZeroThreshold didn't change.
counterReset = true
@ -319,6 +328,7 @@ func (a *HistogramAppender) appendable(h *histogram.Histogram) (
//
// The chunk is not appendable in the following cases:
// - The schema has changed.
// - The custom bounds have changed if the current schema is custom buckets.
// - The threshold for the zero bucket has changed.
// - The last sample in the chunk was stale while the current sample is not stale.
func (a *HistogramAppender) appendableGauge(h *histogram.Histogram) (
@ -345,6 +355,10 @@ func (a *HistogramAppender) appendableGauge(h *histogram.Histogram) (
return
}
if histogram.IsCustomBucketsSchema(h.Schema) && !histogram.FloatBucketsMatch(h.CustomValues, a.customValues) {
return
}
positiveInserts, backwardPositiveInserts, positiveSpans = expandSpansBothWays(a.pSpans, h.PositiveSpans)
negativeInserts, backwardNegativeInserts, negativeSpans = expandSpansBothWays(a.nSpans, h.NegativeSpans)
okToAppend = true
@ -438,7 +452,7 @@ func (a *HistogramAppender) appendHistogram(t int64, h *histogram.Histogram) {
if num == 0 {
// The first append gets the privilege to dictate the layout
// but it's also responsible for encoding it into the chunk!
writeHistogramChunkLayout(a.b, h.Schema, h.ZeroThreshold, h.PositiveSpans, h.NegativeSpans)
writeHistogramChunkLayout(a.b, h.Schema, h.ZeroThreshold, h.PositiveSpans, h.NegativeSpans, h.CustomValues)
a.schema = h.Schema
a.zThreshold = h.ZeroThreshold
@ -454,6 +468,12 @@ func (a *HistogramAppender) appendHistogram(t int64, h *histogram.Histogram) {
} else {
a.nSpans = nil
}
if len(h.CustomValues) > 0 {
a.customValues = make([]float64, len(h.CustomValues))
copy(a.customValues, h.CustomValues)
} else {
a.customValues = nil
}
numPBuckets, numNBuckets := countSpans(h.PositiveSpans), countSpans(h.NegativeSpans)
if numPBuckets > 0 {
@ -737,6 +757,7 @@ type histogramIterator struct {
schema int32
zThreshold float64
pSpans, nSpans []histogram.Span
customValues []float64
// For the fields that are tracked as deltas and ultimately dod's.
t int64
@ -793,6 +814,7 @@ func (it *histogramIterator) AtHistogram(h *histogram.Histogram) (int64, *histog
NegativeSpans: it.nSpans,
PositiveBuckets: it.pBuckets,
NegativeBuckets: it.nBuckets,
CustomValues: it.customValues,
}
}
@ -815,6 +837,9 @@ func (it *histogramIterator) AtHistogram(h *histogram.Histogram) (int64, *histog
h.NegativeBuckets = resize(h.NegativeBuckets, len(it.nBuckets))
copy(h.NegativeBuckets, it.nBuckets)
h.CustomValues = resize(h.CustomValues, len(it.customValues))
copy(h.CustomValues, it.customValues)
return it.t, h
}
@ -835,6 +860,7 @@ func (it *histogramIterator) AtFloatHistogram(fh *histogram.FloatHistogram) (int
NegativeSpans: it.nSpans,
PositiveBuckets: it.pFloatBuckets,
NegativeBuckets: it.nFloatBuckets,
CustomValues: it.customValues,
}
}
@ -865,6 +891,9 @@ func (it *histogramIterator) AtFloatHistogram(fh *histogram.FloatHistogram) (int
fh.NegativeBuckets[i] = currentNegative
}
fh.CustomValues = resize(fh.CustomValues, len(it.customValues))
copy(fh.CustomValues, it.customValues)
return it.t, fh
}
@ -923,7 +952,7 @@ func (it *histogramIterator) Next() ValueType {
// The first read is responsible for reading the chunk layout
// and for initializing fields that depend on it. We give
// counter reset info at chunk level, hence we discard it here.
schema, zeroThreshold, posSpans, negSpans, err := readHistogramChunkLayout(&it.br)
schema, zeroThreshold, posSpans, negSpans, customValues, err := readHistogramChunkLayout(&it.br)
if err != nil {
it.err = err
return ValNone
@ -931,6 +960,7 @@ func (it *histogramIterator) Next() ValueType {
it.schema = schema
it.zThreshold = zeroThreshold
it.pSpans, it.nSpans = posSpans, negSpans
it.customValues = customValues
numPBuckets, numNBuckets := countSpans(posSpans), countSpans(negSpans)
// The code below recycles existing slices in case this iterator
// was reset and already has slices of a sufficient capacity.

83
tsdb/chunkenc/histogram_meta.go
View file

@ -21,17 +21,21 @@ import (
func writeHistogramChunkLayout(
b *bstream, schema int32, zeroThreshold float64,
positiveSpans, negativeSpans []histogram.Span,
positiveSpans, negativeSpans []histogram.Span, customValues []float64,
) {
putZeroThreshold(b, zeroThreshold)
putVarbitInt(b, int64(schema))
putHistogramChunkLayoutSpans(b, positiveSpans)
putHistogramChunkLayoutSpans(b, negativeSpans)
if histogram.IsCustomBucketsSchema(schema) {
putHistogramChunkLayoutCustomBounds(b, customValues)
}
}
func readHistogramChunkLayout(b *bstreamReader) (
schema int32, zeroThreshold float64,
positiveSpans, negativeSpans []histogram.Span,
customValues []float64,
err error,
) {
zeroThreshold, err = readZeroThreshold(b)
@ -55,6 +59,13 @@ func readHistogramChunkLayout(b *bstreamReader) (
return
}
if histogram.IsCustomBucketsSchema(schema) {
customValues, err = readHistogramChunkLayoutCustomBounds(b)
if err != nil {
return
}
}
return
}
@ -92,6 +103,30 @@ func readHistogramChunkLayoutSpans(b *bstreamReader) ([]histogram.Span, error) {
return spans, nil
}
func putHistogramChunkLayoutCustomBounds(b *bstream, customValues []float64) {
putVarbitUint(b, uint64(len(customValues)))
for _, bound := range customValues {
putCustomBound(b, bound)
}
}
func readHistogramChunkLayoutCustomBounds(b *bstreamReader) ([]float64, error) {
var customValues []float64
num, err := readVarbitUint(b)
if err != nil {
return nil, err
}
for i := 0; i < int(num); i++ {
bound, err := readCustomBound(b)
if err != nil {
return nil, err
}
customValues = append(customValues, bound)
}
return customValues, nil
}
// putZeroThreshold writes the zero threshold to the bstream. It stores typical
// values in just one byte, but needs 9 bytes for other values. In detail:
// - If the threshold is 0, store a single zero byte.
@ -140,6 +175,52 @@ func readZeroThreshold(br *bstreamReader) (float64, error) {
}
}
// isWholeWhenMultiplied checks to see if the number when multiplied by 1000 can
// be converted into an integer without losing precision.
func isWholeWhenMultiplied(in float64) bool {
i := uint(math.Round(in * 1000))
out := float64(i) / 1000
return in == out
}
// putCustomBound writes the custom bound to the bstream. It stores values from 0 to
// 16.382 (inclusive) that are multiples of 0.001 in an unsigned var int of up to 2 bytes,
// but needs 1 bit + 8 bytes for other values like negative numbers, numbers greater than
// 16.382, or numbers that are not a multiple of 0.001, on the assumption that they are
// less common. In detail:
// - Multiply the bound by 1000, without rounding.
// - If the multiplied bound is >= 0, <= 16382 and a whole number, store it as an
// unsigned var int.
// - Otherwise, store 0 as an unsigned var int, followed by the 8 bytes of the original
// bound as a float64.
func putCustomBound(b *bstream, f float64) {
tf := f * 1000
if tf < 0 || tf > 16382 || !isWholeWhenMultiplied(f) {
b.putUvarint(0)
b.writeBits(math.Float64bits(f), 64)
return
}
b.putUvarint(uint64(math.Round(tf) + 1))
}
// readCustomBound reads the custom bound written with putCustomBound.
func readCustomBound(br *bstreamReader) (float64, error) {
b, err := br.readUvarint()
if err != nil {
return 0, err
}
switch b {
case 0:
v, err := br.readBits(64)
if err != nil {
return 0, err
}
return math.Float64frombits(v), nil
default:
return float64(b-1) / 1000, nil
}
}
type bucketIterator struct {
spans []histogram.Span
span int // Span position of last yielded bucket.

30
tsdb/chunkenc/histogram_meta_test.go
View file

@ -373,6 +373,7 @@ func TestWriteReadHistogramChunkLayout(t *testing.T) {
schema int32
zeroThreshold float64
positiveSpans, negativeSpans []histogram.Span
customValues []float64
}{
{
schema: 3,
@ -422,23 +423,48 @@ func TestWriteReadHistogramChunkLayout(t *testing.T) {
positiveSpans: nil,
negativeSpans: nil,
},
{
schema: histogram.CustomBucketsSchema,
positiveSpans: []histogram.Span{{Offset: -4, Length: 3}, {Offset: 2, Length: 42}},
negativeSpans: nil,
customValues: []float64{-5, -2.5, 0, 0.1, 0.25, 0.5, 1, 2, 5, 10, 25, 50, 100, 255, 500, 1000},
},
{
schema: histogram.CustomBucketsSchema,
positiveSpans: []histogram.Span{{Offset: -4, Length: 3}, {Offset: 2, Length: 42}},
negativeSpans: nil,
customValues: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 25.0, 50.0, 100.0},
},
{
schema: histogram.CustomBucketsSchema,
positiveSpans: []histogram.Span{{Offset: -4, Length: 3}, {Offset: 2, Length: 42}},
negativeSpans: nil,
customValues: []float64{0.001, 0.002, 0.004, 0.008, 0.016, 0.032, 0.064, 0.128, 0.256, 0.512, 1.024, 2.048, 4.096, 8.192},
},
{
schema: histogram.CustomBucketsSchema,
positiveSpans: []histogram.Span{{Offset: -4, Length: 3}, {Offset: 2, Length: 42}},
negativeSpans: nil,
customValues: []float64{1.001, 1.023, 2.01, 4.007, 4.095, 8.001, 8.19, 16.24},
},
}
bs := bstream{}
for _, l := range layouts {
writeHistogramChunkLayout(&bs, l.schema, l.zeroThreshold, l.positiveSpans, l.negativeSpans)
writeHistogramChunkLayout(&bs, l.schema, l.zeroThreshold, l.positiveSpans, l.negativeSpans, l.customValues)
}
bsr := newBReader(bs.bytes())
for _, want := range layouts {
gotSchema, gotZeroThreshold, gotPositiveSpans, gotNegativeSpans, err := readHistogramChunkLayout(&bsr)
gotSchema, gotZeroThreshold, gotPositiveSpans, gotNegativeSpans, gotCustomBounds, err := readHistogramChunkLayout(&bsr)
require.NoError(t, err)
require.Equal(t, want.schema, gotSchema)
require.Equal(t, want.zeroThreshold, gotZeroThreshold)
require.Equal(t, want.positiveSpans, gotPositiveSpans)
require.Equal(t, want.negativeSpans, gotNegativeSpans)
require.Equal(t, want.customValues, gotCustomBounds)
}
}

391
tsdb/chunkenc/histogram_test.go
View file

@ -294,7 +294,38 @@ func TestHistogramChunkBucketChanges(t *testing.T) {
}
func TestHistogramChunkAppendable(t *testing.T) {
setup := func() (Chunk, *HistogramAppender, int64, *histogram.Histogram) {
eh := &histogram.Histogram{
Count: 5,
ZeroCount: 2,
Sum: 18.4,
ZeroThreshold: 1e-125,
Schema: 1,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 2},
{Offset: 2, Length: 1},
{Offset: 3, Length: 2},
{Offset: 3, Length: 1},
{Offset: 1, Length: 1},
},
PositiveBuckets: []int64{6, -3, 0, -1, 2, 1, -4}, // counts: 6, 3, 3, 2, 4, 5, 1 (total 24)
}
cbh := &histogram.Histogram{
Count: 24,
Sum: 18.4,
Schema: histogram.CustomBucketsSchema,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 2},
{Offset: 2, Length: 1},
{Offset: 3, Length: 2},
{Offset: 3, Length: 1},
{Offset: 1, Length: 1},
},
PositiveBuckets: []int64{6, -3, 0, -1, 2, 1, -4}, // counts: 6, 3, 3, 2, 4, 5, 1 (total 24)
CustomValues: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20},
}
setup := func(h *histogram.Histogram) (Chunk, *HistogramAppender, int64, *histogram.Histogram) {
c := Chunk(NewHistogramChunk())
// Create fresh appender and add the first histogram.
@ -303,32 +334,17 @@ func TestHistogramChunkAppendable(t *testing.T) {
require.Equal(t, 0, c.NumSamples())
ts := int64(1234567890)
h1 := &histogram.Histogram{
Count: 5,
ZeroCount: 2,
Sum: 18.4,
ZeroThreshold: 1e-125,
Schema: 1,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 2},
{Offset: 2, Length: 1},
{Offset: 3, Length: 2},
{Offset: 3, Length: 1},
{Offset: 1, Length: 1},
},
PositiveBuckets: []int64{6, -3, 0, -1, 2, 1, -4}, // counts: 6, 3, 3, 2, 4, 5, 1 (total 24)
}
chk, _, app, err := app.AppendHistogram(nil, ts, h1.Copy(), false)
chk, _, app, err := app.AppendHistogram(nil, ts, h.Copy(), false)
require.NoError(t, err)
require.Nil(t, chk)
require.Equal(t, 1, c.NumSamples())
require.Equal(t, UnknownCounterReset, c.(*HistogramChunk).GetCounterResetHeader())
return c, app.(*HistogramAppender), ts, h1
return c, app.(*HistogramAppender), ts, h
}
{ // Schema change.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.Schema++
_, _, ok, _ := hApp.appendable(h2)
@ -338,7 +354,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
}
{ // Zero threshold change.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.ZeroThreshold += 0.1
_, _, ok, _ := hApp.appendable(h2)
@ -348,7 +364,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
}
{ // New histogram that has more buckets.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.PositiveSpans = []histogram.Span{
{Offset: 0, Length: 3},
@ -374,7 +390,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
}
{ // New histogram that has a bucket missing.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.PositiveSpans = []histogram.Span{
{Offset: 0, Length: 2},
@ -395,7 +411,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
}
{ // New histogram that has a counter reset while buckets are same.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.Sum = 23
h2.PositiveBuckets = []int64{6, -4, 1, -1, 2, 1, -4} // counts: 6, 2, 3, 2, 4, 5, 1 (total 23)
@ -410,7 +426,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
}
{ // New histogram that has a counter reset while new buckets were added.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.PositiveSpans = []histogram.Span{
{Offset: 0, Length: 3},
@ -438,7 +454,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
// added before the first bucket and reset on first bucket. (to
// catch the edge case where the new bucket should be forwarded
// ahead until first old bucket at start)
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.PositiveSpans = []histogram.Span{
{Offset: -3, Length: 2},
@ -464,7 +480,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
}
{ // New histogram that has an explicit counter reset.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.CounterResetHint = histogram.CounterReset
@ -472,7 +488,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
}
{ // Start new chunk explicitly, and append a new histogram that is considered appendable to the previous chunk.
_, hApp, ts, h1 := setup()
_, hApp, ts, h1 := setup(eh)
h2 := h1.Copy() // Identity is appendable.
nextChunk := NewHistogramChunk()
@ -488,7 +504,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
}
{ // Start new chunk explicitly, and append a new histogram that is not considered appendable to the previous chunk.
_, hApp, ts, h1 := setup()
_, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.Count-- // Make this not appendable due to counter reset.
@ -505,7 +521,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
}
{ // Start new chunk explicitly, and append a new histogram that would need recoding if we added it to the chunk.
_, hApp, ts, h1 := setup()
_, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.PositiveSpans = []histogram.Span{
{Offset: 0, Length: 3},
@ -532,6 +548,72 @@ func TestHistogramChunkAppendable(t *testing.T) {
assertSampleCount(t, nextChunk, 1, ValHistogram)
require.Equal(t, NotCounterReset, nextChunk.GetCounterResetHeader())
}
{ // Custom buckets, no change.
c, hApp, ts, h1 := setup(cbh)
h2 := h1.Copy()
_, _, ok, _ := hApp.appendable(h2)
require.True(t, ok)
assertNoNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
}
{ // Custom buckets, increase in bucket counts but no change in layout.
c, hApp, ts, h1 := setup(cbh)
h2 := h1.Copy()
h2.Count++
h2.PositiveBuckets = []int64{6, -3, 0, -1, 2, 1, -3}
_, _, ok, _ := hApp.appendable(h2)
require.True(t, ok)
assertNoNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
}
{ // Custom buckets, decrease in bucket counts but no change in layout.
c, hApp, ts, h1 := setup(cbh)
h2 := h1.Copy()
h2.Count--
h2.PositiveBuckets = []int64{6, -3, 0, -1, 2, 1, -5}
_, _, ok, _ := hApp.appendable(h2)
require.False(t, ok)
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
}
{ // Custom buckets, change only in custom bounds.
c, hApp, ts, h1 := setup(cbh)
h2 := h1.Copy()
h2.CustomValues = []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21}
_, _, ok, _ := hApp.appendable(h2)
require.False(t, ok)
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
}
{ // Custom buckets, with more buckets.
c, hApp, ts, h1 := setup(cbh)
h2 := h1.Copy()
h2.PositiveSpans = []histogram.Span{
{Offset: 0, Length: 3},
{Offset: 1, Length: 1},
{Offset: 1, Length: 4},
{Offset: 3, Length: 3},
}
h2.Count += 6
h2.Sum = 30
// Existing histogram should get values converted from the above to:
// 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between)
// so the new histogram should have new counts >= these per-bucket counts, e.g.:
h2.PositiveBuckets = []int64{7, -2, -4, 2, -2, -1, 2, 3, 0, -5, 1} // 7 5 1 3 1 0 2 5 5 0 1 (total 30)
posInterjections, negInterjections, ok, cr := hApp.appendable(h2)
require.NotEmpty(t, posInterjections)
require.Empty(t, negInterjections)
require.True(t, ok) // Only new buckets came in.
require.False(t, cr)
assertRecodedHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
}
}
func assertNewHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *HistogramAppender, ts int64, h *histogram.Histogram, expectHeader CounterResetHeader) {
@ -548,6 +630,19 @@ func assertNewHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *Histogr
assertSampleCount(t, newChunk, 1, ValHistogram)
}
func assertNoNewHistogramChunkOnAppend(t *testing.T, currChunk Chunk, hApp *HistogramAppender, ts int64, h *histogram.Histogram, expectHeader CounterResetHeader) {
prevChunkBytes := currChunk.Bytes()
newChunk, recoded, newAppender, err := hApp.AppendHistogram(nil, ts, h, false)
require.Greater(t, len(currChunk.Bytes()), len(prevChunkBytes)) // Check that current chunk is bigger than previously.
require.NoError(t, err)
require.Nil(t, newChunk)
require.False(t, recoded)
require.Equal(t, expectHeader, currChunk.(*HistogramChunk).GetCounterResetHeader())
require.NotNil(t, newAppender)
require.Equal(t, hApp, newAppender)
assertSampleCount(t, currChunk, 2, ValHistogram)
}
func assertRecodedHistogramChunkOnAppend(t *testing.T, prevChunk Chunk, hApp *HistogramAppender, ts int64, h *histogram.Histogram, expectHeader CounterResetHeader) {
prevChunkBytes := prevChunk.Bytes()
newChunk, recoded, newAppender, err := hApp.AppendHistogram(nil, ts, h, false)
@ -738,6 +833,32 @@ func TestHistogramChunkAppendableWithEmptySpan(t *testing.T) {
NegativeBuckets: []int64{1, 3, -2, 5, -2, 0, -3},
},
},
"empty span in old and new custom buckets histogram": {
h1: &histogram.Histogram{
Schema: histogram.CustomBucketsSchema,
Count: 7,
Sum: 1234.5,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 4},
{Offset: 0, Length: 0},
{Offset: 0, Length: 3},
},
PositiveBuckets: []int64{1, 1, -1, 0, 0, 0, 0},
CustomValues: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
},
h2: &histogram.Histogram{
Schema: histogram.CustomBucketsSchema,
Count: 10,
Sum: 2345.6,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 4},
{Offset: 0, Length: 0},
{Offset: 0, Length: 3},
},
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 0},
CustomValues: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
},
},
}
for name, tc := range tests {
@ -905,7 +1026,40 @@ func TestAtFloatHistogram(t *testing.T) {
}
func TestHistogramChunkAppendableGauge(t *testing.T) {
setup := func() (Chunk, *HistogramAppender, int64, *histogram.Histogram) {
eh := &histogram.Histogram{
CounterResetHint: histogram.GaugeType,
Count: 5,
ZeroCount: 2,
Sum: 18.4,
ZeroThreshold: 1e-125,
Schema: 1,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 2},
{Offset: 2, Length: 1},
{Offset: 3, Length: 2},
{Offset: 3, Length: 1},
{Offset: 1, Length: 1},
},
PositiveBuckets: []int64{6, -3, 0, -1, 2, 1, -4}, // {6, 3, 3, 2, 4, 5, 1}
}
cbh := &histogram.Histogram{
CounterResetHint: histogram.GaugeType,
Count: 24,
Sum: 18.4,
Schema: histogram.CustomBucketsSchema,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 2},
{Offset: 2, Length: 1},
{Offset: 3, Length: 2},
{Offset: 3, Length: 1},
{Offset: 1, Length: 1},
},
PositiveBuckets: []int64{6, -3, 0, -1, 2, 1, -4}, // {6, 3, 3, 2, 4, 5, 1}
CustomValues: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20},
}
setup := func(h *histogram.Histogram) (Chunk, *HistogramAppender, int64, *histogram.Histogram) {
c := Chunk(NewHistogramChunk())
// Create fresh appender and add the first histogram.
@ -914,66 +1068,38 @@ func TestHistogramChunkAppendableGauge(t *testing.T) {
require.Equal(t, 0, c.NumSamples())
ts := int64(1234567890)
h1 := &histogram.Histogram{
CounterResetHint: histogram.GaugeType,
Count: 5,
ZeroCount: 2,
Sum: 18.4,
ZeroThreshold: 1e-125,
Schema: 1,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 2},
{Offset: 2, Length: 1},
{Offset: 3, Length: 2},
{Offset: 3, Length: 1},
{Offset: 1, Length: 1},
},
PositiveBuckets: []int64{6, -3, 0, -1, 2, 1, -4}, // {6, 3, 3, 2, 4, 5, 1}
}
chk, _, app, err := app.AppendHistogram(nil, ts, h1.Copy(), false)
chk, _, app, err := app.AppendHistogram(nil, ts, h.Copy(), false)
require.NoError(t, err)
require.Nil(t, chk)
require.Equal(t, 1, c.NumSamples())
require.Equal(t, GaugeType, c.(*HistogramChunk).GetCounterResetHeader())
return c, app.(*HistogramAppender), ts, h1
return c, app.(*HistogramAppender), ts, h
}
{ // Schema change.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.Schema++
_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
require.False(t, ok)
newc, recoded, _, err := hApp.AppendHistogram(nil, ts+1, h2, false)
require.NoError(t, err)
require.NotNil(t, newc)
require.False(t, recoded)
require.NotEqual(t, c, newc)
require.Equal(t, GaugeType, c.(*HistogramChunk).GetCounterResetHeader())
require.Equal(t, GaugeType, newc.(*HistogramChunk).GetCounterResetHeader())
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
}
{ // Zero threshold change.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.ZeroThreshold += 0.1
_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
require.False(t, ok)
newc, recoded, _, err := hApp.AppendHistogram(nil, ts+1, h2, false)
require.NoError(t, err)
require.NotNil(t, newc)
require.False(t, recoded)
require.NotEqual(t, c, newc)
require.Equal(t, GaugeType, c.(*HistogramChunk).GetCounterResetHeader())
require.Equal(t, GaugeType, newc.(*HistogramChunk).GetCounterResetHeader())
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
}
{ // New histogram that has more buckets.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.PositiveSpans = []histogram.Span{
{Offset: 0, Length: 3},
@ -993,15 +1119,11 @@ func TestHistogramChunkAppendableGauge(t *testing.T) {
require.Empty(t, nBackwardI)
require.True(t, ok)
newc, recoded, _, err := hApp.AppendHistogram(nil, ts+1, h2, false)
require.NoError(t, err)
require.NotNil(t, newc)
require.True(t, recoded)
require.Equal(t, GaugeType, c.(*HistogramChunk).GetCounterResetHeader())
assertRecodedHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
}
{ // New histogram that has buckets missing.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.PositiveSpans = []histogram.Span{
{Offset: 0, Length: 2},
@ -1021,15 +1143,11 @@ func TestHistogramChunkAppendableGauge(t *testing.T) {
require.Empty(t, nBackwardI)
require.True(t, ok)
newc, recoded, _, err := hApp.AppendHistogram(nil, ts+1, h2, false)
require.NoError(t, err)
require.Nil(t, newc)
require.False(t, recoded)
require.Equal(t, GaugeType, c.(*HistogramChunk).GetCounterResetHeader())
assertNoNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
}
{ // New histogram that has a bucket missing and new buckets.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.PositiveSpans = []histogram.Span{
{Offset: 0, Length: 2},
@ -1047,15 +1165,11 @@ func TestHistogramChunkAppendableGauge(t *testing.T) {
require.Empty(t, nBackwardI)
require.True(t, ok)
newc, recoded, _, err := hApp.AppendHistogram(nil, ts+1, h2, false)
require.NoError(t, err)
require.NotNil(t, newc)
require.True(t, recoded)
require.Equal(t, GaugeType, c.(*HistogramChunk).GetCounterResetHeader())
assertRecodedHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
}
{ // New histogram that has a counter reset while buckets are same.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.Sum = 23
h2.PositiveBuckets = []int64{6, -4, 1, -1, 2, 1, -4} // {6, 2, 3, 2, 4, 5, 1}
@ -1067,15 +1181,11 @@ func TestHistogramChunkAppendableGauge(t *testing.T) {
require.Empty(t, nBackwardI)
require.True(t, ok)
newc, recoded, _, err := hApp.AppendHistogram(nil, ts+1, h2, false)
require.NoError(t, err)
require.Nil(t, newc)
require.False(t, recoded)
require.Equal(t, GaugeType, c.(*HistogramChunk).GetCounterResetHeader())
assertNoNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
}
{ // New histogram that has a counter reset while new buckets were added.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.PositiveSpans = []histogram.Span{
{Offset: 0, Length: 3},
@ -1093,17 +1203,13 @@ func TestHistogramChunkAppendableGauge(t *testing.T) {
require.Empty(t, nBackwardI)
require.True(t, ok)
newc, recoded, _, err := hApp.AppendHistogram(nil, ts+1, h2, false)
require.NoError(t, err)
require.NotNil(t, newc)
require.True(t, recoded)
require.Equal(t, GaugeType, c.(*HistogramChunk).GetCounterResetHeader())
assertRecodedHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
}
{
// New histogram that has a counter reset while new buckets were
// added before the first bucket and reset on first bucket.
c, hApp, ts, h1 := setup()
c, hApp, ts, h1 := setup(eh)
h2 := h1.Copy()
h2.PositiveSpans = []histogram.Span{
{Offset: -3, Length: 2},
@ -1123,11 +1229,74 @@ func TestHistogramChunkAppendableGauge(t *testing.T) {
require.Empty(t, nBackwardI)
require.True(t, ok)
newc, recoded, _, err := hApp.AppendHistogram(nil, ts+1, h2, false)
require.NoError(t, err)
require.NotNil(t, newc)
require.True(t, recoded)
require.Equal(t, GaugeType, c.(*HistogramChunk).GetCounterResetHeader())
assertRecodedHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
}
{ // Custom buckets, no change.
c, hApp, ts, h1 := setup(cbh)
h2 := h1.Copy()
_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
require.True(t, ok)
assertNoNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
}
{ // Custom buckets, increase in bucket counts but no change in layout.
c, hApp, ts, h1 := setup(cbh)
h2 := h1.Copy()
h2.Count++
h2.PositiveBuckets = []int64{6, -3, 0, -1, 2, 1, -3}
_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
require.True(t, ok)
assertNoNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
}
{ // Custom buckets, decrease in bucket counts but no change in layout.
c, hApp, ts, h1 := setup(cbh)
h2 := h1.Copy()
h2.Count--
h2.PositiveBuckets = []int64{6, -3, 0, -1, 2, 1, -5}
_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
require.True(t, ok)
assertNoNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
}
{ // Custom buckets, change only in custom bounds.
c, hApp, ts, h1 := setup(cbh)
h2 := h1.Copy()
h2.CustomValues = []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21}
_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
require.False(t, ok)
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
}
{ // Custom buckets, with more buckets.
c, hApp, ts, h1 := setup(cbh)
h2 := h1.Copy()
h2.PositiveSpans = []histogram.Span{
{Offset: 0, Length: 3},
{Offset: 1, Length: 1},
{Offset: 1, Length: 4},
{Offset: 3, Length: 3},
}
h2.Count += 6
h2.Sum = 30
// Existing histogram should get values converted from the above to:
// 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between)
// so the new histogram should have new counts >= these per-bucket counts, e.g.:
h2.PositiveBuckets = []int64{7, -2, -4, 2, -2, -1, 2, 3, 0, -5, 1} // 7 5 1 3 1 0 2 5 5 0 1 (total 30)
posInterjections, negInterjections, pBackwardI, nBackwardI, _, _, ok := hApp.appendableGauge(h2)
require.NotEmpty(t, posInterjections)
require.Empty(t, negInterjections)
require.Empty(t, pBackwardI)
require.Empty(t, nBackwardI)
require.True(t, ok) // Only new buckets came in.
assertRecodedHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
}
}
@ -1176,4 +1345,26 @@ func TestHistogramAppendOnlyErrors(t *testing.T) {
require.False(t, isRecoded)
require.EqualError(t, err, "histogram counter reset")
})
t.Run("counter reset error with custom buckets", func(t *testing.T) {
c := Chunk(NewHistogramChunk())
// Create fresh appender and add the first histogram.
app, err := c.Appender()
require.NoError(t, err)
h := tsdbutil.GenerateTestCustomBucketsHistogram(0)
var isRecoded bool
c, isRecoded, app, err = app.AppendHistogram(nil, 1, h, true)
require.Nil(t, c)
require.False(t, isRecoded)
require.NoError(t, err)
// Add erroring histogram.
h2 := h.Copy()
h2.CustomValues = []float64{0, 1, 2, 3, 4, 5, 6, 7}
c, isRecoded, _, err = app.AppendHistogram(nil, 2, h2, true)
require.Nil(t, c)
require.False(t, isRecoded)
require.EqualError(t, err, "histogram counter reset")
})
}

28
tsdb/tsdbutil/histogram.go
View file

@ -59,6 +59,20 @@ func GenerateTestHistogram(i int) *histogram.Histogram {
}
}
func GenerateTestCustomBucketsHistogram(i int) *histogram.Histogram {
return &histogram.Histogram{
Count: 5 + uint64(i*4),
Sum: 18.4 * float64(i+1),
Schema: histogram.CustomBucketsSchema,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 2},
{Offset: 1, Length: 2},
},
PositiveBuckets: []int64{int64(i + 1), 1, -1, 0},
CustomValues: []float64{0, 1, 2, 3, 4},
}
}
func GenerateTestGaugeHistograms(n int) (r []*histogram.Histogram) {
for x := 0; x < n; x++ {
i := int(math.Sin(float64(x))*100) + 100
@ -105,6 +119,20 @@ func GenerateTestFloatHistogram(i int) *histogram.FloatHistogram {
}
}
func GenerateTestCustomBucketsFloatHistogram(i int) *histogram.FloatHistogram {
return &histogram.FloatHistogram{
Count: 5 + float64(i*4),
Sum: 18.4 * float64(i+1),
Schema: histogram.CustomBucketsSchema,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 2},
{Offset: 1, Length: 2},
},
PositiveBuckets: []float64{float64(i + 1), float64(i + 2), float64(i + 1), float64(i + 1)},
CustomValues: []float64{0, 1, 2, 3, 4},
}
}
func GenerateTestGaugeFloatHistograms(n int) (r []*histogram.FloatHistogram) {
for x := 0; x < n; x++ {
i := int(math.Sin(float64(x))*100) + 100