perf(nhcb): optimize away most allocations in convertnhcb
Some checks failed
CI / Go tests (push) Has been cancelled
CI / More Go tests (push) Has been cancelled
CI / Go tests with previous Go version (push) Has been cancelled
CI / UI tests (push) Has been cancelled
CI / Go tests on Windows (push) Has been cancelled
CI / Mixins tests (push) Has been cancelled
CI / Build Prometheus for common architectures (0) (push) Has been cancelled
CI / Build Prometheus for common architectures (1) (push) Has been cancelled
CI / Build Prometheus for common architectures (2) (push) Has been cancelled
CI / Build Prometheus for all architectures (0) (push) Has been cancelled
CI / Build Prometheus for all architectures (1) (push) Has been cancelled
CI / Build Prometheus for all architectures (10) (push) Has been cancelled
CI / Build Prometheus for all architectures (11) (push) Has been cancelled
CI / Build Prometheus for all architectures (2) (push) Has been cancelled
CI / Build Prometheus for all architectures (3) (push) Has been cancelled
CI / Build Prometheus for all architectures (4) (push) Has been cancelled
CI / Build Prometheus for all architectures (5) (push) Has been cancelled
CI / Build Prometheus for all architectures (6) (push) Has been cancelled
CI / Build Prometheus for all architectures (7) (push) Has been cancelled
CI / Build Prometheus for all architectures (8) (push) Has been cancelled
CI / Build Prometheus for all architectures (9) (push) Has been cancelled
CI / Check generated parser (push) Has been cancelled
CI / golangci-lint (push) Has been cancelled
CI / fuzzing (push) Has been cancelled
CI / codeql (push) Has been cancelled
CI / Report status of build Prometheus for all architectures (push) Has been cancelled
CI / Publish main branch artifacts (push) Has been cancelled
CI / Publish release artefacts (push) Has been cancelled
CI / Publish UI on npm Registry (push) Has been cancelled

In general aim for the happy case when the exposer lists the buckets
in ascending order.

Use Compact(2) to compact the result of nhcb convert.

This is more in line with how client_golang optimizes spans vs
buckets.
aef8aedb4b/prometheus/histogram.go (L1485)

Signed-off-by: György Krajcsovits <gyorgy.krajcsovits@grafana.com>
This commit is contained in:
György Krajcsovits 2024-10-25 09:42:46 +02:00
parent eb3b349024
commit eafe72a0d0
5 changed files with 461 additions and 151 deletions

View file

@ -283,18 +283,18 @@ func (p *NHCBParser) handleClassicHistogramSeries(lset labels.Labels) bool {
le, err := strconv.ParseFloat(lset.Get(labels.BucketLabel), 64) le, err := strconv.ParseFloat(lset.Get(labels.BucketLabel), 64)
if err == nil && !math.IsNaN(le) { if err == nil && !math.IsNaN(le) {
p.processClassicHistogramSeries(lset, "_bucket", func(hist *convertnhcb.TempHistogram) { p.processClassicHistogramSeries(lset, "_bucket", func(hist *convertnhcb.TempHistogram) {
hist.BucketCounts[le] = p.value _ = hist.SetBucketCount(le, p.value)
}) })
return true return true
} }
case strings.HasSuffix(mName, "_count"): case strings.HasSuffix(mName, "_count"):
p.processClassicHistogramSeries(lset, "_count", func(hist *convertnhcb.TempHistogram) { p.processClassicHistogramSeries(lset, "_count", func(hist *convertnhcb.TempHistogram) {
hist.Count = p.value _ = hist.SetCount(p.value)
}) })
return true return true
case strings.HasSuffix(mName, "_sum"): case strings.HasSuffix(mName, "_sum"):
p.processClassicHistogramSeries(lset, "_sum", func(hist *convertnhcb.TempHistogram) { p.processClassicHistogramSeries(lset, "_sum", func(hist *convertnhcb.TempHistogram) {
hist.Sum = p.value _ = hist.SetSum(p.value)
}) })
return true return true
} }
@ -306,8 +306,8 @@ func (p *NHCBParser) processClassicHistogramSeries(lset labels.Labels, suffix st
p.storeClassicLabels() p.storeClassicLabels()
p.tempCT = p.parser.CreatedTimestamp() p.tempCT = p.parser.CreatedTimestamp()
p.state = stateCollecting p.state = stateCollecting
p.tempLsetNHCB = convertnhcb.GetHistogramMetricBase(lset, suffix)
} }
p.tempLsetNHCB = convertnhcb.GetHistogramMetricBase(lset, suffix)
p.storeExemplars() p.storeExemplars()
updateHist(&p.tempNHCB) updateHist(&p.tempNHCB)
} }
@ -335,7 +335,6 @@ func (p *NHCBParser) nextExemplarPtr() *exemplar.Exemplar {
func (p *NHCBParser) swapExemplars() { func (p *NHCBParser) swapExemplars() {
p.exemplars = p.tempExemplars[:p.tempExemplarCount] p.exemplars = p.tempExemplars[:p.tempExemplarCount]
p.tempExemplars = p.tempExemplars[:0] p.tempExemplars = p.tempExemplars[:0]
p.tempExemplarCount = 0
} }
// processNHCB converts the collated classic histogram series to NHCB and caches the info // processNHCB converts the collated classic histogram series to NHCB and caches the info
@ -344,33 +343,32 @@ func (p *NHCBParser) processNHCB() bool {
if p.state != stateCollecting { if p.state != stateCollecting {
return false return false
} }
ub := make([]float64, 0, len(p.tempNHCB.BucketCounts)) h, fh, err := p.tempNHCB.Convert()
for b := range p.tempNHCB.BucketCounts { if err == nil {
ub = append(ub, b) if h != nil {
} if err := h.Validate(); err != nil {
upperBounds, hBase := convertnhcb.ProcessUpperBoundsAndCreateBaseHistogram(ub, false) return false
fhBase := hBase.ToFloat(nil) }
h, fh := convertnhcb.NewHistogram(p.tempNHCB, upperBounds, hBase, fhBase) p.hNHCB = h
if h != nil { p.fhNHCB = nil
if err := h.Validate(); err != nil { } else if fh != nil {
return false if err := fh.Validate(); err != nil {
return false
}
p.hNHCB = nil
p.fhNHCB = fh
} }
p.hNHCB = h p.metricStringNHCB = p.tempLsetNHCB.Get(labels.MetricName) + strings.ReplaceAll(p.tempLsetNHCB.DropMetricName().String(), ", ", ",")
p.fhNHCB = nil p.bytesNHCB = []byte(p.metricStringNHCB)
} else if fh != nil { p.lsetNHCB = p.tempLsetNHCB
if err := fh.Validate(); err != nil { p.swapExemplars()
return false p.ctNHCB = p.tempCT
} p.state = stateEmitting
p.hNHCB = nil } else {
p.fhNHCB = fh p.state = stateStart
} }
p.metricStringNHCB = p.tempLsetNHCB.Get(labels.MetricName) + strings.ReplaceAll(p.tempLsetNHCB.DropMetricName().String(), ", ", ",") p.tempNHCB.Reset()
p.bytesNHCB = []byte(p.metricStringNHCB) p.tempExemplarCount = 0
p.lsetNHCB = p.tempLsetNHCB
p.swapExemplars()
p.ctNHCB = p.tempCT
p.tempNHCB = convertnhcb.NewTempHistogram()
p.state = stateEmitting
p.tempCT = nil p.tempCT = nil
return true return err == nil
} }

View file

@ -500,8 +500,8 @@ something_bucket{a="b",le="+Inf"} 9 # {id="something-test"} 2e100 123.000
Schema: histogram.CustomBucketsSchema, Schema: histogram.CustomBucketsSchema,
Count: 9, Count: 9,
Sum: 42123.0, Sum: 42123.0,
PositiveSpans: []histogram.Span{{Offset: 0, Length: 1}, {Offset: 1, Length: 1}}, PositiveSpans: []histogram.Span{{Offset: 0, Length: 3}},
PositiveBuckets: []int64{8, -7}, PositiveBuckets: []int64{8, -8, 1},
CustomValues: []float64{0.0, 1.0}, // We do not store the +Inf boundary. CustomValues: []float64{0.0, 1.0}, // We do not store the +Inf boundary.
}, },
lset: labels.FromStrings("__name__", "something", "a", "b"), lset: labels.FromStrings("__name__", "something", "a", "b"),
@ -937,3 +937,48 @@ test_histogram1_bucket{le="-0.0003899999999999998"} 4 1234568
test_histogram1_bucket{le="-0.0002899999999999998"} 16 1234568 test_histogram1_bucket{le="-0.0002899999999999998"} 16 1234568
test_histogram1_bucket{le="+Inf"} 175 1234568` test_histogram1_bucket{le="+Inf"} 175 1234568`
} }
func TestNHCBParserErrorHandling(t *testing.T) {
input := `# HELP something Histogram with non cumulative buckets
# TYPE something histogram
something_count 18
something_sum 324789.4
something_created 1520430001
something_bucket{le="0.0"} 18
something_bucket{le="+Inf"} 1
something_count{a="b"} 9
something_sum{a="b"} 42123
something_created{a="b"} 1520430002
something_bucket{a="b",le="0.0"} 1
something_bucket{a="b",le="+Inf"} 9
# EOF`
exp := []parsedEntry{
{
m: "something",
help: "Histogram with non cumulative buckets",
},
{
m: "something",
typ: model.MetricTypeHistogram,
},
// The parser should skip the series with non-cumulative buckets.
{
m: `something{a="b"}`,
shs: &histogram.Histogram{
Schema: histogram.CustomBucketsSchema,
Count: 9,
Sum: 42123.0,
PositiveSpans: []histogram.Span{{Offset: 0, Length: 2}},
PositiveBuckets: []int64{1, 7},
CustomValues: []float64{0.0}, // We do not store the +Inf boundary.
},
lset: labels.FromStrings("__name__", "something", "a", "b"),
ct: int64p(1520430002000),
},
}
p := NewOpenMetricsParser([]byte(input), labels.NewSymbolTable(), WithOMParserCTSeriesSkipped())
p = NewNHCBParser(p, labels.NewSymbolTable(), false)
got := testParse(t, p)
requireEntries(t, exp, got)
}

View file

@ -482,18 +482,16 @@ func (cmd *loadCmd) append(a storage.Appender) error {
type tempHistogramWrapper struct { type tempHistogramWrapper struct {
metric labels.Labels metric labels.Labels
upperBounds []float64
histogramByTs map[int64]convertnhcb.TempHistogram histogramByTs map[int64]convertnhcb.TempHistogram
} }
func newTempHistogramWrapper() tempHistogramWrapper { func newTempHistogramWrapper() tempHistogramWrapper {
return tempHistogramWrapper{ return tempHistogramWrapper{
upperBounds: []float64{},
histogramByTs: map[int64]convertnhcb.TempHistogram{}, histogramByTs: map[int64]convertnhcb.TempHistogram{},
} }
} }
func processClassicHistogramSeries(m labels.Labels, suffix string, histogramMap map[uint64]tempHistogramWrapper, smpls []promql.Sample, updateHistogramWrapper func(*tempHistogramWrapper), updateHistogram func(*convertnhcb.TempHistogram, float64)) { func processClassicHistogramSeries(m labels.Labels, suffix string, histogramMap map[uint64]tempHistogramWrapper, smpls []promql.Sample, updateHistogram func(*convertnhcb.TempHistogram, float64)) {
m2 := convertnhcb.GetHistogramMetricBase(m, suffix) m2 := convertnhcb.GetHistogramMetricBase(m, suffix)
m2hash := m2.Hash() m2hash := m2.Hash()
histogramWrapper, exists := histogramMap[m2hash] histogramWrapper, exists := histogramMap[m2hash]
@ -501,9 +499,6 @@ func processClassicHistogramSeries(m labels.Labels, suffix string, histogramMap
histogramWrapper = newTempHistogramWrapper() histogramWrapper = newTempHistogramWrapper()
} }
histogramWrapper.metric = m2 histogramWrapper.metric = m2
if updateHistogramWrapper != nil {
updateHistogramWrapper(&histogramWrapper)
}
for _, s := range smpls { for _, s := range smpls {
if s.H != nil { if s.H != nil {
continue continue
@ -534,18 +529,16 @@ func (cmd *loadCmd) appendCustomHistogram(a storage.Appender) error {
if err != nil || math.IsNaN(le) { if err != nil || math.IsNaN(le) {
continue continue
} }
processClassicHistogramSeries(m, "_bucket", histogramMap, smpls, func(histogramWrapper *tempHistogramWrapper) { processClassicHistogramSeries(m, "_bucket", histogramMap, smpls, func(histogram *convertnhcb.TempHistogram, f float64) {
histogramWrapper.upperBounds = append(histogramWrapper.upperBounds, le) _ = histogram.SetBucketCount(le, f)
}, func(histogram *convertnhcb.TempHistogram, f float64) {
histogram.BucketCounts[le] = f
}) })
case strings.HasSuffix(mName, "_count"): case strings.HasSuffix(mName, "_count"):
processClassicHistogramSeries(m, "_count", histogramMap, smpls, nil, func(histogram *convertnhcb.TempHistogram, f float64) { processClassicHistogramSeries(m, "_count", histogramMap, smpls, func(histogram *convertnhcb.TempHistogram, f float64) {
histogram.Count = f _ = histogram.SetCount(f)
}) })
case strings.HasSuffix(mName, "_sum"): case strings.HasSuffix(mName, "_sum"):
processClassicHistogramSeries(m, "_sum", histogramMap, smpls, nil, func(histogram *convertnhcb.TempHistogram, f float64) { processClassicHistogramSeries(m, "_sum", histogramMap, smpls, func(histogram *convertnhcb.TempHistogram, f float64) {
histogram.Sum = f _ = histogram.SetSum(f)
}) })
} }
} }
@ -553,11 +546,12 @@ func (cmd *loadCmd) appendCustomHistogram(a storage.Appender) error {
// Convert the collated classic histogram data into native histograms // Convert the collated classic histogram data into native histograms
// with custom bounds and append them to the storage. // with custom bounds and append them to the storage.
for _, histogramWrapper := range histogramMap { for _, histogramWrapper := range histogramMap {
upperBounds, hBase := convertnhcb.ProcessUpperBoundsAndCreateBaseHistogram(histogramWrapper.upperBounds, true)
fhBase := hBase.ToFloat(nil)
samples := make([]promql.Sample, 0, len(histogramWrapper.histogramByTs)) samples := make([]promql.Sample, 0, len(histogramWrapper.histogramByTs))
for t, histogram := range histogramWrapper.histogramByTs { for t, histogram := range histogramWrapper.histogramByTs {
h, fh := convertnhcb.NewHistogram(histogram, upperBounds, hBase, fhBase) h, fh, err := histogram.Convert()
if err != nil {
return err
}
if fh == nil { if fh == nil {
if err := h.Validate(); err != nil { if err := h.Validate(); err != nil {
return err return err

View file

@ -14,6 +14,7 @@
package convertnhcb package convertnhcb
import ( import (
"errors"
"fmt" "fmt"
"math" "math"
"sort" "sort"
@ -23,129 +24,212 @@ import (
"github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/labels"
) )
var (
errNegativeBucketCount = errors.New("bucket count must be non-negative")
errNegativeCount = errors.New("count must be non-negative")
errCountMismatch = errors.New("count mismatch")
errCountNotCumulative = errors.New("count is not cumulative")
)
type tempHistogramBucket struct {
le float64
count float64
}
// TempHistogram is used to collect information about classic histogram // TempHistogram is used to collect information about classic histogram
// samples incrementally before creating a histogram.Histogram or // samples incrementally before creating a histogram.Histogram or
// histogram.FloatHistogram based on the values collected. // histogram.FloatHistogram based on the values collected.
type TempHistogram struct { type TempHistogram struct {
BucketCounts map[float64]float64 buckets []tempHistogramBucket
Count float64 count float64
Sum float64 sum float64
HasFloat bool err error
hasCount bool
} }
// NewTempHistogram creates a new TempHistogram to // NewTempHistogram creates a new TempHistogram to
// collect information about classic histogram samples. // collect information about classic histogram samples.
func NewTempHistogram() TempHistogram { func NewTempHistogram() TempHistogram {
return TempHistogram{ return TempHistogram{
BucketCounts: map[float64]float64{}, buckets: make([]tempHistogramBucket, 0, 10),
} }
} }
func (h TempHistogram) getIntBucketCounts() (map[float64]int64, error) { func (h TempHistogram) Err() error {
bucketCounts := map[float64]int64{} return h.err
for le, count := range h.BucketCounts { }
intCount := int64(math.Round(count))
if float64(intCount) != count { func (h *TempHistogram) Reset() {
return nil, fmt.Errorf("bucket count %f for le %g is not an integer", count, le) h.buckets = h.buckets[:0]
h.count = 0
h.sum = 0
h.err = nil
h.hasCount = false
}
func (h *TempHistogram) SetBucketCount(boundary, count float64) error {
if h.err != nil {
return h.err
}
if count < 0 {
h.err = fmt.Errorf("%w: le=%g, count=%g", errNegativeBucketCount, boundary, count)
return h.err
}
// Assume that the elements are added in order.
switch {
case len(h.buckets) == 0:
h.buckets = append(h.buckets, tempHistogramBucket{le: boundary, count: count})
case h.buckets[len(h.buckets)-1].le < boundary:
// Happy case is "<".
if count < h.buckets[len(h.buckets)-1].count {
h.err = fmt.Errorf("%w: %g < %g", errCountNotCumulative, count, h.buckets[len(h.buckets)-1].count)
return h.err
} }
bucketCounts[le] = intCount h.buckets = append(h.buckets, tempHistogramBucket{le: boundary, count: count})
} case h.buckets[len(h.buckets)-1].le == boundary:
return bucketCounts, nil // Ignore this, as it is a duplicate sample.
} default:
// Find the correct position to insert.
// ProcessUpperBoundsAndCreateBaseHistogram prepares an integer native i := sort.Search(len(h.buckets), func(i int) bool {
// histogram with custom buckets based on the provided upper bounds. return h.buckets[i].le >= boundary
// Everything is set except the bucket counts. })
// The sorted upper bounds are also returned. if h.buckets[i].le == boundary {
func ProcessUpperBoundsAndCreateBaseHistogram(upperBounds0 []float64, needsDedup bool) ([]float64, *histogram.Histogram) { // Ignore this, as it is a duplicate sample.
sort.Float64s(upperBounds0) return nil
var upperBounds []float64
if needsDedup {
upperBounds = make([]float64, 0, len(upperBounds0))
prevLE := math.Inf(-1)
for _, le := range upperBounds0 {
if le != prevLE {
upperBounds = append(upperBounds, le)
prevLE = le
}
} }
} else { if i > 0 && count < h.buckets[i-1].count {
upperBounds = upperBounds0 h.err = fmt.Errorf("%w: %g < %g", errCountNotCumulative, count, h.buckets[i-1].count)
} return h.err
var customBounds []float64
if upperBounds[len(upperBounds)-1] == math.Inf(1) {
customBounds = upperBounds[:len(upperBounds)-1]
} else {
customBounds = upperBounds
}
return upperBounds, &histogram.Histogram{
Count: 0,
Sum: 0,
Schema: histogram.CustomBucketsSchema,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: uint32(len(upperBounds))},
},
PositiveBuckets: make([]int64, len(upperBounds)),
CustomValues: customBounds,
}
}
// NewHistogram fills the bucket counts in the provided histogram.Histogram
// or histogram.FloatHistogram based on the provided temporary histogram and
// upper bounds.
func NewHistogram(histogram TempHistogram, upperBounds []float64, hBase *histogram.Histogram, fhBase *histogram.FloatHistogram) (*histogram.Histogram, *histogram.FloatHistogram) {
intBucketCounts, err := histogram.getIntBucketCounts()
if err != nil {
return nil, newFloatHistogram(histogram, upperBounds, histogram.BucketCounts, fhBase)
}
return newIntegerHistogram(histogram, upperBounds, intBucketCounts, hBase), nil
}
func newIntegerHistogram(histogram TempHistogram, upperBounds []float64, bucketCounts map[float64]int64, hBase *histogram.Histogram) *histogram.Histogram {
h := hBase.Copy()
absBucketCounts := make([]int64, len(h.PositiveBuckets))
var prevCount, total int64
for i, le := range upperBounds {
currCount, exists := bucketCounts[le]
if !exists {
currCount = 0
} }
count := currCount - prevCount if count > h.buckets[i].count {
absBucketCounts[i] = count h.err = fmt.Errorf("%w: %g > %g", errCountNotCumulative, count, h.buckets[i].count)
total += count return h.err
prevCount = currCount }
// Insert at the correct position unless duplicate.
h.buckets = append(h.buckets, tempHistogramBucket{})
copy(h.buckets[i+1:], h.buckets[i:])
h.buckets[i] = tempHistogramBucket{le: boundary, count: count}
} }
h.PositiveBuckets[0] = absBucketCounts[0] return nil
for i := 1; i < len(h.PositiveBuckets); i++ {
h.PositiveBuckets[i] = absBucketCounts[i] - absBucketCounts[i-1]
}
h.Sum = histogram.Sum
if histogram.Count != 0 {
total = int64(histogram.Count)
}
h.Count = uint64(total)
return h.Compact(0)
} }
func newFloatHistogram(histogram TempHistogram, upperBounds []float64, bucketCounts map[float64]float64, fhBase *histogram.FloatHistogram) *histogram.FloatHistogram { func (h *TempHistogram) SetCount(count float64) error {
fh := fhBase.Copy() if h.err != nil {
var prevCount, total float64 return h.err
for i, le := range upperBounds { }
currCount, exists := bucketCounts[le] if count < 0 {
if !exists { h.err = fmt.Errorf("%w: count=%g", errNegativeCount, count)
currCount = 0 return h.err
}
h.count = count
h.hasCount = true
return nil
}
func (h *TempHistogram) SetSum(sum float64) error {
if h.err != nil {
return h.err
}
h.sum = sum
return nil
}
func (h TempHistogram) Convert() (*histogram.Histogram, *histogram.FloatHistogram, error) {
if h.err != nil {
return nil, nil, h.err
}
if len(h.buckets) == 0 || h.buckets[len(h.buckets)-1].le != math.Inf(1) {
// No +Inf bucket.
if !h.hasCount && len(h.buckets) > 0 {
// No count either, so set count to the last known bucket's count.
h.count = h.buckets[len(h.buckets)-1].count
} }
count := currCount - prevCount // Let the last bucket be +Inf with the overall count.
fh.PositiveBuckets[i] = count h.buckets = append(h.buckets, tempHistogramBucket{le: math.Inf(1), count: h.count})
total += count
prevCount = currCount
} }
fh.Sum = histogram.Sum
if histogram.Count != 0 { if !h.hasCount {
total = histogram.Count h.count = h.buckets[len(h.buckets)-1].count
h.hasCount = true
} }
fh.Count = total
return fh.Compact(0) for _, b := range h.buckets {
intCount := int64(math.Round(b.count))
if b.count != float64(intCount) {
return h.convertToFloatHistogram()
}
}
intCount := uint64(math.Round(h.count))
if h.count != float64(intCount) {
return h.convertToFloatHistogram()
}
return h.convertToIntegerHistogram(intCount)
}
func (h TempHistogram) convertToIntegerHistogram(count uint64) (*histogram.Histogram, *histogram.FloatHistogram, error) {
rh := &histogram.Histogram{
Schema: histogram.CustomBucketsSchema,
Count: count,
Sum: h.sum,
PositiveSpans: []histogram.Span{{Length: uint32(len(h.buckets))}},
PositiveBuckets: make([]int64, len(h.buckets)),
}
if len(h.buckets) > 1 {
rh.CustomValues = make([]float64, len(h.buckets)-1) // Not storing the last +Inf bucket.
}
prevCount := int64(0)
prevDelta := int64(0)
for i, b := range h.buckets {
// delta is the actual bucket count as the input is cumulative.
delta := int64(b.count) - prevCount
rh.PositiveBuckets[i] = delta - prevDelta
prevCount = int64(b.count)
prevDelta = delta
if b.le != math.Inf(1) {
rh.CustomValues[i] = b.le
}
}
if count != uint64(h.buckets[len(h.buckets)-1].count) {
h.err = fmt.Errorf("%w: count=%d != le=%g count=%g", errCountMismatch, count, h.buckets[len(h.buckets)-1].le, h.buckets[len(h.buckets)-1].count)
return nil, nil, h.err
}
return rh.Compact(2), nil, nil
}
func (h TempHistogram) convertToFloatHistogram() (*histogram.Histogram, *histogram.FloatHistogram, error) {
rh := &histogram.FloatHistogram{
Schema: histogram.CustomBucketsSchema,
Count: h.count,
Sum: h.sum,
PositiveSpans: []histogram.Span{{Length: uint32(len(h.buckets))}},
PositiveBuckets: make([]float64, len(h.buckets)),
}
if len(h.buckets) > 1 {
rh.CustomValues = make([]float64, len(h.buckets)-1) // Not storing the last +Inf bucket.
}
prevCount := 0.0
for i, b := range h.buckets {
rh.PositiveBuckets[i] = b.count - prevCount
prevCount = b.count
if b.le != math.Inf(1) {
rh.CustomValues[i] = b.le
}
}
if h.count != h.buckets[len(h.buckets)-1].count {
h.err = fmt.Errorf("%w: count=%g != le=%g count=%g", errCountMismatch, h.count, h.buckets[len(h.buckets)-1].le, h.buckets[len(h.buckets)-1].count)
return nil, nil, h.err
}
return nil, rh.Compact(0), nil
} }
func GetHistogramMetricBase(m labels.Labels, suffix string) labels.Labels { func GetHistogramMetricBase(m labels.Labels, suffix string) labels.Labels {

View file

@ -0,0 +1,189 @@
// Copyright 2024 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package convertnhcb
import (
"math"
"testing"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/model/histogram"
)
func TestNHCBConvert(t *testing.T) {
tests := map[string]struct {
setup func() *TempHistogram
expectedErr error
expectedH *histogram.Histogram
expectedFH *histogram.FloatHistogram
}{
"empty": {
setup: func() *TempHistogram {
h := NewTempHistogram()
return &h
},
expectedH: &histogram.Histogram{
Schema: histogram.CustomBucketsSchema,
PositiveSpans: []histogram.Span{},
PositiveBuckets: []int64{},
},
},
"sum only": {
setup: func() *TempHistogram {
h := NewTempHistogram()
h.SetSum(1000.25)
return &h
},
expectedH: &histogram.Histogram{
Schema: histogram.CustomBucketsSchema,
Sum: 1000.25,
PositiveSpans: []histogram.Span{},
PositiveBuckets: []int64{},
},
},
"single integer bucket": {
setup: func() *TempHistogram {
h := NewTempHistogram()
h.SetSum(1000.25)
h.SetBucketCount(0.5, 1000)
return &h
},
expectedH: &histogram.Histogram{
Schema: histogram.CustomBucketsSchema,
Count: 1000,
Sum: 1000.25,
PositiveSpans: []histogram.Span{{Length: 1}},
PositiveBuckets: []int64{1000},
CustomValues: []float64{0.5},
},
},
"single float bucket": {
setup: func() *TempHistogram {
h := NewTempHistogram()
h.SetSum(1000.25)
h.SetBucketCount(0.5, 1337.42)
return &h
},
expectedFH: &histogram.FloatHistogram{
Schema: histogram.CustomBucketsSchema,
Count: 1337.42,
Sum: 1000.25,
PositiveSpans: []histogram.Span{{Length: 1}},
PositiveBuckets: []float64{1337.42},
CustomValues: []float64{0.5},
},
},
"happy case integer bucket": {
setup: func() *TempHistogram {
h := NewTempHistogram()
h.SetCount(1000)
h.SetSum(1000.25)
h.SetBucketCount(0.5, 50)
h.SetBucketCount(1.0, 950)
h.SetBucketCount(math.Inf(1), 1000)
return &h
},
expectedH: &histogram.Histogram{
Schema: histogram.CustomBucketsSchema,
Count: 1000,
Sum: 1000.25,
PositiveSpans: []histogram.Span{{Length: 3}},
PositiveBuckets: []int64{50, 850, -850},
CustomValues: []float64{0.5, 1.0},
},
},
"happy case float bucket": {
setup: func() *TempHistogram {
h := NewTempHistogram()
h.SetCount(1000)
h.SetSum(1000.25)
h.SetBucketCount(0.5, 50)
h.SetBucketCount(1.0, 950.5)
h.SetBucketCount(math.Inf(1), 1000)
return &h
},
expectedFH: &histogram.FloatHistogram{
Schema: histogram.CustomBucketsSchema,
Count: 1000,
Sum: 1000.25,
PositiveSpans: []histogram.Span{{Length: 3}},
PositiveBuckets: []float64{50, 900.5, 49.5},
CustomValues: []float64{0.5, 1.0},
},
},
"non cumulative bucket": {
setup: func() *TempHistogram {
h := NewTempHistogram()
h.SetCount(1000)
h.SetSum(1000.25)
h.SetBucketCount(0.5, 50)
h.SetBucketCount(1.0, 950)
h.SetBucketCount(math.Inf(1), 900)
return &h
},
expectedErr: errCountNotCumulative,
},
"negative count": {
setup: func() *TempHistogram {
h := NewTempHistogram()
h.SetCount(-1000)
h.SetSum(1000.25)
h.SetBucketCount(0.5, 50)
h.SetBucketCount(1.0, 950)
h.SetBucketCount(math.Inf(1), 900)
return &h
},
expectedErr: errNegativeCount,
},
"mixed order": {
setup: func() *TempHistogram {
h := NewTempHistogram()
h.SetBucketCount(0.5, 50)
h.SetBucketCount(math.Inf(1), 1000)
h.SetBucketCount(1.0, 950)
h.SetCount(1000)
h.SetSum(1000.25)
return &h
},
expectedH: &histogram.Histogram{
Schema: histogram.CustomBucketsSchema,
Count: 1000,
Sum: 1000.25,
PositiveSpans: []histogram.Span{{Length: 3}},
PositiveBuckets: []int64{50, 850, -850},
CustomValues: []float64{0.5, 1.0},
},
},
}
for name, test := range tests {
t.Run(name, func(t *testing.T) {
th := test.setup()
h, fh, err := th.Convert()
if test.expectedErr != nil {
require.ErrorIs(t, err, test.expectedErr)
return
}
require.Equal(t, test.expectedH, h)
if h != nil {
require.NoError(t, h.Validate())
}
require.Equal(t, test.expectedFH, fh)
if fh != nil {
require.NoError(t, fh.Validate())
}
})
}
}