Merge pull request #13662 from prometheus/nhcb

Native histograms custom buckets storage

model/histogram/float_histogram.go

@@ -30,11 +30,12 @@ import (
 type FloatHistogram struct {
 	// Counter reset information.
 	CounterResetHint CounterResetHint
-	// Currently valid schema numbers are -4 <= n <= 8.  They are all for
-	// base-2 bucket schemas, where 1 is a bucket boundary in each case, and
-	// then each power of two is divided into 2^n logarithmic buckets.  Or
-	// in other words, each bucket boundary is the previous boundary times
-	// 2^(2^-n).
+	// Currently valid schema numbers are -4 <= n <= 8 for exponential buckets.
+	// They are all for base-2 bucket schemas, where 1 is a bucket boundary in
+	// each case, and then each power of two is divided into 2^n logarithmic buckets.
+	// Or in other words, each bucket boundary is the previous boundary times
+	// 2^(2^-n). Another valid schema number is -53 for custom buckets, defined by
+	// the CustomValues field.
 	Schema int32
 	// Width of the zero bucket.
 	ZeroThreshold float64
@@ -49,6 +50,16 @@ type FloatHistogram struct {
 	// Observation counts in buckets. Each represents an absolute count and
 	// must be zero or positive.
 	PositiveBuckets, NegativeBuckets []float64
+	// Holds the custom (usually upper) bounds for bucket definitions, otherwise nil.
+	// This slice is interned, to be treated as immutable and copied by reference.
+	// These numbers should be strictly increasing. This field is only used when the
+	// schema is for custom buckets, and the ZeroThreshold, ZeroCount, NegativeSpans
+	// and NegativeBuckets fields are not used in that case.
+	CustomValues []float64
+}
+
+func (h *FloatHistogram) UsesCustomBuckets() bool {
+	return IsCustomBucketsSchema(h.Schema)
 }
 
 // Copy returns a deep copy of the Histogram.
@@ -56,28 +67,37 @@ func (h *FloatHistogram) Copy() *FloatHistogram {
 	c := FloatHistogram{
 		CounterResetHint: h.CounterResetHint,
 		Schema:           h.Schema,
-		ZeroThreshold:    h.ZeroThreshold,
-		ZeroCount:        h.ZeroCount,
 		Count:            h.Count,
 		Sum:              h.Sum,
 	}
 
+	if h.UsesCustomBuckets() {
+		if len(h.CustomValues) != 0 {
+			c.CustomValues = make([]float64, len(h.CustomValues))
+			copy(c.CustomValues, h.CustomValues)
+		}
+	} else {
+		c.ZeroThreshold = h.ZeroThreshold
+		c.ZeroCount = h.ZeroCount
+
+		if len(h.NegativeSpans) != 0 {
+			c.NegativeSpans = make([]Span, len(h.NegativeSpans))
+			copy(c.NegativeSpans, h.NegativeSpans)
+		}
+		if len(h.NegativeBuckets) != 0 {
+			c.NegativeBuckets = make([]float64, len(h.NegativeBuckets))
+			copy(c.NegativeBuckets, h.NegativeBuckets)
+		}
+	}
+
 	if len(h.PositiveSpans) != 0 {
 		c.PositiveSpans = make([]Span, len(h.PositiveSpans))
 		copy(c.PositiveSpans, h.PositiveSpans)
 	}
-	if len(h.NegativeSpans) != 0 {
-		c.NegativeSpans = make([]Span, len(h.NegativeSpans))
-		copy(c.NegativeSpans, h.NegativeSpans)
-	}
 	if len(h.PositiveBuckets) != 0 {
 		c.PositiveBuckets = make([]float64, len(h.PositiveBuckets))
 		copy(c.PositiveBuckets, h.PositiveBuckets)
 	}
-	if len(h.NegativeBuckets) != 0 {
-		c.NegativeBuckets = make([]float64, len(h.NegativeBuckets))
-		copy(c.NegativeBuckets, h.NegativeBuckets)
-	}
 
 	return &c
 }
@@ -87,32 +107,53 @@ func (h *FloatHistogram) Copy() *FloatHistogram {
 func (h *FloatHistogram) CopyTo(to *FloatHistogram) {
 	to.CounterResetHint = h.CounterResetHint
 	to.Schema = h.Schema
-	to.ZeroThreshold = h.ZeroThreshold
-	to.ZeroCount = h.ZeroCount
 	to.Count = h.Count
 	to.Sum = h.Sum
 
+	if h.UsesCustomBuckets() {
+		to.ZeroThreshold = 0
+		to.ZeroCount = 0
+
+		to.NegativeSpans = clearIfNotNil(to.NegativeSpans)
+		to.NegativeBuckets = clearIfNotNil(to.NegativeBuckets)
+
+		to.CustomValues = resize(to.CustomValues, len(h.CustomValues))
+		copy(to.CustomValues, h.CustomValues)
+	} else {
+		to.ZeroThreshold = h.ZeroThreshold
+		to.ZeroCount = h.ZeroCount
+
+		to.NegativeSpans = resize(to.NegativeSpans, len(h.NegativeSpans))
+		copy(to.NegativeSpans, h.NegativeSpans)
+
+		to.NegativeBuckets = resize(to.NegativeBuckets, len(h.NegativeBuckets))
+		copy(to.NegativeBuckets, h.NegativeBuckets)
+
+		to.CustomValues = clearIfNotNil(to.CustomValues)
+	}
+
 	to.PositiveSpans = resize(to.PositiveSpans, len(h.PositiveSpans))
 	copy(to.PositiveSpans, h.PositiveSpans)
 
-	to.NegativeSpans = resize(to.NegativeSpans, len(h.NegativeSpans))
-	copy(to.NegativeSpans, h.NegativeSpans)
-
 	to.PositiveBuckets = resize(to.PositiveBuckets, len(h.PositiveBuckets))
 	copy(to.PositiveBuckets, h.PositiveBuckets)
-
-	to.NegativeBuckets = resize(to.NegativeBuckets, len(h.NegativeBuckets))
-	copy(to.NegativeBuckets, h.NegativeBuckets)
 }
 
 // CopyToSchema works like Copy, but the returned deep copy has the provided
 // target schema, which must be ≤ the original schema (i.e. it must have a lower
-// resolution).
+// resolution). This method panics if a custom buckets schema is used in the
+// receiving FloatHistogram or as the provided targetSchema.
 func (h *FloatHistogram) CopyToSchema(targetSchema int32) *FloatHistogram {
 	if targetSchema == h.Schema {
 		// Fast path.
 		return h.Copy()
 	}
+	if h.UsesCustomBuckets() {
+		panic(fmt.Errorf("cannot reduce resolution to %d when there are custom buckets", targetSchema))
+	}
+	if IsCustomBucketsSchema(targetSchema) {
+		panic("cannot reduce resolution to custom buckets schema")
+	}
 	if targetSchema > h.Schema {
 		panic(fmt.Errorf("cannot copy from schema %d to %d", h.Schema, targetSchema))
 	}
@@ -185,6 +226,9 @@ func (h *FloatHistogram) TestExpression() string {
 	if m.ZeroThreshold != 0 {
 		res = append(res, fmt.Sprintf("z_bucket_w:%g", m.ZeroThreshold))
 	}
+	if m.UsesCustomBuckets() {
+		res = append(res, fmt.Sprintf("custom_values:%g", m.CustomValues))
+	}
 
 	addBuckets := func(kind, bucketsKey, offsetKey string, buckets []float64, spans []Span) []string {
 		if len(spans) > 1 {
@@ -210,14 +254,18 @@ func (h *FloatHistogram) TestExpression() string {
 	return "{{" + strings.Join(res, " ") + "}}"
 }
 
-// ZeroBucket returns the zero bucket.
+// ZeroBucket returns the zero bucket. This method panics if the schema is for custom buckets.
 func (h *FloatHistogram) ZeroBucket() Bucket[float64] {
+	if h.UsesCustomBuckets() {
+		panic("histograms with custom buckets have no zero bucket")
+	}
 	return Bucket[float64]{
 		Lower:          -h.ZeroThreshold,
 		Upper:          h.ZeroThreshold,
 		LowerInclusive: true,
 		UpperInclusive: true,
 		Count:          h.ZeroCount,
+		// Index is irrelevant for the zero bucket.
 	}
 }
 
@@ -263,9 +311,18 @@ func (h *FloatHistogram) Div(scalar float64) *FloatHistogram {
 //
 // The method reconciles differences in the zero threshold and in the schema, and
 // changes them if needed. The other histogram will not be modified in any case.
+// Adding is currently only supported between 2 exponential histograms, or between
+// 2 custom buckets histograms with the exact same custom bounds.
 //
 // This method returns a pointer to the receiving histogram for convenience.
-func (h *FloatHistogram) Add(other *FloatHistogram) *FloatHistogram {
+func (h *FloatHistogram) Add(other *FloatHistogram) (*FloatHistogram, error) {
+	if h.UsesCustomBuckets() != other.UsesCustomBuckets() {
+		return nil, ErrHistogramsIncompatibleSchema
+	}
+	if h.UsesCustomBuckets() && !FloatBucketsMatch(h.CustomValues, other.CustomValues) {
+		return nil, ErrHistogramsIncompatibleBounds
+	}
+
 	switch {
 	case other.CounterResetHint == h.CounterResetHint:
 		// Adding apples to apples, all good. No need to change anything.
@@ -290,19 +347,28 @@ func (h *FloatHistogram) Add(other *FloatHistogram) *FloatHistogram {
 		// TODO(trevorwhitney): Actually issue the warning as soon as the plumbing for it is in place
 	}
 
-	otherZeroCount := h.reconcileZeroBuckets(other)
-	h.ZeroCount += otherZeroCount
+	if !h.UsesCustomBuckets() {
+		otherZeroCount := h.reconcileZeroBuckets(other)
+		h.ZeroCount += otherZeroCount
+	}
 	h.Count += other.Count
 	h.Sum += other.Sum
 
 	var (
-		hPositiveSpans   = h.PositiveSpans
-		hPositiveBuckets = h.PositiveBuckets
-		hNegativeSpans   = h.NegativeSpans
-		hNegativeBuckets = h.NegativeBuckets
-
+		hPositiveSpans       = h.PositiveSpans
+		hPositiveBuckets     = h.PositiveBuckets
 		otherPositiveSpans   = other.PositiveSpans
 		otherPositiveBuckets = other.PositiveBuckets
+	)
+
+	if h.UsesCustomBuckets() {
+		h.PositiveSpans, h.PositiveBuckets = addBuckets(h.Schema, h.ZeroThreshold, false, hPositiveSpans, hPositiveBuckets, otherPositiveSpans, otherPositiveBuckets)
+		return h, nil
+	}
+
+	var (
+		hNegativeSpans       = h.NegativeSpans
+		hNegativeBuckets     = h.NegativeBuckets
 		otherNegativeSpans   = other.NegativeSpans
 		otherNegativeBuckets = other.NegativeBuckets
 	)
@@ -321,24 +387,40 @@ func (h *FloatHistogram) Add(other *FloatHistogram) *FloatHistogram {
 	h.PositiveSpans, h.PositiveBuckets = addBuckets(h.Schema, h.ZeroThreshold, false, hPositiveSpans, hPositiveBuckets, otherPositiveSpans, otherPositiveBuckets)
 	h.NegativeSpans, h.NegativeBuckets = addBuckets(h.Schema, h.ZeroThreshold, false, hNegativeSpans, hNegativeBuckets, otherNegativeSpans, otherNegativeBuckets)
 
-	return h
+	return h, nil
 }
 
 // Sub works like Add but subtracts the other histogram.
-func (h *FloatHistogram) Sub(other *FloatHistogram) *FloatHistogram {
-	otherZeroCount := h.reconcileZeroBuckets(other)
-	h.ZeroCount -= otherZeroCount
+func (h *FloatHistogram) Sub(other *FloatHistogram) (*FloatHistogram, error) {
+	if h.UsesCustomBuckets() != other.UsesCustomBuckets() {
+		return nil, ErrHistogramsIncompatibleSchema
+	}
+	if h.UsesCustomBuckets() && !FloatBucketsMatch(h.CustomValues, other.CustomValues) {
+		return nil, ErrHistogramsIncompatibleBounds
+	}
+
+	if !h.UsesCustomBuckets() {
+		otherZeroCount := h.reconcileZeroBuckets(other)
+		h.ZeroCount -= otherZeroCount
+	}
 	h.Count -= other.Count
 	h.Sum -= other.Sum
 
 	var (
-		hPositiveSpans   = h.PositiveSpans
-		hPositiveBuckets = h.PositiveBuckets
-		hNegativeSpans   = h.NegativeSpans
-		hNegativeBuckets = h.NegativeBuckets
-
+		hPositiveSpans       = h.PositiveSpans
+		hPositiveBuckets     = h.PositiveBuckets
 		otherPositiveSpans   = other.PositiveSpans
 		otherPositiveBuckets = other.PositiveBuckets
+	)
+
+	if h.UsesCustomBuckets() {
+		h.PositiveSpans, h.PositiveBuckets = addBuckets(h.Schema, h.ZeroThreshold, true, hPositiveSpans, hPositiveBuckets, otherPositiveSpans, otherPositiveBuckets)
+		return h, nil
+	}
+
+	var (
+		hNegativeSpans       = h.NegativeSpans
+		hNegativeBuckets     = h.NegativeBuckets
 		otherNegativeSpans   = other.NegativeSpans
 		otherNegativeBuckets = other.NegativeBuckets
 	)
@@ -356,7 +438,7 @@ func (h *FloatHistogram) Sub(other *FloatHistogram) *FloatHistogram {
 	h.PositiveSpans, h.PositiveBuckets = addBuckets(h.Schema, h.ZeroThreshold, true, hPositiveSpans, hPositiveBuckets, otherPositiveSpans, otherPositiveBuckets)
 	h.NegativeSpans, h.NegativeBuckets = addBuckets(h.Schema, h.ZeroThreshold, true, hNegativeSpans, hNegativeBuckets, otherNegativeSpans, otherNegativeBuckets)
 
-	return h
+	return h, nil
 }
 
 // Equals returns true if the given float histogram matches exactly.
@@ -365,29 +447,42 @@ func (h *FloatHistogram) Sub(other *FloatHistogram) *FloatHistogram {
 // but they must represent the same bucket layout to match.
 // Sum, Count, ZeroCount and bucket values are compared based on their bit patterns
 // because this method is about data equality rather than mathematical equality.
+// We ignore fields that are not used based on the exponential / custom buckets schema,
+// but check fields where differences may cause unintended behaviour even if they are not
+// supposed to be used according to the schema.
 func (h *FloatHistogram) Equals(h2 *FloatHistogram) bool {
 	if h2 == nil {
 		return false
 	}
 
-	if h.Schema != h2.Schema || h.ZeroThreshold != h2.ZeroThreshold ||
-		math.Float64bits(h.ZeroCount) != math.Float64bits(h2.ZeroCount) ||
+	if h.Schema != h2.Schema ||
 		math.Float64bits(h.Count) != math.Float64bits(h2.Count) ||
 		math.Float64bits(h.Sum) != math.Float64bits(h2.Sum) {
 		return false
 	}
 
+	if h.UsesCustomBuckets() {
+		if !FloatBucketsMatch(h.CustomValues, h2.CustomValues) {
+			return false
+		}
+	}
+
+	if h.ZeroThreshold != h2.ZeroThreshold ||
+		math.Float64bits(h.ZeroCount) != math.Float64bits(h2.ZeroCount) {
+		return false
+	}
+
+	if !spansMatch(h.NegativeSpans, h2.NegativeSpans) {
+		return false
+	}
+	if !FloatBucketsMatch(h.NegativeBuckets, h2.NegativeBuckets) {
+		return false
+	}
+
 	if !spansMatch(h.PositiveSpans, h2.PositiveSpans) {
 		return false
 	}
-	if !spansMatch(h.NegativeSpans, h2.NegativeSpans) {
-		return false
-	}
-
-	if !floatBucketsMatch(h.PositiveBuckets, h2.PositiveBuckets) {
-		return false
-	}
-	if !floatBucketsMatch(h.NegativeBuckets, h2.NegativeBuckets) {
+	if !FloatBucketsMatch(h.PositiveBuckets, h2.PositiveBuckets) {
 		return false
 	}
 
@@ -403,6 +498,7 @@ func (h *FloatHistogram) Size() int {
 	negSpanSize := len(h.NegativeSpans) * 8     // 8 bytes (int32 + uint32).
 	posBucketSize := len(h.PositiveBuckets) * 8 // 8 bytes (float64).
 	negBucketSize := len(h.NegativeBuckets) * 8 // 8 bytes (float64).
+	customBoundSize := len(h.CustomValues) * 8  // 8 bytes (float64).
 
 	// Total size of the struct.
 
@@ -417,9 +513,10 @@ func (h *FloatHistogram) Size() int {
 	// fh.NegativeSpans is 24 bytes.
 	// fh.PositiveBuckets is 24 bytes.
 	// fh.NegativeBuckets is 24 bytes.
-	structSize := 144
+	// fh.CustomValues is 24 bytes.
+	structSize := 168
 
-	return structSize + posSpanSize + negSpanSize + posBucketSize + negBucketSize
+	return structSize + posSpanSize + negSpanSize + posBucketSize + negBucketSize + customBoundSize
 }
 
 // Compact eliminates empty buckets at the beginning and end of each span, then
@@ -504,6 +601,12 @@ func (h *FloatHistogram) DetectReset(previous *FloatHistogram) bool {
 	if h.Count < previous.Count {
 		return true
 	}
+	if h.UsesCustomBuckets() != previous.UsesCustomBuckets() || (h.UsesCustomBuckets() && !FloatBucketsMatch(h.CustomValues, previous.CustomValues)) {
+		// Mark that something has changed or that the application has been restarted. However, this does
+		// not matter so much since the change in schema will be handled directly in the chunks and PromQL
+		// functions.
+		return true
+	}
 	if h.Schema > previous.Schema {
 		return true
 	}
@@ -609,7 +712,7 @@ func (h *FloatHistogram) NegativeBucketIterator() BucketIterator[float64] {
 // positive buckets in descending order (starting at the highest bucket and
 // going down towards the zero bucket).
 func (h *FloatHistogram) PositiveReverseBucketIterator() BucketIterator[float64] {
-	it := newReverseFloatBucketIterator(h.PositiveSpans, h.PositiveBuckets, h.Schema, true)
+	it := newReverseFloatBucketIterator(h.PositiveSpans, h.PositiveBuckets, h.Schema, true, h.CustomValues)
 	return &it
 }
 
@@ -617,7 +720,7 @@ func (h *FloatHistogram) PositiveReverseBucketIterator() BucketIterator[float64]
 // negative buckets in ascending order (starting at the lowest bucket and going
 // up towards the zero bucket).
 func (h *FloatHistogram) NegativeReverseBucketIterator() BucketIterator[float64] {
-	it := newReverseFloatBucketIterator(h.NegativeSpans, h.NegativeBuckets, h.Schema, false)
+	it := newReverseFloatBucketIterator(h.NegativeSpans, h.NegativeBuckets, h.Schema, false, nil)
 	return &it
 }
 
@@ -629,7 +732,7 @@ func (h *FloatHistogram) NegativeReverseBucketIterator() BucketIterator[float64]
 func (h *FloatHistogram) AllBucketIterator() BucketIterator[float64] {
 	return &allFloatBucketIterator{
 		h:         h,
-		leftIter:  newReverseFloatBucketIterator(h.NegativeSpans, h.NegativeBuckets, h.Schema, false),
+		leftIter:  newReverseFloatBucketIterator(h.NegativeSpans, h.NegativeBuckets, h.Schema, false, nil),
 		rightIter: h.floatBucketIterator(true, 0, h.Schema),
 		state:     -1,
 	}
@@ -643,30 +746,52 @@ func (h *FloatHistogram) AllBucketIterator() BucketIterator[float64] {
 func (h *FloatHistogram) AllReverseBucketIterator() BucketIterator[float64] {
 	return &allFloatBucketIterator{
 		h:         h,
-		leftIter:  newReverseFloatBucketIterator(h.PositiveSpans, h.PositiveBuckets, h.Schema, true),
+		leftIter:  newReverseFloatBucketIterator(h.PositiveSpans, h.PositiveBuckets, h.Schema, true, h.CustomValues),
 		rightIter: h.floatBucketIterator(false, 0, h.Schema),
 		state:     -1,
 	}
 }
 
 // Validate validates consistency between span and bucket slices. Also, buckets are checked
-// against negative values.
+// against negative values. We check to make sure there are no unexpected fields or field values
+// based on the exponential / custom buckets schema.
 // We do not check for h.Count being at least as large as the sum of the
 // counts in the buckets because floating point precision issues can
 // create false positives here.
 func (h *FloatHistogram) Validate() error {
-	if err := checkHistogramSpans(h.NegativeSpans, len(h.NegativeBuckets)); err != nil {
-		return fmt.Errorf("negative side: %w", err)
-	}
-	if err := checkHistogramSpans(h.PositiveSpans, len(h.PositiveBuckets)); err != nil {
-		return fmt.Errorf("positive side: %w", err)
-	}
 	var nCount, pCount float64
-	err := checkHistogramBuckets(h.NegativeBuckets, &nCount, false)
-	if err != nil {
-		return fmt.Errorf("negative side: %w", err)
+	if h.UsesCustomBuckets() {
+		if err := checkHistogramCustomBounds(h.CustomValues, h.PositiveSpans, len(h.PositiveBuckets)); err != nil {
+			return fmt.Errorf("custom buckets: %w", err)
+		}
+		if h.ZeroCount != 0 {
+			return fmt.Errorf("custom buckets: must have zero count of 0")
+		}
+		if h.ZeroThreshold != 0 {
+			return fmt.Errorf("custom buckets: must have zero threshold of 0")
+		}
+		if len(h.NegativeSpans) > 0 {
+			return fmt.Errorf("custom buckets: must not have negative spans")
+		}
+		if len(h.NegativeBuckets) > 0 {
+			return fmt.Errorf("custom buckets: must not have negative buckets")
+		}
+	} else {
+		if err := checkHistogramSpans(h.PositiveSpans, len(h.PositiveBuckets)); err != nil {
+			return fmt.Errorf("positive side: %w", err)
+		}
+		if err := checkHistogramSpans(h.NegativeSpans, len(h.NegativeBuckets)); err != nil {
+			return fmt.Errorf("negative side: %w", err)
+		}
+		err := checkHistogramBuckets(h.NegativeBuckets, &nCount, false)
+		if err != nil {
+			return fmt.Errorf("negative side: %w", err)
+		}
+		if h.CustomValues != nil {
+			return fmt.Errorf("histogram with exponential schema must not have custom bounds")
+		}
 	}
-	err = checkHistogramBuckets(h.PositiveBuckets, &pCount, false)
+	err := checkHistogramBuckets(h.PositiveBuckets, &pCount, false)
 	if err != nil {
 		return fmt.Errorf("positive side: %w", err)
 	}
@@ -790,17 +915,25 @@ func (h *FloatHistogram) reconcileZeroBuckets(other *FloatHistogram) float64 {
 // If positive is true, the returned iterator iterates through the positive
 // buckets, otherwise through the negative buckets.
 //
-// If absoluteStartValue is < the lowest absolute value of any upper bucket
-// boundary, the iterator starts with the first bucket. Otherwise, it will skip
-// all buckets with an absolute value of their upper boundary ≤
-// absoluteStartValue.
+// Only for exponential schemas, if absoluteStartValue is < the lowest absolute
+// value of any upper bucket boundary, the iterator starts with the first bucket.
+// Otherwise, it will skip all buckets with an absolute value of their upper boundary ≤
+// absoluteStartValue. For custom bucket schemas, absoluteStartValue is ignored and
+// no buckets are skipped.
 //
 // targetSchema must be ≤ the schema of FloatHistogram (and of course within the
 // legal values for schemas in general). The buckets are merged to match the
-// targetSchema prior to iterating (without mutating FloatHistogram).
+// targetSchema prior to iterating (without mutating FloatHistogram), but custom buckets
+// schemas cannot be merged with other schemas.
 func (h *FloatHistogram) floatBucketIterator(
 	positive bool, absoluteStartValue float64, targetSchema int32,
 ) floatBucketIterator {
+	if h.UsesCustomBuckets() && targetSchema != h.Schema {
+		panic(fmt.Errorf("cannot merge from custom buckets schema to exponential schema"))
+	}
+	if !h.UsesCustomBuckets() && IsCustomBucketsSchema(targetSchema) {
+		panic(fmt.Errorf("cannot merge from exponential buckets schema to custom schema"))
+	}
 	if targetSchema > h.Schema {
 		panic(fmt.Errorf("cannot merge from schema %d to %d", h.Schema, targetSchema))
 	}
@@ -816,6 +949,7 @@ func (h *FloatHistogram) floatBucketIterator(
 	if positive {
 		i.spans = h.PositiveSpans
 		i.buckets = h.PositiveBuckets
+		i.customValues = h.CustomValues
 	} else {
 		i.spans = h.NegativeSpans
 		i.buckets = h.NegativeBuckets
@@ -825,14 +959,15 @@ func (h *FloatHistogram) floatBucketIterator(
 
 // reverseFloatBucketIterator is a low-level constructor for reverse bucket iterators.
 func newReverseFloatBucketIterator(
-	spans []Span, buckets []float64, schema int32, positive bool,
+	spans []Span, buckets []float64, schema int32, positive bool, customValues []float64,
 ) reverseFloatBucketIterator {
 	r := reverseFloatBucketIterator{
 		baseBucketIterator: baseBucketIterator[float64, float64]{
-			schema:   schema,
-			spans:    spans,
-			buckets:  buckets,
-			positive: positive,
+			schema:       schema,
+			spans:        spans,
+			buckets:      buckets,
+			positive:     positive,
+			customValues: customValues,
 		},
 	}
 
@@ -946,9 +1081,9 @@ func (i *floatBucketIterator) Next() bool {
 		}
 	}
 
-	// Skip buckets before absoluteStartValue.
+	// Skip buckets before absoluteStartValue for exponential schemas.
 	// TODO(beorn7): Maybe do something more efficient than this recursive call.
-	if !i.boundReachedStartValue && getBound(i.currIdx, i.targetSchema) <= i.absoluteStartValue {
+	if !i.boundReachedStartValue && IsExponentialSchema(i.targetSchema) && getBoundExponential(i.currIdx, i.targetSchema) <= i.absoluteStartValue {
 		return i.Next()
 	}
 	i.boundReachedStartValue = true
@@ -1010,14 +1145,7 @@ func (i *allFloatBucketIterator) Next() bool {
 	case 0:
 		i.state = 1
 		if i.h.ZeroCount > 0 {
-			i.currBucket = Bucket[float64]{
-				Lower:          -i.h.ZeroThreshold,
-				Upper:          i.h.ZeroThreshold,
-				LowerInclusive: true,
-				UpperInclusive: true,
-				Count:          i.h.ZeroCount,
-				// Index is irrelevant for the zero bucket.
-			}
+			i.currBucket = i.h.ZeroBucket()
 			return true
 		}
 		return i.Next()
@@ -1076,7 +1204,7 @@ func addBuckets(
 	for _, spanB := range spansB {
 		indexB += spanB.Offset
 		for j := 0; j < int(spanB.Length); j++ {
-			if lowerThanThreshold && getBound(indexB, schema) <= threshold {
+			if lowerThanThreshold && IsExponentialSchema(schema) && getBoundExponential(indexB, schema) <= threshold {
 				goto nextLoop
 			}
 			lowerThanThreshold = false
@@ -1177,7 +1305,7 @@ func addBuckets(
 	return spansA, bucketsA
 }
 
-func floatBucketsMatch(b1, b2 []float64) bool {
+func FloatBucketsMatch(b1, b2 []float64) bool {
 	if len(b1) != len(b2) {
 		return false
 	}
@@ -1191,7 +1319,15 @@ func floatBucketsMatch(b1, b2 []float64) bool {
 
 // ReduceResolution reduces the float histogram's spans, buckets into target schema.
 // The target schema must be smaller than the current float histogram's schema.
+// This will panic if the histogram has custom buckets or if the target schema is
+// a custom buckets schema.
 func (h *FloatHistogram) ReduceResolution(targetSchema int32) *FloatHistogram {
+	if h.UsesCustomBuckets() {
+		panic("cannot reduce resolution when there are custom buckets")
+	}
+	if IsCustomBucketsSchema(targetSchema) {
+		panic("cannot reduce resolution to custom buckets schema")
+	}
 	if targetSchema >= h.Schema {
 		panic(fmt.Errorf("cannot reduce resolution from schema %d to %d", h.Schema, targetSchema))
 	}

model/histogram/generic.go

@@ -20,14 +20,33 @@ import (
 	"strings"
 )
 
-var (
-	ErrHistogramCountNotBigEnough    = errors.New("histogram's observation count should be at least the number of observations found in the buckets")
-	ErrHistogramCountMismatch        = errors.New("histogram's observation count should equal the number of observations found in the buckets (in absence of NaN)")
-	ErrHistogramNegativeBucketCount  = errors.New("histogram has a bucket whose observation count is negative")
-	ErrHistogramSpanNegativeOffset   = errors.New("histogram has a span whose offset is negative")
-	ErrHistogramSpansBucketsMismatch = errors.New("histogram spans specify different number of buckets than provided")
+const (
+	ExponentialSchemaMax int32 = 8
+	ExponentialSchemaMin int32 = -4
+	CustomBucketsSchema  int32 = -53
 )
 
+var (
+	ErrHistogramCountNotBigEnough     = errors.New("histogram's observation count should be at least the number of observations found in the buckets")
+	ErrHistogramCountMismatch         = errors.New("histogram's observation count should equal the number of observations found in the buckets (in absence of NaN)")
+	ErrHistogramNegativeBucketCount   = errors.New("histogram has a bucket whose observation count is negative")
+	ErrHistogramSpanNegativeOffset    = errors.New("histogram has a span whose offset is negative")
+	ErrHistogramSpansBucketsMismatch  = errors.New("histogram spans specify different number of buckets than provided")
+	ErrHistogramCustomBucketsMismatch = errors.New("histogram custom bounds are too few")
+	ErrHistogramCustomBucketsInvalid  = errors.New("histogram custom bounds must be in strictly increasing order")
+	ErrHistogramCustomBucketsInfinite = errors.New("histogram custom bounds must be finite")
+	ErrHistogramsIncompatibleSchema   = errors.New("cannot apply this operation on histograms with a mix of exponential and custom bucket schemas")
+	ErrHistogramsIncompatibleBounds   = errors.New("cannot apply this operation on custom buckets histograms with different custom bounds")
+)
+
+func IsCustomBucketsSchema(s int32) bool {
+	return s == CustomBucketsSchema
+}
+
+func IsExponentialSchema(s int32) bool {
+	return s >= ExponentialSchemaMin && s <= ExponentialSchemaMax
+}
+
 // BucketCount is a type constraint for the count in a bucket, which can be
 // float64 (for type FloatHistogram) or uint64 (for type Histogram).
 type BucketCount interface {
@@ -115,6 +134,8 @@ type baseBucketIterator[BC BucketCount, IBC InternalBucketCount] struct {
 
 	currCount IBC   // Count in the current bucket.
 	currIdx   int32 // The actual bucket index.
+
+	customValues []float64 // Bounds (usually upper) for histograms with custom buckets.
 }
 
 func (b *baseBucketIterator[BC, IBC]) At() Bucket[BC] {
@@ -128,14 +149,19 @@ func (b *baseBucketIterator[BC, IBC]) at(schema int32) Bucket[BC] {
 		Index: b.currIdx,
 	}
 	if b.positive {
-		bucket.Upper = getBound(b.currIdx, schema)
-		bucket.Lower = getBound(b.currIdx-1, schema)
+		bucket.Upper = getBound(b.currIdx, schema, b.customValues)
+		bucket.Lower = getBound(b.currIdx-1, schema, b.customValues)
 	} else {
-		bucket.Lower = -getBound(b.currIdx, schema)
-		bucket.Upper = -getBound(b.currIdx-1, schema)
+		bucket.Lower = -getBound(b.currIdx, schema, b.customValues)
+		bucket.Upper = -getBound(b.currIdx-1, schema, b.customValues)
+	}
+	if IsCustomBucketsSchema(schema) {
+		bucket.LowerInclusive = b.currIdx == 0
+		bucket.UpperInclusive = true
+	} else {
+		bucket.LowerInclusive = bucket.Lower < 0
+		bucket.UpperInclusive = bucket.Upper > 0
 	}
-	bucket.LowerInclusive = bucket.Lower < 0
-	bucket.UpperInclusive = bucket.Upper > 0
 	return bucket
 }
 
@@ -393,7 +419,55 @@ func checkHistogramBuckets[BC BucketCount, IBC InternalBucketCount](buckets []IB
 	return nil
 }
 
-func getBound(idx, schema int32) float64 {
+func checkHistogramCustomBounds(bounds []float64, spans []Span, numBuckets int) error {
+	prev := math.Inf(-1)
+	for _, curr := range bounds {
+		if curr <= prev {
+			return fmt.Errorf("previous bound is %f and current is %f: %w", prev, curr, ErrHistogramCustomBucketsInvalid)
+		}
+		prev = curr
+	}
+	if prev == math.Inf(1) {
+		return fmt.Errorf("last +Inf bound must not be explicitly defined: %w", ErrHistogramCustomBucketsInfinite)
+	}
+
+	var spanBuckets int
+	var totalSpanLength int
+	for n, span := range spans {
+		if span.Offset < 0 {
+			return fmt.Errorf("span number %d with offset %d: %w", n+1, span.Offset, ErrHistogramSpanNegativeOffset)
+		}
+		spanBuckets += int(span.Length)
+		totalSpanLength += int(span.Length) + int(span.Offset)
+	}
+	if spanBuckets != numBuckets {
+		return fmt.Errorf("spans need %d buckets, have %d buckets: %w", spanBuckets, numBuckets, ErrHistogramSpansBucketsMismatch)
+	}
+	if (len(bounds) + 1) < totalSpanLength {
+		return fmt.Errorf("only %d custom bounds defined which is insufficient to cover total span length of %d: %w", len(bounds), totalSpanLength, ErrHistogramCustomBucketsMismatch)
+	}
+
+	return nil
+}
+
+func getBound(idx, schema int32, customValues []float64) float64 {
+	if IsCustomBucketsSchema(schema) {
+		length := int32(len(customValues))
+		switch {
+		case idx > length || idx < -1:
+			panic(fmt.Errorf("index %d out of bounds for custom bounds of length %d", idx, length))
+		case idx == length:
+			return math.Inf(1)
+		case idx == -1:
+			return math.Inf(-1)
+		default:
+			return customValues[idx]
+		}
+	}
+	return getBoundExponential(idx, schema)
+}
+
+func getBoundExponential(idx, schema int32) float64 {
 	// Here a bit of context about the behavior for the last bucket counting
 	// regular numbers (called simply "last bucket" below) and the bucket
 	// counting observations of ±Inf (called "inf bucket" below, with an idx
@@ -703,3 +777,10 @@ func reduceResolution[IBC InternalBucketCount](
 
 	return targetSpans, targetBuckets
 }
+
+func clearIfNotNil[T any](items []T) []T {
+	if items == nil {
+		return nil
+	}
+	return items[:0]
+}

model/histogram/generic_test.go

@@ -21,7 +21,7 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestGetBound(t *testing.T) {
+func TestGetBoundExponential(t *testing.T) {
 	scenarios := []struct {
 		idx    int32
 		schema int32
@@ -105,7 +105,7 @@ func TestGetBound(t *testing.T) {
 	}
 
 	for _, s := range scenarios {
-		got := getBound(s.idx, s.schema)
+		got := getBoundExponential(s.idx, s.schema)
 		if s.want != got {
 			require.Equal(t, s.want, got, "idx %d, schema %d", s.idx, s.schema)
 		}

model/histogram/histogram.go

@@ -49,11 +49,12 @@ const (
 type Histogram struct {
 	// Counter reset information.
 	CounterResetHint CounterResetHint
-	// Currently valid schema numbers are -4 <= n <= 8.  They are all for
-	// base-2 bucket schemas, where 1 is a bucket boundary in each case, and
-	// then each power of two is divided into 2^n logarithmic buckets.  Or
-	// in other words, each bucket boundary is the previous boundary times
-	// 2^(2^-n).
+	// Currently valid schema numbers are -4 <= n <= 8 for exponential buckets,
+	// They are all for base-2 bucket schemas, where 1 is a bucket boundary in
+	// each case, and then each power of two is divided into 2^n logarithmic buckets.
+	// Or in other words, each bucket boundary is the previous boundary times
+	// 2^(2^-n). Another valid schema number is -53 for custom buckets, defined by
+	// the CustomValues field.
 	Schema int32
 	// Width of the zero bucket.
 	ZeroThreshold float64
@@ -69,6 +70,12 @@ type Histogram struct {
 	// count. All following ones are deltas relative to the previous
 	// element.
 	PositiveBuckets, NegativeBuckets []int64
+	// Holds the custom (usually upper) bounds for bucket definitions, otherwise nil.
+	// This slice is interned, to be treated as immutable and copied by reference.
+	// These numbers should be strictly increasing. This field is only used when the
+	// schema is for custom buckets, and the ZeroThreshold, ZeroCount, NegativeSpans
+	// and NegativeBuckets fields are not used in that case.
+	CustomValues []float64
 }
 
 // A Span defines a continuous sequence of buckets.
@@ -80,33 +87,46 @@ type Span struct {
 	Length uint32
 }
 
+func (h *Histogram) UsesCustomBuckets() bool {
+	return IsCustomBucketsSchema(h.Schema)
+}
+
 // Copy returns a deep copy of the Histogram.
 func (h *Histogram) Copy() *Histogram {
 	c := Histogram{
 		CounterResetHint: h.CounterResetHint,
 		Schema:           h.Schema,
-		ZeroThreshold:    h.ZeroThreshold,
-		ZeroCount:        h.ZeroCount,
 		Count:            h.Count,
 		Sum:              h.Sum,
 	}
 
+	if h.UsesCustomBuckets() {
+		if len(h.CustomValues) != 0 {
+			c.CustomValues = make([]float64, len(h.CustomValues))
+			copy(c.CustomValues, h.CustomValues)
+		}
+	} else {
+		c.ZeroThreshold = h.ZeroThreshold
+		c.ZeroCount = h.ZeroCount
+
+		if len(h.NegativeSpans) != 0 {
+			c.NegativeSpans = make([]Span, len(h.NegativeSpans))
+			copy(c.NegativeSpans, h.NegativeSpans)
+		}
+		if len(h.NegativeBuckets) != 0 {
+			c.NegativeBuckets = make([]int64, len(h.NegativeBuckets))
+			copy(c.NegativeBuckets, h.NegativeBuckets)
+		}
+	}
+
 	if len(h.PositiveSpans) != 0 {
 		c.PositiveSpans = make([]Span, len(h.PositiveSpans))
 		copy(c.PositiveSpans, h.PositiveSpans)
 	}
-	if len(h.NegativeSpans) != 0 {
-		c.NegativeSpans = make([]Span, len(h.NegativeSpans))
-		copy(c.NegativeSpans, h.NegativeSpans)
-	}
 	if len(h.PositiveBuckets) != 0 {
 		c.PositiveBuckets = make([]int64, len(h.PositiveBuckets))
 		copy(c.PositiveBuckets, h.PositiveBuckets)
 	}
-	if len(h.NegativeBuckets) != 0 {
-		c.NegativeBuckets = make([]int64, len(h.NegativeBuckets))
-		copy(c.NegativeBuckets, h.NegativeBuckets)
-	}
 
 	return &c
 }
@@ -116,22 +136,36 @@ func (h *Histogram) Copy() *Histogram {
 func (h *Histogram) CopyTo(to *Histogram) {
 	to.CounterResetHint = h.CounterResetHint
 	to.Schema = h.Schema
-	to.ZeroThreshold = h.ZeroThreshold
-	to.ZeroCount = h.ZeroCount
 	to.Count = h.Count
 	to.Sum = h.Sum
 
+	if h.UsesCustomBuckets() {
+		to.ZeroThreshold = 0
+		to.ZeroCount = 0
+
+		to.NegativeSpans = clearIfNotNil(to.NegativeSpans)
+		to.NegativeBuckets = clearIfNotNil(to.NegativeBuckets)
+
+		to.CustomValues = resize(to.CustomValues, len(h.CustomValues))
+		copy(to.CustomValues, h.CustomValues)
+	} else {
+		to.ZeroThreshold = h.ZeroThreshold
+		to.ZeroCount = h.ZeroCount
+
+		to.NegativeSpans = resize(to.NegativeSpans, len(h.NegativeSpans))
+		copy(to.NegativeSpans, h.NegativeSpans)
+
+		to.NegativeBuckets = resize(to.NegativeBuckets, len(h.NegativeBuckets))
+		copy(to.NegativeBuckets, h.NegativeBuckets)
+
+		to.CustomValues = clearIfNotNil(to.CustomValues)
+	}
+
 	to.PositiveSpans = resize(to.PositiveSpans, len(h.PositiveSpans))
 	copy(to.PositiveSpans, h.PositiveSpans)
 
-	to.NegativeSpans = resize(to.NegativeSpans, len(h.NegativeSpans))
-	copy(to.NegativeSpans, h.NegativeSpans)
-
 	to.PositiveBuckets = resize(to.PositiveBuckets, len(h.PositiveBuckets))
 	copy(to.PositiveBuckets, h.PositiveBuckets)
-
-	to.NegativeBuckets = resize(to.NegativeBuckets, len(h.NegativeBuckets))
-	copy(to.NegativeBuckets, h.NegativeBuckets)
 }
 
 // String returns a string representation of the Histogram.
@@ -165,8 +199,11 @@ func (h *Histogram) String() string {
 	return sb.String()
 }
 
-// ZeroBucket returns the zero bucket.
+// ZeroBucket returns the zero bucket. This method panics if the schema is for custom buckets.
 func (h *Histogram) ZeroBucket() Bucket[uint64] {
+	if h.UsesCustomBuckets() {
+		panic("histograms with custom buckets have no zero bucket")
+	}
 	return Bucket[uint64]{
 		Lower:          -h.ZeroThreshold,
 		Upper:          h.ZeroThreshold,
@@ -179,14 +216,14 @@ func (h *Histogram) ZeroBucket() Bucket[uint64] {
 // PositiveBucketIterator returns a BucketIterator to iterate over all positive
 // buckets in ascending order (starting next to the zero bucket and going up).
 func (h *Histogram) PositiveBucketIterator() BucketIterator[uint64] {
-	it := newRegularBucketIterator(h.PositiveSpans, h.PositiveBuckets, h.Schema, true)
+	it := newRegularBucketIterator(h.PositiveSpans, h.PositiveBuckets, h.Schema, true, h.CustomValues)
 	return &it
 }
 
 // NegativeBucketIterator returns a BucketIterator to iterate over all negative
 // buckets in descending order (starting next to the zero bucket and going down).
 func (h *Histogram) NegativeBucketIterator() BucketIterator[uint64] {
-	it := newRegularBucketIterator(h.NegativeSpans, h.NegativeBuckets, h.Schema, false)
+	it := newRegularBucketIterator(h.NegativeSpans, h.NegativeBuckets, h.Schema, false, nil)
 	return &it
 }
 
@@ -207,30 +244,42 @@ func (h *Histogram) CumulativeBucketIterator() BucketIterator[uint64] {
 // but they must represent the same bucket layout to match.
 // Sum is compared based on its bit pattern because this method
 // is about data equality rather than mathematical equality.
+// We ignore fields that are not used based on the exponential / custom buckets schema,
+// but check fields where differences may cause unintended behaviour even if they are not
+// supposed to be used according to the schema.
 func (h *Histogram) Equals(h2 *Histogram) bool {
 	if h2 == nil {
 		return false
 	}
 
-	if h.Schema != h2.Schema || h.ZeroThreshold != h2.ZeroThreshold ||
-		h.ZeroCount != h2.ZeroCount || h.Count != h2.Count ||
+	if h.Schema != h2.Schema || h.Count != h2.Count ||
 		math.Float64bits(h.Sum) != math.Float64bits(h2.Sum) {
 		return false
 	}
 
+	if h.UsesCustomBuckets() {
+		if !FloatBucketsMatch(h.CustomValues, h2.CustomValues) {
+			return false
+		}
+	}
+
+	if h.ZeroThreshold != h2.ZeroThreshold || h.ZeroCount != h2.ZeroCount {
+		return false
+	}
+
+	if !spansMatch(h.NegativeSpans, h2.NegativeSpans) {
+		return false
+	}
+	if !slices.Equal(h.NegativeBuckets, h2.NegativeBuckets) {
+		return false
+	}
+
 	if !spansMatch(h.PositiveSpans, h2.PositiveSpans) {
 		return false
 	}
-	if !spansMatch(h.NegativeSpans, h2.NegativeSpans) {
-		return false
-	}
-
 	if !slices.Equal(h.PositiveBuckets, h2.PositiveBuckets) {
 		return false
 	}
-	if !slices.Equal(h.NegativeBuckets, h2.NegativeBuckets) {
-		return false
-	}
 
 	return true
 }
@@ -321,17 +370,36 @@ func (h *Histogram) ToFloat(fh *FloatHistogram) *FloatHistogram {
 	}
 	fh.CounterResetHint = h.CounterResetHint
 	fh.Schema = h.Schema
-	fh.ZeroThreshold = h.ZeroThreshold
-	fh.ZeroCount = float64(h.ZeroCount)
 	fh.Count = float64(h.Count)
 	fh.Sum = h.Sum
 
+	if h.UsesCustomBuckets() {
+		fh.ZeroThreshold = 0
+		fh.ZeroCount = 0
+		fh.NegativeSpans = clearIfNotNil(fh.NegativeSpans)
+		fh.NegativeBuckets = clearIfNotNil(fh.NegativeBuckets)
+
+		fh.CustomValues = resize(fh.CustomValues, len(h.CustomValues))
+		copy(fh.CustomValues, h.CustomValues)
+	} else {
+		fh.ZeroThreshold = h.ZeroThreshold
+		fh.ZeroCount = float64(h.ZeroCount)
+
+		fh.NegativeSpans = resize(fh.NegativeSpans, len(h.NegativeSpans))
+		copy(fh.NegativeSpans, h.NegativeSpans)
+
+		fh.NegativeBuckets = resize(fh.NegativeBuckets, len(h.NegativeBuckets))
+		var currentNegative float64
+		for i, b := range h.NegativeBuckets {
+			currentNegative += float64(b)
+			fh.NegativeBuckets[i] = currentNegative
+		}
+		fh.CustomValues = clearIfNotNil(fh.CustomValues)
+	}
+
 	fh.PositiveSpans = resize(fh.PositiveSpans, len(h.PositiveSpans))
 	copy(fh.PositiveSpans, h.PositiveSpans)
 
-	fh.NegativeSpans = resize(fh.NegativeSpans, len(h.NegativeSpans))
-	copy(fh.NegativeSpans, h.NegativeSpans)
-
 	fh.PositiveBuckets = resize(fh.PositiveBuckets, len(h.PositiveBuckets))
 	var currentPositive float64
 	for i, b := range h.PositiveBuckets {
@@ -339,13 +407,6 @@ func (h *Histogram) ToFloat(fh *FloatHistogram) *FloatHistogram {
 		fh.PositiveBuckets[i] = currentPositive
 	}
 
-	fh.NegativeBuckets = resize(fh.NegativeBuckets, len(h.NegativeBuckets))
-	var currentNegative float64
-	for i, b := range h.NegativeBuckets {
-		currentNegative += float64(b)
-		fh.NegativeBuckets[i] = currentNegative
-	}
-
 	return fh
 }
 
@@ -357,25 +418,47 @@ func resize[T any](items []T, n int) []T {
 }
 
 // Validate validates consistency between span and bucket slices. Also, buckets are checked
-// against negative values.
+// against negative values. We check to make sure there are no unexpected fields or field values
+// based on the exponential / custom buckets schema.
 // For histograms that have not observed any NaN values (based on IsNaN(h.Sum) check), a
 // strict h.Count = nCount + pCount + h.ZeroCount check is performed.
 // Otherwise, only a lower bound check will be done (h.Count >= nCount + pCount + h.ZeroCount),
 // because NaN observations do not increment the values of buckets (but they do increment
 // the total h.Count).
 func (h *Histogram) Validate() error {
-	if err := checkHistogramSpans(h.NegativeSpans, len(h.NegativeBuckets)); err != nil {
-		return fmt.Errorf("negative side: %w", err)
-	}
-	if err := checkHistogramSpans(h.PositiveSpans, len(h.PositiveBuckets)); err != nil {
-		return fmt.Errorf("positive side: %w", err)
-	}
 	var nCount, pCount uint64
-	err := checkHistogramBuckets(h.NegativeBuckets, &nCount, true)
-	if err != nil {
-		return fmt.Errorf("negative side: %w", err)
+	if h.UsesCustomBuckets() {
+		if err := checkHistogramCustomBounds(h.CustomValues, h.PositiveSpans, len(h.PositiveBuckets)); err != nil {
+			return fmt.Errorf("custom buckets: %w", err)
+		}
+		if h.ZeroCount != 0 {
+			return fmt.Errorf("custom buckets: must have zero count of 0")
+		}
+		if h.ZeroThreshold != 0 {
+			return fmt.Errorf("custom buckets: must have zero threshold of 0")
+		}
+		if len(h.NegativeSpans) > 0 {
+			return fmt.Errorf("custom buckets: must not have negative spans")
+		}
+		if len(h.NegativeBuckets) > 0 {
+			return fmt.Errorf("custom buckets: must not have negative buckets")
+		}
+	} else {
+		if err := checkHistogramSpans(h.PositiveSpans, len(h.PositiveBuckets)); err != nil {
+			return fmt.Errorf("positive side: %w", err)
+		}
+		if err := checkHistogramSpans(h.NegativeSpans, len(h.NegativeBuckets)); err != nil {
+			return fmt.Errorf("negative side: %w", err)
+		}
+		err := checkHistogramBuckets(h.NegativeBuckets, &nCount, true)
+		if err != nil {
+			return fmt.Errorf("negative side: %w", err)
+		}
+		if h.CustomValues != nil {
+			return fmt.Errorf("histogram with exponential schema must not have custom bounds")
+		}
 	}
-	err = checkHistogramBuckets(h.PositiveBuckets, &pCount, true)
+	err := checkHistogramBuckets(h.PositiveBuckets, &pCount, true)
 	if err != nil {
 		return fmt.Errorf("positive side: %w", err)
 	}
@@ -398,12 +481,13 @@ type regularBucketIterator struct {
 	baseBucketIterator[uint64, int64]
 }
 
-func newRegularBucketIterator(spans []Span, buckets []int64, schema int32, positive bool) regularBucketIterator {
+func newRegularBucketIterator(spans []Span, buckets []int64, schema int32, positive bool, customValues []float64) regularBucketIterator {
 	i := baseBucketIterator[uint64, int64]{
-		schema:   schema,
-		spans:    spans,
-		buckets:  buckets,
-		positive: positive,
+		schema:       schema,
+		spans:        spans,
+		buckets:      buckets,
+		positive:     positive,
+		customValues: customValues,
 	}
 	return regularBucketIterator{i}
 }
@@ -477,7 +561,7 @@ func (c *cumulativeBucketIterator) Next() bool {
 
 	if c.emptyBucketCount > 0 {
 		// We are traversing through empty buckets at the moment.
-		c.currUpper = getBound(c.currIdx, c.h.Schema)
+		c.currUpper = getBound(c.currIdx, c.h.Schema, c.h.CustomValues)
 		c.currIdx++
 		c.emptyBucketCount--
 		return true
@@ -494,7 +578,7 @@ func (c *cumulativeBucketIterator) Next() bool {
 
 	c.currCount += c.h.PositiveBuckets[c.posBucketsIdx]
 	c.currCumulativeCount += uint64(c.currCount)
-	c.currUpper = getBound(c.currIdx, c.h.Schema)
+	c.currUpper = getBound(c.currIdx, c.h.Schema, c.h.CustomValues)
 
 	c.posBucketsIdx++
 	c.idxInSpan++
@@ -524,7 +608,15 @@ func (c *cumulativeBucketIterator) At() Bucket[uint64] {
 
 // ReduceResolution reduces the histogram's spans, buckets into target schema.
 // The target schema must be smaller than the current histogram's schema.
+// This will panic if the histogram has custom buckets or if the target schema is
+// a custom buckets schema.
 func (h *Histogram) ReduceResolution(targetSchema int32) *Histogram {
+	if h.UsesCustomBuckets() {
+		panic("cannot reduce resolution when there are custom buckets")
+	}
+	if IsCustomBucketsSchema(targetSchema) {
+		panic("cannot reduce resolution to custom buckets schema")
+	}
 	if targetSchema >= h.Schema {
 		panic(fmt.Errorf("cannot reduce resolution from schema %d to %d", h.Schema, targetSchema))
 	}

model/histogram/histogram_test.go

@@ -69,6 +69,21 @@ func TestHistogramString(t *testing.T) {
 			},
 			expectedString: "{count:19, sum:2.7, [-64,-32):1, [-16,-8):1, [-8,-4):2, [-4,-2):1, [-2,-1):3, [-1,-0.5):1, (0.5,1]:1, (1,2]:3, (2,4]:1, (4,8]:2, (8,16]:1, (16,32]:1, (32,64]:1}",
 		},
+		{
+			histogram: Histogram{
+				Schema: CustomBucketsSchema,
+				Count:  19,
+				Sum:    2.7,
+				PositiveSpans: []Span{
+					{Offset: 0, Length: 4},
+					{Offset: 0, Length: 0},
+					{Offset: 0, Length: 3},
+				},
+				PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 0},
+				CustomValues:    []float64{1, 2, 5, 10, 15, 20, 25, 50},
+			},
+			expectedString: "{count:19, sum:2.7, [-Inf,1]:1, (1,2]:3, (2,5]:1, (5,10]:2, (10,15]:1, (15,20]:1, (20,25]:1}",
+		},
 	}
 
 	for i, c := range cases {
@@ -208,6 +223,26 @@ func TestCumulativeBucketIterator(t *testing.T) {
 				{Lower: math.Inf(-1), Upper: 16, Count: 8, LowerInclusive: true, UpperInclusive: true, Index: 2},
 			},
 		},
+		{
+			histogram: Histogram{
+				Schema: CustomBucketsSchema,
+				PositiveSpans: []Span{
+					{Offset: 0, Length: 2},
+					{Offset: 1, Length: 2},
+				},
+				PositiveBuckets: []int64{1, 1, -1, 0},
+				CustomValues:    []float64{5, 10, 20, 50},
+			},
+			expectedBuckets: []Bucket[uint64]{
+				{Lower: math.Inf(-1), Upper: 5, Count: 1, LowerInclusive: true, UpperInclusive: true, Index: 0},
+				{Lower: math.Inf(-1), Upper: 10, Count: 3, LowerInclusive: true, UpperInclusive: true, Index: 1},
+
+				{Lower: math.Inf(-1), Upper: 20, Count: 3, LowerInclusive: true, UpperInclusive: true, Index: 2},
+
+				{Lower: math.Inf(-1), Upper: 50, Count: 4, LowerInclusive: true, UpperInclusive: true, Index: 3},
+				{Lower: math.Inf(-1), Upper: math.Inf(1), Count: 5, LowerInclusive: true, UpperInclusive: true, Index: 4},
+			},
+		},
 	}
 
 	for i, c := range cases {
@@ -368,6 +403,62 @@ func TestRegularBucketIterator(t *testing.T) {
 			},
 			expectedNegativeBuckets: []Bucket[uint64]{},
 		},
+		{
+			histogram: Histogram{
+				Schema: CustomBucketsSchema,
+				PositiveSpans: []Span{
+					{Offset: 0, Length: 2},
+					{Offset: 1, Length: 2},
+				},
+				PositiveBuckets: []int64{1, 1, -1, 0},
+				CustomValues:    []float64{5, 10, 20, 50},
+			},
+			expectedPositiveBuckets: []Bucket[uint64]{
+				{Lower: math.Inf(-1), Upper: 5, Count: 1, LowerInclusive: true, UpperInclusive: true, Index: 0},
+				{Lower: 5, Upper: 10, Count: 2, LowerInclusive: false, UpperInclusive: true, Index: 1},
+
+				{Lower: 20, Upper: 50, Count: 1, LowerInclusive: false, UpperInclusive: true, Index: 3},
+				{Lower: 50, Upper: math.Inf(1), Count: 1, LowerInclusive: false, UpperInclusive: true, Index: 4},
+			},
+			expectedNegativeBuckets: []Bucket[uint64]{},
+		},
+		{
+			histogram: Histogram{
+				Schema: CustomBucketsSchema,
+				PositiveSpans: []Span{
+					{Offset: 0, Length: 2},
+					{Offset: 1, Length: 2},
+				},
+				PositiveBuckets: []int64{1, 1, -1, 0},
+				CustomValues:    []float64{0, 10, 20, 50},
+			},
+			expectedPositiveBuckets: []Bucket[uint64]{
+				{Lower: math.Inf(-1), Upper: 0, Count: 1, LowerInclusive: true, UpperInclusive: true, Index: 0},
+				{Lower: 0, Upper: 10, Count: 2, LowerInclusive: false, UpperInclusive: true, Index: 1},
+
+				{Lower: 20, Upper: 50, Count: 1, LowerInclusive: false, UpperInclusive: true, Index: 3},
+				{Lower: 50, Upper: math.Inf(1), Count: 1, LowerInclusive: false, UpperInclusive: true, Index: 4},
+			},
+			expectedNegativeBuckets: []Bucket[uint64]{},
+		},
+		{
+			histogram: Histogram{
+				Schema: CustomBucketsSchema,
+				PositiveSpans: []Span{
+					{Offset: 0, Length: 5},
+				},
+				PositiveBuckets: []int64{1, 1, 0, -1, 0},
+				CustomValues:    []float64{-5, 0, 20, 50},
+			},
+			expectedPositiveBuckets: []Bucket[uint64]{
+				{Lower: math.Inf(-1), Upper: -5, Count: 1, LowerInclusive: true, UpperInclusive: true, Index: 0},
+				{Lower: -5, Upper: 0, Count: 2, LowerInclusive: false, UpperInclusive: true, Index: 1},
+				{Lower: 0, Upper: 20, Count: 2, LowerInclusive: false, UpperInclusive: true, Index: 2},
+				{Lower: 20, Upper: 50, Count: 1, LowerInclusive: false, UpperInclusive: true, Index: 3},
+				{Lower: 50, Upper: math.Inf(1), Count: 1, LowerInclusive: false, UpperInclusive: true, Index: 4},
+			},
+			expectedNegativeBuckets: []Bucket[uint64]{},
+		},
 	}
 
 	for i, c := range cases {
@@ -461,11 +552,81 @@ func TestHistogramToFloat(t *testing.T) {
 	}
 }
 
+func TestCustomBucketsHistogramToFloat(t *testing.T) {
+	h := Histogram{
+		Schema: CustomBucketsSchema,
+		Count:  10,
+		Sum:    2.7,
+		PositiveSpans: []Span{
+			{Offset: 0, Length: 4},
+			{Offset: 0, Length: 0},
+			{Offset: 0, Length: 3},
+		},
+		PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 0},
+		CustomValues:    []float64{5, 10, 20, 50, 100, 500},
+	}
+	cases := []struct {
+		name string
+		fh   *FloatHistogram
+	}{
+		{name: "without prior float histogram"},
+		{name: "prior float histogram with more buckets", fh: &FloatHistogram{
+			Schema:        2,
+			Count:         3,
+			Sum:           5,
+			ZeroThreshold: 4,
+			ZeroCount:     1,
+			PositiveSpans: []Span{
+				{Offset: 1, Length: 2},
+				{Offset: 1, Length: 2},
+				{Offset: 1, Length: 2},
+			},
+			PositiveBuckets: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9},
+			NegativeSpans: []Span{
+				{Offset: 20, Length: 6},
+				{Offset: 12, Length: 7},
+				{Offset: 33, Length: 10},
+			},
+			NegativeBuckets: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9},
+		}},
+		{name: "prior float histogram with fewer buckets", fh: &FloatHistogram{
+			Schema:        2,
+			Count:         3,
+			Sum:           5,
+			ZeroThreshold: 4,
+			ZeroCount:     1,
+			PositiveSpans: []Span{
+				{Offset: 1, Length: 2},
+				{Offset: 1, Length: 2},
+				{Offset: 1, Length: 2},
+			},
+			PositiveBuckets: []float64{1, 2},
+			NegativeSpans: []Span{
+				{Offset: 20, Length: 6},
+				{Offset: 12, Length: 7},
+				{Offset: 33, Length: 10},
+			},
+			NegativeBuckets: []float64{1, 2},
+		}},
+	}
+
+	require.NoError(t, h.Validate())
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			hStr := h.String()
+			fh := h.ToFloat(c.fh)
+			require.NoError(t, fh.Validate())
+			require.Equal(t, hStr, h.String())
+			require.Equal(t, hStr, fh.String())
+		})
+	}
+}
+
 // TestHistogramEquals tests both Histogram and FloatHistogram.
 func TestHistogramEquals(t *testing.T) {
 	h1 := Histogram{
 		Schema:        3,
-		Count:         61,
+		Count:         62,
 		Sum:           2.7,
 		ZeroThreshold: 0.1,
 		ZeroCount:     42,
@@ -495,6 +656,15 @@ func TestHistogramEquals(t *testing.T) {
 		require.False(t, h1f.Equals(h2f))
 		require.False(t, h2f.Equals(h1f))
 	}
+	notEqualsUntilFloatConv := func(h1, h2 Histogram) {
+		require.False(t, h1.Equals(&h2))
+		require.False(t, h2.Equals(&h1))
+		h1f, h2f := h1.ToFloat(nil), h2.ToFloat(nil)
+		require.True(t, h1f.Equals(h2f))
+		require.True(t, h2f.Equals(h1f))
+	}
+
+	require.NoError(t, h1.Validate())
 
 	h2 := h1.Copy()
 	equals(h1, *h2)
@@ -602,6 +772,45 @@ func TestHistogramEquals(t *testing.T) {
 
 	// Sum StaleNaN vs regular NaN.
 	notEquals(*hStale, *hNaN)
+
+	// Has non-empty custom bounds for exponential schema.
+	hCustom := h1.Copy()
+	hCustom.CustomValues = []float64{1, 2, 3}
+	equals(h1, *hCustom)
+
+	cbh1 := Histogram{
+		Schema: CustomBucketsSchema,
+		Count:  10,
+		Sum:    2.7,
+		PositiveSpans: []Span{
+			{Offset: 0, Length: 4},
+			{Offset: 10, Length: 3},
+		},
+		PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 0},
+		CustomValues:    []float64{0.1, 0.2, 0.5, 1, 2, 5, 10, 15, 20, 25, 50, 75, 100, 200, 250, 500, 1000},
+	}
+
+	require.NoError(t, cbh1.Validate())
+
+	cbh2 := cbh1.Copy()
+	equals(cbh1, *cbh2)
+
+	// Has different custom bounds for custom buckets schema.
+	cbh2 = cbh1.Copy()
+	cbh2.CustomValues = []float64{0.1, 0.2, 0.5}
+	notEquals(cbh1, *cbh2)
+
+	// Has non-empty negative spans and buckets for custom buckets schema.
+	cbh2 = cbh1.Copy()
+	cbh2.NegativeSpans = []Span{{Offset: 0, Length: 1}}
+	cbh2.NegativeBuckets = []int64{1}
+	notEqualsUntilFloatConv(cbh1, *cbh2)
+
+	// Has non-zero zero count and threshold for custom buckets schema.
+	cbh2 = cbh1.Copy()
+	cbh2.ZeroThreshold = 0.1
+	cbh2.ZeroCount = 10
+	notEqualsUntilFloatConv(cbh1, *cbh2)
 }
 
 func TestHistogramCopy(t *testing.T) {
@@ -640,6 +849,21 @@ func TestHistogramCopy(t *testing.T) {
 			},
 			expected: &Histogram{},
 		},
+		{
+			name: "with custom buckets",
+			orig: &Histogram{
+				Schema:          CustomBucketsSchema,
+				PositiveSpans:   []Span{{-2, 1}},
+				PositiveBuckets: []int64{1, 3, -3, 42},
+				CustomValues:    []float64{5, 10, 15},
+			},
+			expected: &Histogram{
+				Schema:          CustomBucketsSchema,
+				PositiveSpans:   []Span{{-2, 1}},
+				PositiveBuckets: []int64{1, 3, -3, 42},
+				CustomValues:    []float64{5, 10, 15},
+			},
+		},
 	}
 
 	for _, tcase := range cases {
@@ -690,6 +914,21 @@ func TestHistogramCopyTo(t *testing.T) {
 			},
 			expected: &Histogram{},
 		},
+		{
+			name: "with custom buckets",
+			orig: &Histogram{
+				Schema:          CustomBucketsSchema,
+				PositiveSpans:   []Span{{-2, 1}},
+				PositiveBuckets: []int64{1, 3, -3, 42},
+				CustomValues:    []float64{5, 10, 15},
+			},
+			expected: &Histogram{
+				Schema:          CustomBucketsSchema,
+				PositiveSpans:   []Span{{-2, 1}},
+				PositiveBuckets: []int64{1, 3, -3, 42},
+				CustomValues:    []float64{5, 10, 15},
+			},
+		},
 	}
 
 	for _, tcase := range cases {
@@ -971,6 +1210,86 @@ func TestHistogramCompact(t *testing.T) {
 				NegativeBuckets: []int64{2, 3},
 			},
 		},
+		{
+			"nothing should happen with custom buckets",
+			&Histogram{
+				Schema:          CustomBucketsSchema,
+				PositiveSpans:   []Span{{-2, 1}, {2, 3}},
+				PositiveBuckets: []int64{1, 3, -3, 42},
+				CustomValues:    []float64{5, 10, 15},
+			},
+			0,
+			&Histogram{
+				Schema:          CustomBucketsSchema,
+				PositiveSpans:   []Span{{-2, 1}, {2, 3}},
+				PositiveBuckets: []int64{1, 3, -3, 42},
+				CustomValues:    []float64{5, 10, 15},
+			},
+		},
+		{
+			"eliminate zero offsets with custom buckets",
+			&Histogram{
+				Schema:          CustomBucketsSchema,
+				PositiveSpans:   []Span{{-2, 1}, {0, 3}, {0, 1}},
+				PositiveBuckets: []int64{1, 3, -3, 42, 3},
+				CustomValues:    []float64{5, 10, 15, 20},
+			},
+			0,
+			&Histogram{
+				Schema:          CustomBucketsSchema,
+				PositiveSpans:   []Span{{-2, 5}},
+				PositiveBuckets: []int64{1, 3, -3, 42, 3},
+				CustomValues:    []float64{5, 10, 15, 20},
+			},
+		},
+		{
+			"eliminate zero length with custom buckets",
+			&Histogram{
+				Schema:          CustomBucketsSchema,
+				PositiveSpans:   []Span{{-2, 2}, {2, 0}, {3, 3}},
+				PositiveBuckets: []int64{1, 3, -3, 42, 3},
+				CustomValues:    []float64{5, 10, 15, 20},
+			},
+			0,
+			&Histogram{
+				Schema:          CustomBucketsSchema,
+				PositiveSpans:   []Span{{-2, 2}, {5, 3}},
+				PositiveBuckets: []int64{1, 3, -3, 42, 3},
+				CustomValues:    []float64{5, 10, 15, 20},
+			},
+		},
+		{
+			"eliminate multiple zero length spans with custom buckets",
+			&Histogram{
+				Schema:          CustomBucketsSchema,
+				PositiveSpans:   []Span{{-2, 2}, {2, 0}, {2, 0}, {2, 0}, {3, 3}},
+				PositiveBuckets: []int64{1, 3, -3, 42, 3},
+				CustomValues:    []float64{5, 10, 15, 20},
+			},
+			0,
+			&Histogram{
+				Schema:          CustomBucketsSchema,
+				PositiveSpans:   []Span{{-2, 2}, {9, 3}},
+				PositiveBuckets: []int64{1, 3, -3, 42, 3},
+				CustomValues:    []float64{5, 10, 15, 20},
+			},
+		},
+		{
+			"cut empty buckets at start or end of spans, even in the middle, with custom buckets",
+			&Histogram{
+				Schema:          CustomBucketsSchema,
+				PositiveSpans:   []Span{{-4, 6}, {3, 6}},
+				PositiveBuckets: []int64{0, 0, 1, 3, -4, 0, 1, 42, 3, -46, 0, 0},
+				CustomValues:    []float64{5, 10, 15, 20},
+			},
+			0,
+			&Histogram{
+				Schema:          CustomBucketsSchema,
+				PositiveSpans:   []Span{{-2, 2}, {5, 3}},
+				PositiveBuckets: []int64{1, 3, -3, 42, 3},
+				CustomValues:    []float64{5, 10, 15, 20},
+			},
+		},
 	}
 
 	for _, c := range cases {
@@ -1107,6 +1426,145 @@ func TestHistogramValidation(t *testing.T) {
 			errMsg:    `3 observations found in buckets, but the Count field is 2: histogram's observation count should equal the number of observations found in the buckets (in absence of NaN)`,
 			skipFloat: true,
 		},
+		"rejects an exponential histogram with custom buckets schema": {
+			h: &Histogram{
+				Count:         12,
+				ZeroCount:     2,
+				ZeroThreshold: 0.001,
+				Sum:           19.4,
+				Schema:        CustomBucketsSchema,
+				PositiveSpans: []Span{
+					{Offset: 0, Length: 2},
+					{Offset: 1, Length: 2},
+				},
+				PositiveBuckets: []int64{1, 1, -1, 0},
+				NegativeSpans: []Span{
+					{Offset: 0, Length: 2},
+					{Offset: 1, Length: 2},
+				},
+				NegativeBuckets: []int64{1, 1, -1, 0},
+			},
+			errMsg: `custom buckets: only 0 custom bounds defined which is insufficient to cover total span length of 5: histogram custom bounds are too few`,
+		},
+		"rejects a custom buckets histogram with exponential schema": {
+			h: &Histogram{
+				Count:  5,
+				Sum:    19.4,
+				Schema: 0,
+				PositiveSpans: []Span{
+					{Offset: 0, Length: 2},
+					{Offset: 1, Length: 2},
+				},
+				PositiveBuckets: []int64{1, 1, -1, 0},
+				CustomValues:    []float64{1, 2, 3, 4},
+			},
+			errMsg:    `histogram with exponential schema must not have custom bounds`,
+			skipFloat: true, // Converting to float will remove the wrong fields so only the float version will pass validation
+		},
+		"rejects a custom buckets histogram with zero/negative buckets": {
+			h: &Histogram{
+				Count:         12,
+				ZeroCount:     2,
+				ZeroThreshold: 0.001,
+				Sum:           19.4,
+				Schema:        CustomBucketsSchema,
+				PositiveSpans: []Span{
+					{Offset: 0, Length: 2},
+					{Offset: 1, Length: 2},
+				},
+				PositiveBuckets: []int64{1, 1, -1, 0},
+				NegativeSpans: []Span{
+					{Offset: 0, Length: 2},
+					{Offset: 1, Length: 2},
+				},
+				NegativeBuckets: []int64{1, 1, -1, 0},
+				CustomValues:    []float64{1, 2, 3, 4},
+			},
+			errMsg:    `custom buckets: must have zero count of 0`,
+			skipFloat: true, // Converting to float will remove the wrong fields so only the float version will pass validation
+		},
+		"rejects a custom buckets histogram with negative offset in first span": {
+			h: &Histogram{
+				Count:  5,
+				Sum:    19.4,
+				Schema: CustomBucketsSchema,
+				PositiveSpans: []Span{
+					{Offset: -1, Length: 2},
+					{Offset: 1, Length: 2},
+				},
+				PositiveBuckets: []int64{1, 1, -1, 0},
+				CustomValues:    []float64{1, 2, 3, 4},
+			},
+			errMsg: `custom buckets: span number 1 with offset -1: histogram has a span whose offset is negative`,
+		},
+		"rejects a custom buckets histogram with negative offset in subsequent spans": {
+			h: &Histogram{
+				Count:  5,
+				Sum:    19.4,
+				Schema: CustomBucketsSchema,
+				PositiveSpans: []Span{
+					{Offset: 0, Length: 2},
+					{Offset: -1, Length: 2},
+				},
+				PositiveBuckets: []int64{1, 1, -1, 0},
+				CustomValues:    []float64{1, 2, 3, 4},
+			},
+			errMsg: `custom buckets: span number 2 with offset -1: histogram has a span whose offset is negative`,
+		},
+		"rejects a custom buckets histogram with non-matching bucket counts": {
+			h: &Histogram{
+				Count:  5,
+				Sum:    19.4,
+				Schema: CustomBucketsSchema,
+				PositiveSpans: []Span{
+					{Offset: 0, Length: 2},
+					{Offset: 1, Length: 2},
+				},
+				PositiveBuckets: []int64{1, 1, -1},
+				CustomValues:    []float64{1, 2, 3, 4},
+			},
+			errMsg: `custom buckets: spans need 4 buckets, have 3 buckets: histogram spans specify different number of buckets than provided`,
+		},
+		"rejects a custom buckets histogram with too few bounds": {
+			h: &Histogram{
+				Count:  5,
+				Sum:    19.4,
+				Schema: CustomBucketsSchema,
+				PositiveSpans: []Span{
+					{Offset: 0, Length: 2},
+					{Offset: 1, Length: 2},
+				},
+				PositiveBuckets: []int64{1, 1, -1, 0},
+				CustomValues:    []float64{1, 2, 3},
+			},
+			errMsg: `custom buckets: only 3 custom bounds defined which is insufficient to cover total span length of 5: histogram custom bounds are too few`,
+		},
+		"valid custom buckets histogram": {
+			h: &Histogram{
+				Count:  5,
+				Sum:    19.4,
+				Schema: CustomBucketsSchema,
+				PositiveSpans: []Span{
+					{Offset: 0, Length: 2},
+					{Offset: 1, Length: 2},
+				},
+				PositiveBuckets: []int64{1, 1, -1, 0},
+				CustomValues:    []float64{1, 2, 3, 4},
+			},
+		},
+		"valid custom buckets histogram with extra bounds": {
+			h: &Histogram{
+				Count:  5,
+				Sum:    19.4,
+				Schema: CustomBucketsSchema,
+				PositiveSpans: []Span{
+					{Offset: 0, Length: 2},
+					{Offset: 1, Length: 2},
+				},
+				PositiveBuckets: []int64{1, 1, -1, 0},
+				CustomValues:    []float64{1, 2, 3, 4, 5, 6, 7, 8},
+			},
+		},
 	}
 
 	for testName, tc := range tests {

promql/engine.go

@@ -1793,18 +1793,21 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, annotations.Annotatio
 				}, e.LHS, e.RHS)
 			default:
 				return ev.rangeEval(initSignatures, func(v []parser.Value, sh [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, annotations.Annotations) {
-					return ev.VectorBinop(e.Op, v[0].(Vector), v[1].(Vector), e.VectorMatching, e.ReturnBool, sh[0], sh[1], enh), nil
+					vec, err := ev.VectorBinop(e.Op, v[0].(Vector), v[1].(Vector), e.VectorMatching, e.ReturnBool, sh[0], sh[1], enh)
+					return vec, handleVectorBinopError(err, e)
 				}, e.LHS, e.RHS)
 			}
 
 		case lt == parser.ValueTypeVector && rt == parser.ValueTypeScalar:
 			return ev.rangeEval(nil, func(v []parser.Value, _ [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, annotations.Annotations) {
-				return ev.VectorscalarBinop(e.Op, v[0].(Vector), Scalar{V: v[1].(Vector)[0].F}, false, e.ReturnBool, enh), nil
+				vec, err := ev.VectorscalarBinop(e.Op, v[0].(Vector), Scalar{V: v[1].(Vector)[0].F}, false, e.ReturnBool, enh)
+				return vec, handleVectorBinopError(err, e)
 			}, e.LHS, e.RHS)
 
 		case lt == parser.ValueTypeScalar && rt == parser.ValueTypeVector:
 			return ev.rangeEval(nil, func(v []parser.Value, _ [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, annotations.Annotations) {
-				return ev.VectorscalarBinop(e.Op, v[1].(Vector), Scalar{V: v[0].(Vector)[0].F}, true, e.ReturnBool, enh), nil
+				vec, err := ev.VectorscalarBinop(e.Op, v[1].(Vector), Scalar{V: v[0].(Vector)[0].F}, true, e.ReturnBool, enh)
+				return vec, handleVectorBinopError(err, e)
 			}, e.LHS, e.RHS)
 		}
 
@@ -2437,12 +2440,12 @@ func (ev *evaluator) VectorUnless(lhs, rhs Vector, matching *parser.VectorMatchi
 }
 
 // VectorBinop evaluates a binary operation between two Vectors, excluding set operators.
-func (ev *evaluator) VectorBinop(op parser.ItemType, lhs, rhs Vector, matching *parser.VectorMatching, returnBool bool, lhsh, rhsh []EvalSeriesHelper, enh *EvalNodeHelper) Vector {
+func (ev *evaluator) VectorBinop(op parser.ItemType, lhs, rhs Vector, matching *parser.VectorMatching, returnBool bool, lhsh, rhsh []EvalSeriesHelper, enh *EvalNodeHelper) (Vector, error) {
 	if matching.Card == parser.CardManyToMany {
 		panic("many-to-many only allowed for set operators")
 	}
 	if len(lhs) == 0 || len(rhs) == 0 {
-		return nil // Short-circuit: nothing is going to match.
+		return nil, nil // Short-circuit: nothing is going to match.
 	}
 
 	// The control flow below handles one-to-one or many-to-one matching.
@@ -2495,6 +2498,7 @@ func (ev *evaluator) VectorBinop(op parser.ItemType, lhs, rhs Vector, matching *
 
 	// For all lhs samples find a respective rhs sample and perform
 	// the binary operation.
+	var lastErr error
 	for i, ls := range lhs {
 		sig := lhsh[i].signature
 
@@ -2510,7 +2514,10 @@ func (ev *evaluator) VectorBinop(op parser.ItemType, lhs, rhs Vector, matching *
 			fl, fr = fr, fl
 			hl, hr = hr, hl
 		}
-		floatValue, histogramValue, keep := vectorElemBinop(op, fl, fr, hl, hr)
+		floatValue, histogramValue, keep, err := vectorElemBinop(op, fl, fr, hl, hr)
+		if err != nil {
+			lastErr = err
+		}
 		switch {
 		case returnBool:
 			if keep {
@@ -2552,7 +2559,7 @@ func (ev *evaluator) VectorBinop(op parser.ItemType, lhs, rhs Vector, matching *
 			H:      histogramValue,
 		})
 	}
-	return enh.Out
+	return enh.Out, lastErr
 }
 
 func signatureFunc(on bool, b []byte, names ...string) func(labels.Labels) string {
@@ -2615,7 +2622,8 @@ func resultMetric(lhs, rhs labels.Labels, op parser.ItemType, matching *parser.V
 }
 
 // VectorscalarBinop evaluates a binary operation between a Vector and a Scalar.
-func (ev *evaluator) VectorscalarBinop(op parser.ItemType, lhs Vector, rhs Scalar, swap, returnBool bool, enh *EvalNodeHelper) Vector {
+func (ev *evaluator) VectorscalarBinop(op parser.ItemType, lhs Vector, rhs Scalar, swap, returnBool bool, enh *EvalNodeHelper) (Vector, error) {
+	var lastErr error
 	for _, lhsSample := range lhs {
 		lf, rf := lhsSample.F, rhs.V
 		var rh *histogram.FloatHistogram
@@ -2626,7 +2634,10 @@ func (ev *evaluator) VectorscalarBinop(op parser.ItemType, lhs Vector, rhs Scala
 			lf, rf = rf, lf
 			lh, rh = rh, lh
 		}
-		float, histogram, keep := vectorElemBinop(op, lf, rf, lh, rh)
+		float, histogram, keep, err := vectorElemBinop(op, lf, rf, lh, rh)
+		if err != nil {
+			lastErr = err
+		}
 		// Catch cases where the scalar is the LHS in a scalar-vector comparison operation.
 		// We want to always keep the vector element value as the output value, even if it's on the RHS.
 		if op.IsComparisonOperator() && swap {
@@ -2650,7 +2661,7 @@ func (ev *evaluator) VectorscalarBinop(op parser.ItemType, lhs Vector, rhs Scala
 			enh.Out = append(enh.Out, lhsSample)
 		}
 	}
-	return enh.Out
+	return enh.Out, lastErr
 }
 
 // scalarBinop evaluates a binary operation between two Scalars.
@@ -2687,49 +2698,57 @@ func scalarBinop(op parser.ItemType, lhs, rhs float64) float64 {
 }
 
 // vectorElemBinop evaluates a binary operation between two Vector elements.
-func vectorElemBinop(op parser.ItemType, lhs, rhs float64, hlhs, hrhs *histogram.FloatHistogram) (float64, *histogram.FloatHistogram, bool) {
+func vectorElemBinop(op parser.ItemType, lhs, rhs float64, hlhs, hrhs *histogram.FloatHistogram) (float64, *histogram.FloatHistogram, bool, error) {
 	switch op {
 	case parser.ADD:
 		if hlhs != nil && hrhs != nil {
-			return 0, hlhs.Copy().Add(hrhs).Compact(0), true
+			res, err := hlhs.Copy().Add(hrhs)
+			if err != nil {
+				return 0, nil, false, err
+			}
+			return 0, res.Compact(0), true, nil
 		}
-		return lhs + rhs, nil, true
+		return lhs + rhs, nil, true, nil
 	case parser.SUB:
 		if hlhs != nil && hrhs != nil {
-			return 0, hlhs.Copy().Sub(hrhs).Compact(0), true
+			res, err := hlhs.Copy().Sub(hrhs)
+			if err != nil {
+				return 0, nil, false, err
+			}
+			return 0, res.Compact(0), true, nil
 		}
-		return lhs - rhs, nil, true
+		return lhs - rhs, nil, true, nil
 	case parser.MUL:
 		if hlhs != nil && hrhs == nil {
-			return 0, hlhs.Copy().Mul(rhs), true
+			return 0, hlhs.Copy().Mul(rhs), true, nil
 		}
 		if hlhs == nil && hrhs != nil {
-			return 0, hrhs.Copy().Mul(lhs), true
+			return 0, hrhs.Copy().Mul(lhs), true, nil
 		}
-		return lhs * rhs, nil, true
+		return lhs * rhs, nil, true, nil
 	case parser.DIV:
 		if hlhs != nil && hrhs == nil {
-			return 0, hlhs.Copy().Div(rhs), true
+			return 0, hlhs.Copy().Div(rhs), true, nil
 		}
-		return lhs / rhs, nil, true
+		return lhs / rhs, nil, true, nil
 	case parser.POW:
-		return math.Pow(lhs, rhs), nil, true
+		return math.Pow(lhs, rhs), nil, true, nil
 	case parser.MOD:
-		return math.Mod(lhs, rhs), nil, true
+		return math.Mod(lhs, rhs), nil, true, nil
 	case parser.EQLC:
-		return lhs, nil, lhs == rhs
+		return lhs, nil, lhs == rhs, nil
 	case parser.NEQ:
-		return lhs, nil, lhs != rhs
+		return lhs, nil, lhs != rhs, nil
 	case parser.GTR:
-		return lhs, nil, lhs > rhs
+		return lhs, nil, lhs > rhs, nil
 	case parser.LSS:
-		return lhs, nil, lhs < rhs
+		return lhs, nil, lhs < rhs, nil
 	case parser.GTE:
-		return lhs, nil, lhs >= rhs
+		return lhs, nil, lhs >= rhs, nil
 	case parser.LTE:
-		return lhs, nil, lhs <= rhs
+		return lhs, nil, lhs <= rhs, nil
 	case parser.ATAN2:
-		return math.Atan2(lhs, rhs), nil, true
+		return math.Atan2(lhs, rhs), nil, true, nil
 	}
 	panic(fmt.Errorf("operator %q not allowed for operations between Vectors", op))
 }
@@ -2798,7 +2817,10 @@ func (ev *evaluator) aggregation(e *parser.AggregateExpr, q float64, inputMatrix
 			if h != nil {
 				group.hasHistogram = true
 				if group.histogramValue != nil {
-					group.histogramValue.Add(h)
+					_, err := group.histogramValue.Add(h)
+					if err != nil {
+						handleAggregationError(err, e, inputMatrix[si].Metric.Get(model.MetricNameLabel), &annos)
+					}
 				}
 				// Otherwise the aggregation contained floats
 				// previously and will be invalid anyway. No
@@ -2815,8 +2837,14 @@ func (ev *evaluator) aggregation(e *parser.AggregateExpr, q float64, inputMatrix
 				if group.histogramValue != nil {
 					left := h.Copy().Div(float64(group.groupCount))
 					right := group.histogramValue.Copy().Div(float64(group.groupCount))
-					toAdd := left.Sub(right)
-					group.histogramValue.Add(toAdd)
+					toAdd, err := left.Sub(right)
+					if err != nil {
+						handleAggregationError(err, e, inputMatrix[si].Metric.Get(model.MetricNameLabel), &annos)
+					}
+					_, err = group.histogramValue.Add(toAdd)
+					if err != nil {
+						handleAggregationError(err, e, inputMatrix[si].Metric.Get(model.MetricNameLabel), &annos)
+					}
 				}
 				// Otherwise the aggregation contained floats
 				// previously and will be invalid anyway. No
@@ -3115,6 +3143,31 @@ func (ev *evaluator) nextValues(ts int64, series *Series) (f float64, h *histogr
 	return f, h, true
 }
 
+// handleAggregationError adds the appropriate annotation based on the aggregation error.
+func handleAggregationError(err error, e *parser.AggregateExpr, metricName string, annos *annotations.Annotations) {
+	pos := e.Expr.PositionRange()
+	if errors.Is(err, histogram.ErrHistogramsIncompatibleSchema) {
+		annos.Add(annotations.NewMixedExponentialCustomHistogramsWarning(metricName, pos))
+	} else if errors.Is(err, histogram.ErrHistogramsIncompatibleBounds) {
+		annos.Add(annotations.NewIncompatibleCustomBucketsHistogramsWarning(metricName, pos))
+	}
+}
+
+// handleVectorBinopError returns the appropriate annotation based on the vector binary operation error.
+func handleVectorBinopError(err error, e *parser.BinaryExpr) annotations.Annotations {
+	if err == nil {
+		return nil
+	}
+	metricName := ""
+	pos := e.PositionRange()
+	if errors.Is(err, histogram.ErrHistogramsIncompatibleSchema) {
+		return annotations.New().Add(annotations.NewMixedExponentialCustomHistogramsWarning(metricName, pos))
+	} else if errors.Is(err, histogram.ErrHistogramsIncompatibleBounds) {
+		return annotations.New().Add(annotations.NewIncompatibleCustomBucketsHistogramsWarning(metricName, pos))
+	}
+	return nil
+}
+
 // groupingKey builds and returns the grouping key for the given metric and
 // grouping labels.
 func generateGroupingKey(metric labels.Labels, grouping []string, without bool, buf []byte) (uint64, []byte) {

promql/engine_test.go

@@ -834,10 +834,10 @@ load 10s
 		{
 			Query:        "metricWith1HistogramEvery10Seconds",
 			Start:        time.Unix(21, 0),
-			PeakSamples:  12,
-			TotalSamples: 12, // 1 histogram sample of size 12 / 10 seconds
+			PeakSamples:  13,
+			TotalSamples: 13, // 1 histogram HPoint of size 13 / 10 seconds
 			TotalSamplesPerStep: stats.TotalSamplesPerStep{
-				21000: 12,
+				21000: 13,
 			},
 		},
 		{
@@ -934,10 +934,10 @@ load 10s
 		{
 			Query:        "metricWith1HistogramEvery10Seconds[60s]",
 			Start:        time.Unix(201, 0),
-			PeakSamples:  72,
-			TotalSamples: 72, // 1 histogram (size 12) / 10 seconds * 60 seconds
+			PeakSamples:  78,
+			TotalSamples: 78, // 1 histogram (size 13 HPoint) / 10 seconds * 60 seconds
 			TotalSamplesPerStep: stats.TotalSamplesPerStep{
-				201000: 72,
+				201000: 78,
 			},
 		},
 		{
@@ -964,11 +964,11 @@ load 10s
 		{
 			Query:        "max_over_time(metricWith1HistogramEvery10Seconds[60s])[20s:5s]",
 			Start:        time.Unix(201, 0),
-			PeakSamples:  72,
-			TotalSamples: 312, // (1 histogram (size 12) / 10 seconds * 60 seconds) * 4 + 2 * 12 as
+			PeakSamples:  78,
+			TotalSamples: 338, // (1 histogram (size 13 HPoint) / 10 seconds * 60 seconds) * 4 + 2 * 13 as
 			// max_over_time(metricWith1SampleEvery10Seconds[60s]) @ 190 and 200 will return 7 samples.
 			TotalSamplesPerStep: stats.TotalSamplesPerStep{
-				201000: 312,
+				201000: 338,
 			},
 		},
 		{
@@ -983,10 +983,10 @@ load 10s
 		{
 			Query:        "metricWith1HistogramEvery10Seconds[60s] @ 30",
 			Start:        time.Unix(201, 0),
-			PeakSamples:  48,
-			TotalSamples: 48, // @ modifier force the evaluation to at 30 seconds - So it brings 4 datapoints (0, 10, 20, 30 seconds) * 1 series
+			PeakSamples:  52,
+			TotalSamples: 52, // @ modifier force the evaluation to at 30 seconds - So it brings 4 datapoints (0, 10, 20, 30 seconds) * 1 series
 			TotalSamplesPerStep: stats.TotalSamplesPerStep{
-				201000: 48,
+				201000: 52,
 			},
 		},
 		{
@@ -1121,13 +1121,13 @@ load 10s
 			Start:        time.Unix(204, 0),
 			End:          time.Unix(223, 0),
 			Interval:     5 * time.Second,
-			PeakSamples:  48,
-			TotalSamples: 48, // 1 histogram (size 12) per query * 4 steps
+			PeakSamples:  52,
+			TotalSamples: 52, // 1 histogram (size 13 HPoint) per query * 4 steps
 			TotalSamplesPerStep: stats.TotalSamplesPerStep{
-				204000: 12, // aligned to the step time, not the sample time
-				209000: 12,
-				214000: 12,
-				219000: 12,
+				204000: 13, // aligned to the step time, not the sample time
+				209000: 13,
+				214000: 13,
+				219000: 13,
 			},
 		},
 		{

promql/functions.go

@@ -14,6 +14,7 @@
 package promql
 
 import (
+	"errors"
 	"fmt"
 	"math"
 	"slices"
@@ -210,14 +211,28 @@ func histogramRate(points []HPoint, isCounter bool, metricName string, pos posra
 	}
 
 	h := last.CopyToSchema(minSchema)
-	h.Sub(prev)
+	_, err := h.Sub(prev)
+	if err != nil {
+		if errors.Is(err, histogram.ErrHistogramsIncompatibleSchema) {
+			return nil, annotations.New().Add(annotations.NewMixedExponentialCustomHistogramsWarning(metricName, pos))
+		} else if errors.Is(err, histogram.ErrHistogramsIncompatibleBounds) {
+			return nil, annotations.New().Add(annotations.NewIncompatibleCustomBucketsHistogramsWarning(metricName, pos))
+		}
+	}
 
 	if isCounter {
 		// Second iteration to deal with counter resets.
 		for _, currPoint := range points[1:] {
 			curr := currPoint.H
 			if curr.DetectReset(prev) {
-				h.Add(prev)
+				_, err := h.Add(prev)
+				if err != nil {
+					if errors.Is(err, histogram.ErrHistogramsIncompatibleSchema) {
+						return nil, annotations.New().Add(annotations.NewMixedExponentialCustomHistogramsWarning(metricName, pos))
+					} else if errors.Is(err, histogram.ErrHistogramsIncompatibleBounds) {
+						return nil, annotations.New().Add(annotations.NewIncompatibleCustomBucketsHistogramsWarning(metricName, pos))
+					}
+				}
 			}
 			prev = curr
 		}
@@ -513,10 +528,11 @@ func aggrOverTime(vals []parser.Value, enh *EvalNodeHelper, aggrFn func(Series)
 	return append(enh.Out, Sample{F: aggrFn(el)})
 }
 
-func aggrHistOverTime(vals []parser.Value, enh *EvalNodeHelper, aggrFn func(Series) *histogram.FloatHistogram) Vector {
+func aggrHistOverTime(vals []parser.Value, enh *EvalNodeHelper, aggrFn func(Series) (*histogram.FloatHistogram, error)) (Vector, error) {
 	el := vals[0].(Matrix)[0]
+	res, err := aggrFn(el)
 
-	return append(enh.Out, Sample{H: aggrFn(el)})
+	return append(enh.Out, Sample{H: res}), err
 }
 
 // === avg_over_time(Matrix parser.ValueTypeMatrix) (Vector, Annotations)  ===
@@ -528,18 +544,33 @@ func funcAvgOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNode
 	}
 	if len(firstSeries.Floats) == 0 {
 		// The passed values only contain histograms.
-		return aggrHistOverTime(vals, enh, func(s Series) *histogram.FloatHistogram {
+		vec, err := aggrHistOverTime(vals, enh, func(s Series) (*histogram.FloatHistogram, error) {
 			count := 1
 			mean := s.Histograms[0].H.Copy()
 			for _, h := range s.Histograms[1:] {
 				count++
 				left := h.H.Copy().Div(float64(count))
 				right := mean.Copy().Div(float64(count))
-				toAdd := left.Sub(right)
-				mean.Add(toAdd)
+				toAdd, err := left.Sub(right)
+				if err != nil {
+					return mean, err
+				}
+				_, err = mean.Add(toAdd)
+				if err != nil {
+					return mean, err
+				}
 			}
-			return mean
-		}), nil
+			return mean, nil
+		})
+		if err != nil {
+			metricName := firstSeries.Metric.Get(labels.MetricName)
+			if errors.Is(err, histogram.ErrHistogramsIncompatibleSchema) {
+				return enh.Out, annotations.New().Add(annotations.NewMixedExponentialCustomHistogramsWarning(metricName, args[0].PositionRange()))
+			} else if errors.Is(err, histogram.ErrHistogramsIncompatibleBounds) {
+				return enh.Out, annotations.New().Add(annotations.NewIncompatibleCustomBucketsHistogramsWarning(metricName, args[0].PositionRange()))
+			}
+		}
+		return vec, nil
 	}
 	return aggrOverTime(vals, enh, func(s Series) float64 {
 		var mean, count, c float64
@@ -673,13 +704,25 @@ func funcSumOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNode
 	}
 	if len(firstSeries.Floats) == 0 {
 		// The passed values only contain histograms.
-		return aggrHistOverTime(vals, enh, func(s Series) *histogram.FloatHistogram {
+		vec, err := aggrHistOverTime(vals, enh, func(s Series) (*histogram.FloatHistogram, error) {
 			sum := s.Histograms[0].H.Copy()
 			for _, h := range s.Histograms[1:] {
-				sum.Add(h.H)
+				_, err := sum.Add(h.H)
+				if err != nil {
+					return sum, err
+				}
 			}
-			return sum
-		}), nil
+			return sum, nil
+		})
+		if err != nil {
+			metricName := firstSeries.Metric.Get(labels.MetricName)
+			if errors.Is(err, histogram.ErrHistogramsIncompatibleSchema) {
+				return enh.Out, annotations.New().Add(annotations.NewMixedExponentialCustomHistogramsWarning(metricName, args[0].PositionRange()))
+			} else if errors.Is(err, histogram.ErrHistogramsIncompatibleBounds) {
+				return enh.Out, annotations.New().Add(annotations.NewIncompatibleCustomBucketsHistogramsWarning(metricName, args[0].PositionRange()))
+			}
+		}
+		return vec, nil
 	}
 	return aggrOverTime(vals, enh, func(s Series) float64 {
 		var sum, c float64

promql/parser/generated_parser.y

@@ -84,6 +84,7 @@ BUCKETS_DESC
 NEGATIVE_BUCKETS_DESC
 ZERO_BUCKET_DESC
 ZERO_BUCKET_WIDTH_DESC
+CUSTOM_VALUES_DESC
 %token histogramDescEnd
 
 // Operators.
@@ -797,6 +798,11 @@ histogram_desc_item
                    $$ = yylex.(*parser).newMap()
                    $$["z_bucket_w"] = $3
                 }
+                | CUSTOM_VALUES_DESC COLON bucket_set
+                {
+                   $$ = yylex.(*parser).newMap()
+                   $$["custom_values"] = $3
+                }
                 | BUCKETS_DESC COLON bucket_set
                 {
                    $$ = yylex.(*parser).newMap()

promql/parser/generated_parser.y.go

@@ -67,62 +67,63 @@ const BUCKETS_DESC = 57375
 const NEGATIVE_BUCKETS_DESC = 57376
 const ZERO_BUCKET_DESC = 57377
 const ZERO_BUCKET_WIDTH_DESC = 57378
-const histogramDescEnd = 57379
-const operatorsStart = 57380
-const ADD = 57381
-const DIV = 57382
-const EQLC = 57383
-const EQL_REGEX = 57384
-const GTE = 57385
-const GTR = 57386
-const LAND = 57387
-const LOR = 57388
-const LSS = 57389
-const LTE = 57390
-const LUNLESS = 57391
-const MOD = 57392
-const MUL = 57393
-const NEQ = 57394
-const NEQ_REGEX = 57395
-const POW = 57396
-const SUB = 57397
-const AT = 57398
-const ATAN2 = 57399
-const operatorsEnd = 57400
-const aggregatorsStart = 57401
-const AVG = 57402
-const BOTTOMK = 57403
-const COUNT = 57404
-const COUNT_VALUES = 57405
-const GROUP = 57406
-const MAX = 57407
-const MIN = 57408
-const QUANTILE = 57409
-const STDDEV = 57410
-const STDVAR = 57411
-const SUM = 57412
-const TOPK = 57413
-const aggregatorsEnd = 57414
-const keywordsStart = 57415
-const BOOL = 57416
-const BY = 57417
-const GROUP_LEFT = 57418
-const GROUP_RIGHT = 57419
-const IGNORING = 57420
-const OFFSET = 57421
-const ON = 57422
-const WITHOUT = 57423
-const keywordsEnd = 57424
-const preprocessorStart = 57425
-const START = 57426
-const END = 57427
-const preprocessorEnd = 57428
-const startSymbolsStart = 57429
-const START_METRIC = 57430
-const START_SERIES_DESCRIPTION = 57431
-const START_EXPRESSION = 57432
-const START_METRIC_SELECTOR = 57433
-const startSymbolsEnd = 57434
+const CUSTOM_VALUES_DESC = 57379
+const histogramDescEnd = 57380
+const operatorsStart = 57381
+const ADD = 57382
+const DIV = 57383
+const EQLC = 57384
+const EQL_REGEX = 57385
+const GTE = 57386
+const GTR = 57387
+const LAND = 57388
+const LOR = 57389
+const LSS = 57390
+const LTE = 57391
+const LUNLESS = 57392
+const MOD = 57393
+const MUL = 57394
+const NEQ = 57395
+const NEQ_REGEX = 57396
+const POW = 57397
+const SUB = 57398
+const AT = 57399
+const ATAN2 = 57400
+const operatorsEnd = 57401
+const aggregatorsStart = 57402
+const AVG = 57403
+const BOTTOMK = 57404
+const COUNT = 57405
+const COUNT_VALUES = 57406
+const GROUP = 57407
+const MAX = 57408
+const MIN = 57409
+const QUANTILE = 57410
+const STDDEV = 57411
+const STDVAR = 57412
+const SUM = 57413
+const TOPK = 57414
+const aggregatorsEnd = 57415
+const keywordsStart = 57416
+const BOOL = 57417
+const BY = 57418
+const GROUP_LEFT = 57419
+const GROUP_RIGHT = 57420
+const IGNORING = 57421
+const OFFSET = 57422
+const ON = 57423
+const WITHOUT = 57424
+const keywordsEnd = 57425
+const preprocessorStart = 57426
+const START = 57427
+const END = 57428
+const preprocessorEnd = 57429
+const startSymbolsStart = 57430
+const START_METRIC = 57431
+const START_SERIES_DESCRIPTION = 57432
+const START_EXPRESSION = 57433
+const START_METRIC_SELECTOR = 57434
+const startSymbolsEnd = 57435
 
 var yyToknames = [...]string{
 	"$end",
@@ -161,6 +162,7 @@ var yyToknames = [...]string{
 	"NEGATIVE_BUCKETS_DESC",
 	"ZERO_BUCKET_DESC",
 	"ZERO_BUCKET_WIDTH_DESC",
+	"CUSTOM_VALUES_DESC",
 	"histogramDescEnd",
 	"operatorsStart",
 	"ADD",
@@ -235,270 +237,273 @@ var yyExca = [...]int16{
 	24, 134,
 	-2, 0,
 	-1, 58,
-	2, 171,
-	15, 171,
-	75, 171,
-	81, 171,
-	-2, 100,
-	-1, 59,
 	2, 172,
 	15, 172,
-	75, 172,
-	81, 172,
-	-2, 101,
-	-1, 60,
+	76, 172,
+	82, 172,
+	-2, 100,
+	-1, 59,
 	2, 173,
 	15, 173,
-	75, 173,
-	81, 173,
-	-2, 103,
-	-1, 61,
+	76, 173,
+	82, 173,
+	-2, 101,
+	-1, 60,
 	2, 174,
 	15, 174,
-	75, 174,
-	81, 174,
-	-2, 104,
-	-1, 62,
+	76, 174,
+	82, 174,
+	-2, 103,
+	-1, 61,
 	2, 175,
 	15, 175,
-	75, 175,
-	81, 175,
-	-2, 105,
-	-1, 63,
+	76, 175,
+	82, 175,
+	-2, 104,
+	-1, 62,
 	2, 176,
 	15, 176,
-	75, 176,
-	81, 176,
-	-2, 110,
-	-1, 64,
+	76, 176,
+	82, 176,
+	-2, 105,
+	-1, 63,
 	2, 177,
 	15, 177,
-	75, 177,
-	81, 177,
-	-2, 112,
-	-1, 65,
+	76, 177,
+	82, 177,
+	-2, 110,
+	-1, 64,
 	2, 178,
 	15, 178,
-	75, 178,
-	81, 178,
-	-2, 114,
-	-1, 66,
+	76, 178,
+	82, 178,
+	-2, 112,
+	-1, 65,
 	2, 179,
 	15, 179,
-	75, 179,
-	81, 179,
-	-2, 115,
-	-1, 67,
+	76, 179,
+	82, 179,
+	-2, 114,
+	-1, 66,
 	2, 180,
 	15, 180,
-	75, 180,
-	81, 180,
-	-2, 116,
-	-1, 68,
+	76, 180,
+	82, 180,
+	-2, 115,
+	-1, 67,
 	2, 181,
 	15, 181,
-	75, 181,
-	81, 181,
-	-2, 117,
-	-1, 69,
+	76, 181,
+	82, 181,
+	-2, 116,
+	-1, 68,
 	2, 182,
 	15, 182,
-	75, 182,
-	81, 182,
+	76, 182,
+	82, 182,
+	-2, 117,
+	-1, 69,
+	2, 183,
+	15, 183,
+	76, 183,
+	82, 183,
 	-2, 118,
 	-1, 195,
-	12, 230,
-	13, 230,
-	18, 230,
-	19, 230,
-	25, 230,
-	39, 230,
-	45, 230,
-	46, 230,
-	49, 230,
-	55, 230,
-	60, 230,
-	61, 230,
-	62, 230,
-	63, 230,
-	64, 230,
-	65, 230,
-	66, 230,
-	67, 230,
-	68, 230,
-	69, 230,
-	70, 230,
-	71, 230,
-	75, 230,
-	79, 230,
-	81, 230,
-	84, 230,
-	85, 230,
+	12, 231,
+	13, 231,
+	18, 231,
+	19, 231,
+	25, 231,
+	40, 231,
+	46, 231,
+	47, 231,
+	50, 231,
+	56, 231,
+	61, 231,
+	62, 231,
+	63, 231,
+	64, 231,
+	65, 231,
+	66, 231,
+	67, 231,
+	68, 231,
+	69, 231,
+	70, 231,
+	71, 231,
+	72, 231,
+	76, 231,
+	80, 231,
+	82, 231,
+	85, 231,
+	86, 231,
 	-2, 0,
 	-1, 196,
-	12, 230,
-	13, 230,
-	18, 230,
-	19, 230,
-	25, 230,
-	39, 230,
-	45, 230,
-	46, 230,
-	49, 230,
-	55, 230,
-	60, 230,
-	61, 230,
-	62, 230,
-	63, 230,
-	64, 230,
-	65, 230,
-	66, 230,
-	67, 230,
-	68, 230,
-	69, 230,
-	70, 230,
-	71, 230,
-	75, 230,
-	79, 230,
-	81, 230,
-	84, 230,
-	85, 230,
+	12, 231,
+	13, 231,
+	18, 231,
+	19, 231,
+	25, 231,
+	40, 231,
+	46, 231,
+	47, 231,
+	50, 231,
+	56, 231,
+	61, 231,
+	62, 231,
+	63, 231,
+	64, 231,
+	65, 231,
+	66, 231,
+	67, 231,
+	68, 231,
+	69, 231,
+	70, 231,
+	71, 231,
+	72, 231,
+	76, 231,
+	80, 231,
+	82, 231,
+	85, 231,
+	86, 231,
 	-2, 0,
 	-1, 217,
-	21, 228,
-	-2, 0,
-	-1, 285,
 	21, 229,
 	-2, 0,
+	-1, 286,
+	21, 230,
+	-2, 0,
 }
 
 const yyPrivate = 57344
 
-const yyLast = 742
+const yyLast = 778
 
 var yyAct = [...]int16{
-	151, 322, 320, 268, 327, 148, 221, 37, 187, 144,
-	281, 280, 152, 113, 77, 173, 104, 102, 101, 6,
-	128, 223, 105, 193, 155, 194, 195, 196, 339, 262,
-	260, 233, 317, 316, 57, 100, 294, 239, 103, 146,
-	300, 313, 263, 156, 156, 283, 147, 338, 259, 123,
-	337, 106, 252, 311, 155, 299, 340, 301, 264, 157,
-	157, 108, 298, 109, 235, 236, 292, 251, 237, 107,
-	155, 292, 174, 191, 175, 96, 250, 99, 258, 224,
+	151, 324, 322, 268, 329, 148, 221, 37, 187, 144,
+	282, 281, 152, 113, 77, 173, 104, 102, 101, 6,
+	223, 193, 105, 194, 195, 196, 128, 262, 260, 155,
+	233, 103, 342, 293, 100, 319, 239, 116, 146, 318,
+	315, 263, 156, 123, 106, 147, 284, 114, 295, 116,
+	156, 341, 175, 259, 340, 253, 57, 264, 157, 114,
+	117, 108, 313, 109, 235, 236, 157, 112, 237, 107,
+	323, 174, 117, 175, 155, 96, 250, 99, 293, 224,
 	226, 228, 229, 230, 238, 240, 243, 244, 245, 246,
-	247, 110, 145, 225, 227, 231, 232, 234, 241, 242,
-	98, 257, 321, 248, 249, 2, 3, 4, 5, 218,
-	158, 104, 177, 217, 168, 162, 165, 105, 175, 160,
-	164, 161, 176, 178, 189, 213, 106, 328, 216, 256,
-	183, 179, 192, 163, 181, 100, 190, 197, 198, 199,
+	247, 177, 145, 225, 227, 231, 232, 234, 241, 242,
+	98, 176, 178, 248, 249, 104, 2, 3, 4, 5,
+	158, 105, 177, 110, 168, 162, 165, 302, 150, 160,
+	191, 161, 176, 178, 189, 155, 213, 343, 106, 330,
+	72, 179, 192, 33, 181, 155, 190, 197, 198, 199,
 	200, 201, 202, 203, 204, 205, 206, 207, 208, 209,
-	210, 211, 255, 182, 72, 212, 177, 214, 215, 33,
-	82, 84, 85, 7, 86, 87, 176, 178, 90, 91,
-	223, 93, 94, 95, 116, 96, 97, 99, 83, 147,
-	233, 286, 289, 116, 114, 254, 239, 288, 147, 172,
-	220, 124, 253, 114, 171, 310, 309, 117, 120, 261,
-	98, 112, 287, 119, 278, 279, 117, 170, 282, 10,
-	308, 159, 307, 235, 236, 312, 118, 237, 147, 74,
-	306, 305, 304, 303, 302, 250, 81, 285, 224, 226,
-	228, 229, 230, 238, 240, 243, 244, 245, 246, 247,
-	79, 79, 225, 227, 231, 232, 234, 241, 242, 48,
-	78, 78, 248, 249, 122, 73, 121, 150, 180, 76,
-	290, 291, 293, 56, 295, 8, 9, 9, 34, 35,
-	1, 284, 296, 297, 155, 129, 130, 131, 132, 133,
+	210, 211, 185, 301, 258, 212, 156, 214, 215, 188,
+	256, 183, 290, 191, 252, 164, 155, 289, 300, 218,
+	223, 79, 157, 217, 7, 299, 312, 257, 163, 251,
+	233, 78, 288, 255, 182, 254, 239, 156, 216, 180,
+	220, 124, 172, 120, 147, 311, 314, 171, 119, 261,
+	287, 153, 154, 157, 279, 280, 79, 147, 283, 310,
+	170, 118, 159, 10, 235, 236, 78, 309, 237, 147,
+	308, 307, 306, 74, 76, 305, 250, 286, 304, 224,
+	226, 228, 229, 230, 238, 240, 243, 244, 245, 246,
+	247, 303, 81, 225, 227, 231, 232, 234, 241, 242,
+	48, 34, 1, 248, 249, 122, 73, 121, 285, 47,
+	291, 292, 294, 56, 296, 8, 9, 9, 46, 35,
+	45, 44, 297, 298, 127, 129, 130, 131, 132, 133,
 	134, 135, 136, 137, 138, 139, 140, 141, 142, 143,
-	47, 46, 45, 44, 156, 314, 315, 127, 43, 42,
-	41, 185, 319, 125, 166, 324, 325, 326, 188, 323,
-	157, 329, 191, 331, 330, 155, 40, 126, 332, 333,
-	100, 51, 72, 334, 53, 39, 38, 22, 52, 336,
-	49, 167, 186, 335, 54, 156, 265, 80, 341, 153,
-	154, 184, 219, 75, 115, 82, 84, 149, 70, 55,
-	222, 157, 50, 111, 18, 19, 93, 94, 20, 0,
-	96, 97, 99, 83, 71, 0, 0, 0, 0, 58,
+	43, 42, 41, 125, 166, 40, 316, 317, 126, 39,
+	38, 49, 186, 321, 338, 265, 326, 327, 328, 80,
+	325, 184, 219, 332, 331, 334, 333, 75, 115, 149,
+	335, 336, 100, 51, 72, 337, 53, 55, 222, 22,
+	52, 339, 50, 167, 111, 0, 54, 0, 0, 0,
+	0, 344, 0, 0, 0, 0, 0, 0, 82, 84,
+	0, 70, 0, 0, 0, 0, 0, 18, 19, 93,
+	94, 20, 0, 96, 97, 99, 83, 71, 0, 0,
+	0, 0, 58, 59, 60, 61, 62, 63, 64, 65,
+	66, 67, 68, 69, 0, 0, 0, 13, 98, 0,
+	0, 24, 0, 30, 0, 0, 31, 32, 36, 100,
+	51, 72, 0, 53, 267, 0, 22, 52, 0, 0,
+	0, 266, 0, 54, 0, 270, 271, 269, 276, 278,
+	275, 277, 272, 273, 274, 0, 84, 0, 70, 0,
+	0, 0, 0, 0, 18, 19, 93, 94, 20, 0,
+	96, 0, 99, 83, 71, 0, 0, 0, 0, 58,
 	59, 60, 61, 62, 63, 64, 65, 66, 67, 68,
 	69, 0, 0, 0, 13, 98, 0, 0, 24, 0,
-	30, 0, 0, 31, 32, 36, 100, 51, 72, 0,
-	53, 267, 0, 22, 52, 0, 0, 0, 266, 0,
-	54, 0, 270, 271, 269, 275, 277, 274, 276, 272,
-	273, 0, 84, 0, 70, 0, 0, 0, 0, 0,
-	18, 19, 93, 94, 20, 0, 96, 0, 99, 83,
-	71, 0, 0, 0, 0, 58, 59, 60, 61, 62,
-	63, 64, 65, 66, 67, 68, 69, 0, 0, 0,
-	13, 98, 0, 0, 24, 0, 30, 0, 0, 31,
-	32, 51, 72, 0, 53, 318, 0, 22, 52, 0,
-	0, 0, 0, 0, 54, 0, 270, 271, 269, 275,
-	277, 274, 276, 272, 273, 0, 0, 0, 70, 0,
-	0, 0, 0, 0, 18, 19, 0, 0, 20, 0,
-	0, 0, 17, 72, 71, 0, 0, 0, 22, 58,
-	59, 60, 61, 62, 63, 64, 65, 66, 67, 68,
-	69, 0, 0, 0, 13, 0, 0, 0, 24, 0,
-	30, 0, 0, 31, 32, 18, 19, 0, 0, 20,
-	0, 0, 0, 17, 33, 0, 0, 0, 0, 22,
-	11, 12, 14, 15, 16, 21, 23, 25, 26, 27,
-	28, 29, 0, 0, 0, 13, 0, 0, 0, 24,
-	0, 30, 0, 0, 31, 32, 18, 19, 0, 0,
-	20, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-	0, 11, 12, 14, 15, 16, 21, 23, 25, 26,
-	27, 28, 29, 100, 0, 0, 13, 0, 0, 0,
-	24, 169, 30, 0, 0, 31, 32, 0, 0, 0,
-	0, 0, 100, 0, 0, 0, 0, 0, 82, 84,
-	85, 0, 86, 87, 88, 89, 90, 91, 92, 93,
+	30, 0, 0, 31, 32, 51, 72, 0, 53, 320,
+	0, 22, 52, 0, 0, 0, 0, 0, 54, 0,
+	270, 271, 269, 276, 278, 275, 277, 272, 273, 274,
+	0, 0, 0, 70, 0, 0, 17, 72, 0, 18,
+	19, 0, 22, 20, 0, 0, 0, 0, 0, 71,
+	0, 0, 0, 0, 58, 59, 60, 61, 62, 63,
+	64, 65, 66, 67, 68, 69, 0, 0, 0, 13,
+	18, 19, 0, 24, 20, 30, 0, 0, 31, 32,
+	0, 0, 0, 0, 0, 11, 12, 14, 15, 16,
+	21, 23, 25, 26, 27, 28, 29, 17, 33, 0,
+	13, 0, 0, 22, 24, 0, 30, 0, 0, 31,
+	32, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 18, 19, 0, 0, 20, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 11, 12, 14, 15,
+	16, 21, 23, 25, 26, 27, 28, 29, 100, 0,
+	0, 13, 0, 0, 0, 24, 169, 30, 0, 0,
+	31, 32, 0, 0, 0, 0, 0, 100, 0, 0,
+	0, 0, 0, 0, 82, 84, 85, 0, 86, 87,
+	88, 89, 90, 91, 92, 93, 94, 95, 0, 96,
+	97, 99, 83, 82, 84, 85, 0, 86, 87, 88,
+	89, 90, 91, 92, 93, 94, 95, 0, 96, 97,
+	99, 83, 100, 0, 98, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 100, 0, 98, 0, 0, 0, 0, 82, 84,
+	85, 0, 86, 87, 88, 0, 90, 91, 92, 93,
 	94, 95, 0, 96, 97, 99, 83, 82, 84, 85,
-	0, 86, 87, 88, 89, 90, 91, 92, 93, 94,
-	95, 0, 96, 97, 99, 83, 100, 0, 98, 0,
+	0, 86, 87, 0, 0, 90, 91, 0, 93, 94,
+	95, 0, 96, 97, 99, 83, 0, 0, 98, 0,
 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-	0, 0, 0, 0, 0, 0, 0, 98, 0, 0,
-	0, 82, 84, 85, 0, 86, 87, 88, 0, 90,
-	91, 92, 93, 94, 95, 0, 96, 97, 99, 83,
-	0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-	0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-	0, 98,
+	0, 0, 0, 0, 0, 0, 0, 98,
 }
 
 var yyPact = [...]int16{
-	17, 153, 541, 541, 385, 500, -1000, -1000, -1000, 146,
+	17, 164, 555, 555, 388, 494, -1000, -1000, -1000, 120,
 	-1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000,
 	-1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000,
-	-1000, -1000, -1000, 239, -1000, 224, -1000, 618, -1000, -1000,
+	-1000, -1000, -1000, 204, -1000, 240, -1000, 633, -1000, -1000,
 	-1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000,
-	36, 111, -1000, 459, -1000, 459, 141, -1000, -1000, -1000,
+	29, 113, -1000, 463, -1000, 463, 117, -1000, -1000, -1000,
 	-1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000,
-	-1000, -1000, 181, -1000, -1000, 196, -1000, -1000, 252, -1000,
-	25, -1000, -54, -54, -54, -54, -54, -54, -54, -54,
-	-54, -54, -54, -54, -54, -54, -54, -54, 37, 255,
-	209, 111, -59, -1000, 118, 118, 309, -1000, 599, 21,
-	-1000, 187, -1000, -1000, 70, 114, -1000, -1000, -1000, 238,
-	-1000, 128, -1000, 296, 459, -1000, -55, -50, -1000, 459,
-	459, 459, 459, 459, 459, 459, 459, 459, 459, 459,
-	459, 459, 459, 459, -1000, 170, -1000, -1000, -1000, 110,
-	-1000, -1000, -1000, -1000, -1000, -1000, 51, 51, 107, -1000,
-	-1000, -1000, -1000, 168, -1000, -1000, 45, -1000, 618, -1000,
-	-1000, 172, -1000, 127, -1000, -1000, -1000, -1000, -1000, 76,
-	-1000, -1000, -1000, -1000, -1000, 22, 4, 3, -1000, -1000,
-	-1000, 384, 382, 118, 118, 118, 118, 21, 21, 306,
-	306, 306, 121, 662, 306, 306, 121, 21, 21, 306,
-	21, 382, -1000, 23, -1000, -1000, -1000, 179, -1000, 180,
+	-1000, -1000, 47, -1000, -1000, 191, -1000, -1000, 253, -1000,
+	19, -1000, -49, -49, -49, -49, -49, -49, -49, -49,
+	-49, -49, -49, -49, -49, -49, -49, -49, 36, 116,
+	210, 113, -60, -1000, 163, 163, 311, -1000, 614, 20,
+	-1000, 190, -1000, -1000, 69, 48, -1000, -1000, -1000, 169,
+	-1000, 159, -1000, 147, 463, -1000, -58, -53, -1000, 463,
+	463, 463, 463, 463, 463, 463, 463, 463, 463, 463,
+	463, 463, 463, 463, -1000, 185, -1000, -1000, -1000, 111,
+	-1000, -1000, -1000, -1000, -1000, -1000, 55, 55, 167, -1000,
+	-1000, -1000, -1000, 168, -1000, -1000, 157, -1000, 633, -1000,
+	-1000, 35, -1000, 158, -1000, -1000, -1000, -1000, -1000, 152,
+	-1000, -1000, -1000, -1000, -1000, 27, 2, 1, -1000, -1000,
+	-1000, 387, 385, 163, 163, 163, 163, 20, 20, 308,
+	308, 308, 697, 678, 308, 308, 697, 20, 20, 308,
+	20, 385, -1000, 24, -1000, -1000, -1000, 198, -1000, 160,
 	-1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000,
 	-1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000,
 	-1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000,
-	-1000, -1000, 459, -1000, -1000, -1000, -1000, -1000, -1000, 52,
-	52, 10, 52, 57, 57, 38, 40, -1000, -1000, 218,
-	217, 216, 215, 214, 206, 204, 190, 189, -1000, -1000,
-	-1000, -1000, -1000, -1000, 32, 213, -1000, -1000, 19, -1000,
-	618, -1000, -1000, -1000, 52, -1000, 7, 6, 458, -1000,
-	-1000, -1000, 47, 5, 51, 51, 51, 113, 47, 113,
-	47, -1000, -1000, -1000, -1000, -1000, 52, 52, -1000, -1000,
-	-1000, 52, -1000, -1000, -1000, -1000, -1000, -1000, 51, -1000,
-	-1000, -1000, -1000, -1000, -1000, 26, -1000, 35, -1000, -1000,
-	-1000, -1000,
+	-1000, -1000, 463, -1000, -1000, -1000, -1000, -1000, -1000, 59,
+	59, 22, 59, 104, 104, 151, 100, -1000, -1000, 235,
+	222, 219, 216, 215, 214, 211, 203, 189, 170, -1000,
+	-1000, -1000, -1000, -1000, -1000, 41, 194, -1000, -1000, 18,
+	-1000, 633, -1000, -1000, -1000, 59, -1000, 13, 9, 462,
+	-1000, -1000, -1000, 14, 10, 55, 55, 55, 115, 115,
+	14, 115, 14, -1000, -1000, -1000, -1000, -1000, 59, 59,
+	-1000, -1000, -1000, 59, -1000, -1000, -1000, -1000, -1000, -1000,
+	55, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 30, -1000,
+	106, -1000, -1000, -1000, -1000,
 }
 
 var yyPgo = [...]int16{
-	0, 353, 13, 352, 6, 15, 350, 263, 349, 347,
-	344, 209, 265, 343, 14, 342, 10, 11, 341, 337,
-	8, 336, 3, 4, 333, 2, 1, 0, 332, 12,
-	5, 330, 326, 18, 191, 325, 317, 7, 316, 304,
-	17, 303, 34, 300, 299, 298, 297, 293, 292, 291,
-	290, 249, 9, 271, 270, 268,
+	0, 334, 13, 332, 6, 15, 328, 263, 327, 319,
+	318, 213, 265, 317, 14, 312, 10, 11, 311, 309,
+	8, 305, 3, 4, 304, 2, 1, 0, 302, 12,
+	5, 301, 300, 18, 191, 299, 298, 7, 295, 294,
+	17, 293, 56, 292, 291, 290, 274, 271, 270, 268,
+	259, 250, 9, 258, 252, 251,
 }
 
 var yyR1 = [...]int8{
@@ -518,14 +523,14 @@ var yyR1 = [...]int8{
 	14, 14, 14, 55, 19, 19, 19, 19, 18, 18,
 	18, 18, 18, 18, 18, 18, 18, 28, 28, 28,
 	20, 20, 20, 20, 21, 21, 21, 22, 22, 22,
-	22, 22, 22, 22, 22, 22, 23, 23, 24, 24,
-	24, 3, 3, 3, 3, 3, 3, 3, 3, 3,
-	3, 3, 3, 6, 6, 6, 6, 6, 6, 6,
+	22, 22, 22, 22, 22, 22, 22, 23, 23, 24,
+	24, 24, 3, 3, 3, 3, 3, 3, 3, 3,
+	3, 3, 3, 3, 6, 6, 6, 6, 6, 6,
 	6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
 	6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
-	8, 8, 5, 5, 5, 5, 44, 27, 29, 29,
-	30, 30, 26, 25, 25, 52, 48, 10, 53, 53,
-	17, 17,
+	6, 8, 8, 5, 5, 5, 5, 44, 27, 29,
+	29, 30, 30, 26, 25, 25, 52, 48, 10, 53,
+	53, 17, 17,
 }
 
 var yyR2 = [...]int8{
@@ -545,52 +550,52 @@ var yyR2 = [...]int8{
 	3, 2, 1, 2, 0, 3, 2, 1, 1, 3,
 	1, 3, 4, 1, 3, 5, 5, 1, 1, 1,
 	4, 3, 3, 2, 3, 1, 2, 3, 3, 3,
-	3, 3, 3, 3, 3, 3, 4, 3, 3, 1,
-	2, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+	3, 3, 3, 3, 3, 3, 3, 4, 3, 3,
+	1, 2, 1, 1, 1, 1, 1, 1, 1, 1,
 	1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
 	1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
 	1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-	1, 1, 1, 1, 1, 1, 1, 1, 2, 2,
-	1, 1, 1, 2, 1, 1, 1, 1, 0, 1,
-	0, 1,
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 2,
+	2, 1, 1, 1, 2, 1, 1, 1, 1, 0,
+	1, 0, 1,
 }
 
 var yyChk = [...]int16{
-	-1000, -54, 88, 89, 90, 91, 2, 10, -12, -7,
-	-11, 60, 61, 75, 62, 63, 64, 12, 45, 46,
-	49, 65, 18, 66, 79, 67, 68, 69, 70, 71,
-	81, 84, 85, 13, -55, -12, 10, -37, -32, -35,
+	-1000, -54, 89, 90, 91, 92, 2, 10, -12, -7,
+	-11, 61, 62, 76, 63, 64, 65, 12, 46, 47,
+	50, 66, 18, 67, 80, 68, 69, 70, 71, 72,
+	82, 85, 86, 13, -55, -12, 10, -37, -32, -35,
 	-38, -43, -44, -45, -47, -48, -49, -50, -51, -31,
-	-3, 12, 19, 15, 25, -8, -7, -42, 60, 61,
-	62, 63, 64, 65, 66, 67, 68, 69, 70, 71,
-	39, 55, 13, -51, -11, -13, 20, -14, 12, 2,
-	-19, 2, 39, 57, 40, 41, 43, 44, 45, 46,
-	47, 48, 49, 50, 51, 52, 54, 55, 79, 56,
-	14, -33, -40, 2, 75, 81, 15, -40, -37, -37,
+	-3, 12, 19, 15, 25, -8, -7, -42, 61, 62,
+	63, 64, 65, 66, 67, 68, 69, 70, 71, 72,
+	40, 56, 13, -51, -11, -13, 20, -14, 12, 2,
+	-19, 2, 40, 58, 41, 42, 44, 45, 46, 47,
+	48, 49, 50, 51, 52, 53, 55, 56, 80, 57,
+	14, -33, -40, 2, 76, 82, 15, -40, -37, -37,
 	-42, -1, 20, -2, 12, -10, 2, 25, 20, 7,
-	2, 4, 2, 24, -34, -41, -36, -46, 74, -34,
+	2, 4, 2, 24, -34, -41, -36, -46, 75, -34,
 	-34, -34, -34, -34, -34, -34, -34, -34, -34, -34,
-	-34, -34, -34, -34, -52, 55, 2, 9, -30, -9,
-	2, -27, -29, 84, 85, 19, 39, 55, -52, 2,
+	-34, -34, -34, -34, -52, 56, 2, 9, -30, -9,
+	2, -27, -29, 85, 86, 19, 40, 56, -52, 2,
 	-40, -33, -16, 15, 2, -16, -39, 22, -37, 22,
-	20, 7, 2, -5, 2, 4, 52, 42, 53, -5,
+	20, 7, 2, -5, 2, 4, 53, 43, 54, -5,
 	20, -14, 25, 2, -18, 5, -28, -20, 12, -27,
-	-29, 16, -37, 78, 80, 76, 77, -37, -37, -37,
+	-29, 16, -37, 79, 81, 77, 78, -37, -37, -37,
 	-37, -37, -37, -37, -37, -37, -37, -37, -37, -37,
 	-37, -37, -52, 15, -27, -27, 21, 6, 2, -15,
-	22, -4, -6, 2, 60, 74, 61, 75, 62, 63,
-	64, 76, 77, 12, 78, 45, 46, 49, 65, 18,
-	66, 79, 80, 67, 68, 69, 70, 71, 84, 85,
-	57, 22, 7, 20, -2, 25, 2, 25, 2, 26,
-	26, -29, 26, 39, 55, -21, 24, 17, -22, 30,
-	28, 29, 35, 36, 33, 31, 34, 32, -16, -16,
-	-17, -16, -17, 22, -53, -52, 2, 22, 7, 2,
-	-37, -26, 19, -26, 26, -26, -20, -20, 24, 17,
-	2, 17, 6, 6, 6, 6, 6, 6, 6, 6,
-	6, 21, 2, 22, -4, -26, 26, 26, 17, -22,
-	-25, 55, -26, -30, -27, -27, -27, -23, 14, -25,
-	-23, -25, -26, -26, -26, -24, -27, 24, 21, 2,
-	21, -27,
+	22, -4, -6, 2, 61, 75, 62, 76, 63, 64,
+	65, 77, 78, 12, 79, 46, 47, 50, 66, 18,
+	67, 80, 81, 68, 69, 70, 71, 72, 85, 86,
+	58, 22, 7, 20, -2, 25, 2, 25, 2, 26,
+	26, -29, 26, 40, 56, -21, 24, 17, -22, 30,
+	28, 29, 35, 36, 37, 33, 31, 34, 32, -16,
+	-16, -17, -16, -17, 22, -53, -52, 2, 22, 7,
+	2, -37, -26, 19, -26, 26, -26, -20, -20, 24,
+	17, 2, 17, 6, 6, 6, 6, 6, 6, 6,
+	6, 6, 6, 21, 2, 22, -4, -26, 26, 26,
+	17, -22, -25, 56, -26, -30, -27, -27, -27, -23,
+	14, -23, -25, -23, -25, -26, -26, -26, -24, -27,
+	24, 21, 2, 21, -27,
 }
 
 var yyDef = [...]int16{
@@ -599,36 +604,36 @@ var yyDef = [...]int16{
 	109, 110, 111, 112, 113, 114, 115, 116, 117, 118,
 	119, 120, 121, 0, 2, -2, 3, 4, 8, 9,
 	10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
-	0, 106, 216, 0, 226, 0, 83, 84, -2, -2,
+	0, 106, 217, 0, 227, 0, 83, 84, -2, -2,
 	-2, -2, -2, -2, -2, -2, -2, -2, -2, -2,
-	210, 211, 0, 5, 98, 0, 124, 127, 0, 132,
+	211, 212, 0, 5, 98, 0, 124, 127, 0, 132,
 	133, 137, 43, 43, 43, 43, 43, 43, 43, 43,
 	43, 43, 43, 43, 43, 43, 43, 43, 0, 0,
 	0, 0, 22, 23, 0, 0, 0, 60, 0, 81,
-	82, 0, 87, 89, 0, 93, 97, 227, 122, 0,
+	82, 0, 87, 89, 0, 93, 97, 228, 122, 0,
 	128, 0, 131, 136, 0, 42, 47, 48, 44, 0,
 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-	0, 0, 0, 0, 67, 0, 69, 225, 70, 0,
-	72, 220, 221, 73, 74, 217, 0, 0, 0, 80,
+	0, 0, 0, 0, 67, 0, 69, 226, 70, 0,
+	72, 221, 222, 73, 74, 218, 0, 0, 0, 80,
 	20, 21, 24, 0, 54, 25, 0, 62, 64, 66,
-	85, 0, 90, 0, 96, 212, 213, 214, 215, 0,
+	85, 0, 90, 0, 96, 213, 214, 215, 216, 0,
 	123, 126, 129, 130, 135, 138, 140, 143, 147, 148,
 	149, 0, 26, 0, 0, -2, -2, 27, 28, 29,
 	30, 31, 32, 33, 34, 35, 36, 37, 38, 39,
-	40, 41, 68, 0, 218, 219, 75, -2, 79, 0,
-	53, 56, 58, 59, 183, 184, 185, 186, 187, 188,
-	189, 190, 191, 192, 193, 194, 195, 196, 197, 198,
-	199, 200, 201, 202, 203, 204, 205, 206, 207, 208,
-	209, 61, 65, 86, 88, 91, 95, 92, 94, 0,
+	40, 41, 68, 0, 219, 220, 75, -2, 79, 0,
+	53, 56, 58, 59, 184, 185, 186, 187, 188, 189,
+	190, 191, 192, 193, 194, 195, 196, 197, 198, 199,
+	200, 201, 202, 203, 204, 205, 206, 207, 208, 209,
+	210, 61, 65, 86, 88, 91, 95, 92, 94, 0,
 	0, 0, 0, 0, 0, 0, 0, 153, 155, 0,
-	0, 0, 0, 0, 0, 0, 0, 0, 45, 46,
-	49, 231, 50, 71, 0, -2, 78, 51, 0, 57,
-	63, 139, 222, 141, 0, 144, 0, 0, 0, 151,
-	156, 152, 0, 0, 0, 0, 0, 0, 0, 0,
-	0, 76, 77, 52, 55, 142, 0, 0, 150, 154,
-	157, 0, 224, 158, 159, 160, 161, 162, 0, 163,
-	164, 165, 145, 146, 223, 0, 169, 0, 167, 170,
-	166, 168,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 45,
+	46, 49, 232, 50, 71, 0, -2, 78, 51, 0,
+	57, 63, 139, 223, 141, 0, 144, 0, 0, 0,
+	151, 156, 152, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 76, 77, 52, 55, 142, 0, 0,
+	150, 154, 157, 0, 225, 158, 159, 160, 161, 162,
+	0, 163, 164, 165, 166, 145, 146, 224, 0, 170,
+	0, 168, 171, 167, 169,
 }
 
 var yyTok1 = [...]int8{
@@ -645,7 +650,7 @@ var yyTok2 = [...]int8{
 	62, 63, 64, 65, 66, 67, 68, 69, 70, 71,
 	72, 73, 74, 75, 76, 77, 78, 79, 80, 81,
 	82, 83, 84, 85, 86, 87, 88, 89, 90, 91,
-	92,
+	92, 93,
 }
 
 var yyTok3 = [...]int8{
@@ -1738,47 +1743,53 @@ yydefault:
 		yyDollar = yyS[yypt-3 : yypt+1]
 		{
 			yyVAL.descriptors = yylex.(*parser).newMap()
-			yyVAL.descriptors["buckets"] = yyDollar[3].bucket_set
+			yyVAL.descriptors["custom_values"] = yyDollar[3].bucket_set
 		}
 	case 163:
 		yyDollar = yyS[yypt-3 : yypt+1]
 		{
 			yyVAL.descriptors = yylex.(*parser).newMap()
-			yyVAL.descriptors["offset"] = yyDollar[3].int
+			yyVAL.descriptors["buckets"] = yyDollar[3].bucket_set
 		}
 	case 164:
 		yyDollar = yyS[yypt-3 : yypt+1]
 		{
 			yyVAL.descriptors = yylex.(*parser).newMap()
-			yyVAL.descriptors["n_buckets"] = yyDollar[3].bucket_set
+			yyVAL.descriptors["offset"] = yyDollar[3].int
 		}
 	case 165:
 		yyDollar = yyS[yypt-3 : yypt+1]
 		{
 			yyVAL.descriptors = yylex.(*parser).newMap()
-			yyVAL.descriptors["n_offset"] = yyDollar[3].int
+			yyVAL.descriptors["n_buckets"] = yyDollar[3].bucket_set
 		}
 	case 166:
-		yyDollar = yyS[yypt-4 : yypt+1]
+		yyDollar = yyS[yypt-3 : yypt+1]
 		{
-			yyVAL.bucket_set = yyDollar[2].bucket_set
+			yyVAL.descriptors = yylex.(*parser).newMap()
+			yyVAL.descriptors["n_offset"] = yyDollar[3].int
 		}
 	case 167:
-		yyDollar = yyS[yypt-3 : yypt+1]
+		yyDollar = yyS[yypt-4 : yypt+1]
 		{
 			yyVAL.bucket_set = yyDollar[2].bucket_set
 		}
 	case 168:
 		yyDollar = yyS[yypt-3 : yypt+1]
 		{
-			yyVAL.bucket_set = append(yyDollar[1].bucket_set, yyDollar[3].float)
+			yyVAL.bucket_set = yyDollar[2].bucket_set
 		}
 	case 169:
+		yyDollar = yyS[yypt-3 : yypt+1]
+		{
+			yyVAL.bucket_set = append(yyDollar[1].bucket_set, yyDollar[3].float)
+		}
+	case 170:
 		yyDollar = yyS[yypt-1 : yypt+1]
 		{
 			yyVAL.bucket_set = []float64{yyDollar[1].float}
 		}
-	case 216:
+	case 217:
 		yyDollar = yyS[yypt-1 : yypt+1]
 		{
 			yyVAL.node = &NumberLiteral{
@@ -1786,22 +1797,22 @@ yydefault:
 				PosRange: yyDollar[1].item.PositionRange(),
 			}
 		}
-	case 217:
+	case 218:
 		yyDollar = yyS[yypt-1 : yypt+1]
 		{
 			yyVAL.float = yylex.(*parser).number(yyDollar[1].item.Val)
 		}
-	case 218:
+	case 219:
 		yyDollar = yyS[yypt-2 : yypt+1]
 		{
 			yyVAL.float = yyDollar[2].float
 		}
-	case 219:
+	case 220:
 		yyDollar = yyS[yypt-2 : yypt+1]
 		{
 			yyVAL.float = -yyDollar[2].float
 		}
-	case 222:
+	case 223:
 		yyDollar = yyS[yypt-1 : yypt+1]
 		{
 			var err error
@@ -1810,17 +1821,17 @@ yydefault:
 				yylex.(*parser).addParseErrf(yyDollar[1].item.PositionRange(), "invalid repetition in series values: %s", err)
 			}
 		}
-	case 223:
+	case 224:
 		yyDollar = yyS[yypt-2 : yypt+1]
 		{
 			yyVAL.int = -int64(yyDollar[2].uint)
 		}
-	case 224:
+	case 225:
 		yyDollar = yyS[yypt-1 : yypt+1]
 		{
 			yyVAL.int = int64(yyDollar[1].uint)
 		}
-	case 225:
+	case 226:
 		yyDollar = yyS[yypt-1 : yypt+1]
 		{
 			var err error
@@ -1829,7 +1840,7 @@ yydefault:
 				yylex.(*parser).addParseErr(yyDollar[1].item.PositionRange(), err)
 			}
 		}
-	case 226:
+	case 227:
 		yyDollar = yyS[yypt-1 : yypt+1]
 		{
 			yyVAL.node = &StringLiteral{
@@ -1837,7 +1848,7 @@ yydefault:
 				PosRange: yyDollar[1].item.PositionRange(),
 			}
 		}
-	case 227:
+	case 228:
 		yyDollar = yyS[yypt-1 : yypt+1]
 		{
 			yyVAL.item = Item{
@@ -1846,12 +1857,12 @@ yydefault:
 				Val: yylex.(*parser).unquoteString(yyDollar[1].item.Val),
 			}
 		}
-	case 228:
+	case 229:
 		yyDollar = yyS[yypt-0 : yypt+1]
 		{
 			yyVAL.duration = 0
 		}
-	case 230:
+	case 231:
 		yyDollar = yyS[yypt-0 : yypt+1]
 		{
 			yyVAL.strings = nil

promql/parser/lex.go

@@ -135,15 +135,16 @@ var key = map[string]ItemType{
 }
 
 var histogramDesc = map[string]ItemType{
-	"sum":        SUM_DESC,
-	"count":      COUNT_DESC,
-	"schema":     SCHEMA_DESC,
-	"offset":     OFFSET_DESC,
-	"n_offset":   NEGATIVE_OFFSET_DESC,
-	"buckets":    BUCKETS_DESC,
-	"n_buckets":  NEGATIVE_BUCKETS_DESC,
-	"z_bucket":   ZERO_BUCKET_DESC,
-	"z_bucket_w": ZERO_BUCKET_WIDTH_DESC,
+	"sum":           SUM_DESC,
+	"count":         COUNT_DESC,
+	"schema":        SCHEMA_DESC,
+	"offset":        OFFSET_DESC,
+	"n_offset":      NEGATIVE_OFFSET_DESC,
+	"buckets":       BUCKETS_DESC,
+	"n_buckets":     NEGATIVE_BUCKETS_DESC,
+	"z_bucket":      ZERO_BUCKET_DESC,
+	"z_bucket_w":    ZERO_BUCKET_WIDTH_DESC,
+	"custom_values": CUSTOM_VALUES_DESC,
 }
 
 // ItemTypeStr is the default string representations for common Items. It does not

promql/parser/parse.go

@@ -481,19 +481,19 @@ func (p *parser) mergeMaps(left, right *map[string]interface{}) (ret *map[string
 }
 
 func (p *parser) histogramsIncreaseSeries(base, inc *histogram.FloatHistogram, times uint64) ([]SequenceValue, error) {
-	return p.histogramsSeries(base, inc, times, func(a, b *histogram.FloatHistogram) *histogram.FloatHistogram {
+	return p.histogramsSeries(base, inc, times, func(a, b *histogram.FloatHistogram) (*histogram.FloatHistogram, error) {
 		return a.Add(b)
 	})
 }
 
 func (p *parser) histogramsDecreaseSeries(base, inc *histogram.FloatHistogram, times uint64) ([]SequenceValue, error) {
-	return p.histogramsSeries(base, inc, times, func(a, b *histogram.FloatHistogram) *histogram.FloatHistogram {
+	return p.histogramsSeries(base, inc, times, func(a, b *histogram.FloatHistogram) (*histogram.FloatHistogram, error) {
 		return a.Sub(b)
 	})
 }
 
 func (p *parser) histogramsSeries(base, inc *histogram.FloatHistogram, times uint64,
-	combine func(*histogram.FloatHistogram, *histogram.FloatHistogram) *histogram.FloatHistogram,
+	combine func(*histogram.FloatHistogram, *histogram.FloatHistogram) (*histogram.FloatHistogram, error),
 ) ([]SequenceValue, error) {
 	ret := make([]SequenceValue, times+1)
 	// Add an additional value (the base) for time 0, which we ignore in tests.
@@ -504,7 +504,11 @@ func (p *parser) histogramsSeries(base, inc *histogram.FloatHistogram, times uin
 			return nil, fmt.Errorf("error combining histograms: cannot merge from schema %d to %d", inc.Schema, cur.Schema)
 		}
 
-		cur = combine(cur.Copy(), inc)
+		var err error
+		cur, err = combine(cur.Copy(), inc)
+		if err != nil {
+			return ret, err
+		}
 		ret[i] = SequenceValue{Histogram: cur}
 	}
 
@@ -562,6 +566,15 @@ func (p *parser) buildHistogramFromMap(desc *map[string]interface{}) *histogram.
 			p.addParseErrf(p.yyParser.lval.item.PositionRange(), "error parsing z_bucket_w number: %v", val)
 		}
 	}
+	val, ok = (*desc)["custom_values"]
+	if ok {
+		customValues, ok := val.([]float64)
+		if ok {
+			output.CustomValues = customValues
+		} else {
+			p.addParseErrf(p.yyParser.lval.item.PositionRange(), "error parsing custom_values: %v", val)
+		}
+	}
 
 	buckets, spans := p.buildHistogramBucketsAndSpans(desc, "buckets", "offset")
 	output.PositiveBuckets = buckets

promql/promqltest/README.md

@@ -63,6 +63,10 @@ load 1m
 Each `load` command is additive - it does not replace any data loaded in a previous `load` command.
 Use `clear` to remove all loaded data.
 
+### Native histograms with custom buckets (NHCB)
+
+When loading a batch of classic histogram float series, you can optionally append the suffix `_with_nhcb` to convert them to native histograms with custom buckets and load both the original float series and the new histogram series.
+
 ## `clear` command
 
 `clear` removes all data previously loaded with `load` commands.

promql/promqltest/test.go

@@ -19,6 +19,8 @@ import (
 	"errors"
 	"fmt"
 	"io/fs"
+	"math"
+	"sort"
 	"strconv"
 	"strings"
 	"testing"
@@ -43,9 +45,9 @@ import (
 
 var (
 	patSpace       = regexp.MustCompile("[\t ]+")
-	patLoad        = regexp.MustCompile(`^load\s+(.+?)$`)
-	patEvalInstant = regexp.MustCompile(`^eval(?:_(fail|ordered))?\s+instant\s+(?:at\s+(.+?))?\s+(.+)$`)
-	patEvalRange   = regexp.MustCompile(`^eval(?:_(fail))?\s+range\s+from\s+(.+)\s+to\s+(.+)\s+step\s+(.+?)\s+(.+)$`)
+	patLoad        = regexp.MustCompile(`^load(?:_(with_nhcb))?\s+(.+?)$`)
+	patEvalInstant = regexp.MustCompile(`^eval(?:_(fail|warn|ordered))?\s+instant\s+(?:at\s+(.+?))?\s+(.+)$`)
+	patEvalRange   = regexp.MustCompile(`^eval(?:_(fail|warn))?\s+range\s+from\s+(.+)\s+to\s+(.+)\s+step\s+(.+?)\s+(.+)$`)
 )
 
 const (
@@ -177,15 +179,18 @@ func raise(line int, format string, v ...interface{}) error {
 
 func parseLoad(lines []string, i int) (int, *loadCmd, error) {
 	if !patLoad.MatchString(lines[i]) {
-		return i, nil, raise(i, "invalid load command. (load <step:duration>)")
+		return i, nil, raise(i, "invalid load command. (load[_with_nhcb] <step:duration>)")
 	}
 	parts := patLoad.FindStringSubmatch(lines[i])
-
-	gap, err := model.ParseDuration(parts[1])
+	var (
+		withNHCB = parts[1] == "with_nhcb"
+		step     = parts[2]
+	)
+	gap, err := model.ParseDuration(step)
 	if err != nil {
-		return i, nil, raise(i, "invalid step definition %q: %s", parts[1], err)
+		return i, nil, raise(i, "invalid step definition %q: %s", step, err)
 	}
-	cmd := newLoadCmd(time.Duration(gap))
+	cmd := newLoadCmd(time.Duration(gap), withNHCB)
 	for i+1 < len(lines) {
 		i++
 		defLine := lines[i]
@@ -218,7 +223,7 @@ func (t *test) parseEval(lines []string, i int) (int, *evalCmd, error) {
 	rangeParts := patEvalRange.FindStringSubmatch(lines[i])
 
 	if instantParts == nil && rangeParts == nil {
-		return i, nil, raise(i, "invalid evaluation command. Must be either 'eval[_fail|_ordered] instant [at <offset:duration>] <query>' or 'eval[_fail] range from <from> to <to> step <step> <query>'")
+		return i, nil, raise(i, "invalid evaluation command. Must be either 'eval[_fail|_warn|_ordered] instant [at <offset:duration>] <query>' or 'eval[_fail|_warn] range from <from> to <to> step <step> <query>'")
 	}
 
 	isInstant := instantParts != nil
@@ -297,6 +302,8 @@ func (t *test) parseEval(lines []string, i int) (int, *evalCmd, error) {
 		cmd.ordered = true
 	case "fail":
 		cmd.fail = true
+	case "warn":
+		cmd.warn = true
 	}
 
 	for j := 1; i+1 < len(lines); j++ {
@@ -367,7 +374,7 @@ func (t *test) parse(input string) error {
 		switch c := strings.ToLower(patSpace.Split(l, 2)[0]); {
 		case c == "clear":
 			cmd = &clearCmd{}
-		case c == "load":
+		case strings.HasPrefix(c, "load"):
 			i, cmd, err = parseLoad(lines, i)
 		case strings.HasPrefix(c, "eval"):
 			i, cmd, err = t.parseEval(lines, i)
@@ -399,14 +406,16 @@ type loadCmd struct {
 	metrics   map[uint64]labels.Labels
 	defs      map[uint64][]promql.Sample
 	exemplars map[uint64][]exemplar.Exemplar
+	withNHCB  bool
 }
 
-func newLoadCmd(gap time.Duration) *loadCmd {
+func newLoadCmd(gap time.Duration, withNHCB bool) *loadCmd {
 	return &loadCmd{
 		gap:       gap,
 		metrics:   map[uint64]labels.Labels{},
 		defs:      map[uint64][]promql.Sample{},
 		exemplars: map[uint64][]exemplar.Exemplar{},
+		withNHCB:  withNHCB,
 	}
 }
 
@@ -445,6 +454,167 @@ func (cmd *loadCmd) append(a storage.Appender) error {
 			}
 		}
 	}
+	if cmd.withNHCB {
+		return cmd.appendCustomHistogram(a)
+	}
+	return nil
+}
+
+func getHistogramMetricBase(m labels.Labels, suffix string) (labels.Labels, uint64) {
+	mName := m.Get(labels.MetricName)
+	baseM := labels.NewBuilder(m).
+		Set(labels.MetricName, strings.TrimSuffix(mName, suffix)).
+		Del(labels.BucketLabel).
+		Labels()
+	hash := baseM.Hash()
+	return baseM, hash
+}
+
+type tempHistogramWrapper struct {
+	metric        labels.Labels
+	upperBounds   []float64
+	histogramByTs map[int64]tempHistogram
+}
+
+func newTempHistogramWrapper() tempHistogramWrapper {
+	return tempHistogramWrapper{
+		upperBounds:   []float64{},
+		histogramByTs: map[int64]tempHistogram{},
+	}
+}
+
+type tempHistogram struct {
+	bucketCounts map[float64]float64
+	count        float64
+	sum          float64
+}
+
+func newTempHistogram() tempHistogram {
+	return tempHistogram{
+		bucketCounts: map[float64]float64{},
+	}
+}
+
+func processClassicHistogramSeries(m labels.Labels, suffix string, histogramMap map[uint64]tempHistogramWrapper, smpls []promql.Sample, updateHistogramWrapper func(*tempHistogramWrapper), updateHistogram func(*tempHistogram, float64)) {
+	m2, m2hash := getHistogramMetricBase(m, suffix)
+	histogramWrapper, exists := histogramMap[m2hash]
+	if !exists {
+		histogramWrapper = newTempHistogramWrapper()
+	}
+	histogramWrapper.metric = m2
+	if updateHistogramWrapper != nil {
+		updateHistogramWrapper(&histogramWrapper)
+	}
+	for _, s := range smpls {
+		if s.H != nil {
+			continue
+		}
+		histogram, exists := histogramWrapper.histogramByTs[s.T]
+		if !exists {
+			histogram = newTempHistogram()
+		}
+		updateHistogram(&histogram, s.F)
+		histogramWrapper.histogramByTs[s.T] = histogram
+	}
+	histogramMap[m2hash] = histogramWrapper
+}
+
+func processUpperBoundsAndCreateBaseHistogram(upperBounds0 []float64) ([]float64, *histogram.FloatHistogram) {
+	sort.Float64s(upperBounds0)
+	upperBounds := make([]float64, 0, len(upperBounds0))
+	prevLE := math.Inf(-1)
+	for _, le := range upperBounds0 {
+		if le != prevLE { // deduplicate
+			upperBounds = append(upperBounds, le)
+			prevLE = le
+		}
+	}
+	var customBounds []float64
+	if upperBounds[len(upperBounds)-1] == math.Inf(1) {
+		customBounds = upperBounds[:len(upperBounds)-1]
+	} else {
+		customBounds = upperBounds
+	}
+	return upperBounds, &histogram.FloatHistogram{
+		Count:  0,
+		Sum:    0,
+		Schema: histogram.CustomBucketsSchema,
+		PositiveSpans: []histogram.Span{
+			{Offset: 0, Length: uint32(len(upperBounds))},
+		},
+		PositiveBuckets: make([]float64, len(upperBounds)),
+		CustomValues:    customBounds,
+	}
+}
+
+// If classic histograms are defined, convert them into native histograms with custom
+// bounds and append the defined time series to the storage.
+func (cmd *loadCmd) appendCustomHistogram(a storage.Appender) error {
+	histogramMap := map[uint64]tempHistogramWrapper{}
+
+	// Go through all the time series to collate classic histogram data
+	// and organise them by timestamp.
+	for hash, smpls := range cmd.defs {
+		m := cmd.metrics[hash]
+		mName := m.Get(labels.MetricName)
+		switch {
+		case strings.HasSuffix(mName, "_bucket") && m.Has(labels.BucketLabel):
+			le, err := strconv.ParseFloat(m.Get(labels.BucketLabel), 64)
+			if err != nil || math.IsNaN(le) {
+				continue
+			}
+			processClassicHistogramSeries(m, "_bucket", histogramMap, smpls, func(histogramWrapper *tempHistogramWrapper) {
+				histogramWrapper.upperBounds = append(histogramWrapper.upperBounds, le)
+			}, func(histogram *tempHistogram, f float64) {
+				histogram.bucketCounts[le] = f
+			})
+		case strings.HasSuffix(mName, "_count"):
+			processClassicHistogramSeries(m, "_count", histogramMap, smpls, nil, func(histogram *tempHistogram, f float64) {
+				histogram.count = f
+			})
+		case strings.HasSuffix(mName, "_sum"):
+			processClassicHistogramSeries(m, "_sum", histogramMap, smpls, nil, func(histogram *tempHistogram, f float64) {
+				histogram.sum = f
+			})
+		}
+	}
+
+	// Convert the collated classic histogram data into native histograms
+	// with custom bounds and append them to the storage.
+	for _, histogramWrapper := range histogramMap {
+		upperBounds, fhBase := processUpperBoundsAndCreateBaseHistogram(histogramWrapper.upperBounds)
+		samples := make([]promql.Sample, 0, len(histogramWrapper.histogramByTs))
+		for t, histogram := range histogramWrapper.histogramByTs {
+			fh := fhBase.Copy()
+			var prevCount, total float64
+			for i, le := range upperBounds {
+				currCount, exists := histogram.bucketCounts[le]
+				if !exists {
+					currCount = 0
+				}
+				count := currCount - prevCount
+				fh.PositiveBuckets[i] = count
+				total += count
+				prevCount = currCount
+			}
+			fh.Sum = histogram.sum
+			if histogram.count != 0 {
+				total = histogram.count
+			}
+			fh.Count = total
+			s := promql.Sample{T: t, H: fh.Compact(0)}
+			if err := s.H.Validate(); err != nil {
+				return err
+			}
+			samples = append(samples, s)
+		}
+		sort.Slice(samples, func(i, j int) bool { return samples[i].T < samples[j].T })
+		for _, s := range samples {
+			if err := appendSample(a, s, histogramWrapper.metric); err != nil {
+				return err
+			}
+		}
+	}
 	return nil
 }
 
@@ -471,7 +641,7 @@ type evalCmd struct {
 	line  int
 
 	isRange             bool // if false, instant query
-	fail, ordered       bool
+	fail, warn, ordered bool
 	expectedFailMessage string
 	expectedFailRegexp  *regexp.Regexp
 
@@ -828,6 +998,13 @@ func (t *test) execRangeEval(cmd *evalCmd, engine promql.QueryEngine) error {
 		return fmt.Errorf("error creating range query for %q (line %d): %w", cmd.expr, cmd.line, err)
 	}
 	res := q.Exec(t.context)
+	countWarnings, _ := res.Warnings.CountWarningsAndInfo()
+	if !cmd.warn && countWarnings > 0 {
+		return fmt.Errorf("unexpected warnings evaluating query %q (line %d): %v", cmd.expr, cmd.line, res.Warnings)
+	}
+	if cmd.warn && countWarnings == 0 {
+		return fmt.Errorf("expected warnings evaluating query %q (line %d) but got none", cmd.expr, cmd.line)
+	}
 	if res.Err != nil {
 		if cmd.fail {
 			return cmd.checkExpectedFailure(res.Err)
@@ -854,76 +1031,89 @@ func (t *test) execInstantEval(cmd *evalCmd, engine promql.QueryEngine) error {
 	}
 	queries = append([]atModifierTestCase{{expr: cmd.expr, evalTime: cmd.start}}, queries...)
 	for _, iq := range queries {
-		q, err := engine.NewInstantQuery(t.context, t.storage, nil, iq.expr, iq.evalTime)
-		if err != nil {
-			return fmt.Errorf("error creating instant query for %q (line %d): %w", cmd.expr, cmd.line, err)
-		}
-		defer q.Close()
-		res := q.Exec(t.context)
-		if res.Err != nil {
-			if cmd.fail {
-				if err := cmd.checkExpectedFailure(res.Err); err != nil {
-					return err
-				}
-
-				continue
-			}
-			return fmt.Errorf("error evaluating query %q (line %d): %w", iq.expr, cmd.line, res.Err)
-		}
-		if res.Err == nil && cmd.fail {
-			return fmt.Errorf("expected error evaluating query %q (line %d) but got none", iq.expr, cmd.line)
-		}
-		err = cmd.compareResult(res.Value)
-		if err != nil {
-			return fmt.Errorf("error in %s %s (line %d): %w", cmd, iq.expr, cmd.line, err)
-		}
-
-		// Check query returns same result in range mode,
-		// by checking against the middle step.
-		q, err = engine.NewRangeQuery(t.context, t.storage, nil, iq.expr, iq.evalTime.Add(-time.Minute), iq.evalTime.Add(time.Minute), time.Minute)
-		if err != nil {
-			return fmt.Errorf("error creating range query for %q (line %d): %w", cmd.expr, cmd.line, err)
-		}
-		rangeRes := q.Exec(t.context)
-		if rangeRes.Err != nil {
-			return fmt.Errorf("error evaluating query %q (line %d) in range mode: %w", iq.expr, cmd.line, rangeRes.Err)
-		}
-		defer q.Close()
-		if cmd.ordered {
-			// Range queries are always sorted by labels, so skip this test case that expects results in a particular order.
-			continue
-		}
-		mat := rangeRes.Value.(promql.Matrix)
-		if err := assertMatrixSorted(mat); err != nil {
+		if err := t.runInstantQuery(iq, cmd, engine); err != nil {
 			return err
 		}
+	}
+	return nil
+}
 
-		vec := make(promql.Vector, 0, len(mat))
-		for _, series := range mat {
-			// We expect either Floats or Histograms.
-			for _, point := range series.Floats {
-				if point.T == timeMilliseconds(iq.evalTime) {
-					vec = append(vec, promql.Sample{Metric: series.Metric, T: point.T, F: point.F})
-					break
-				}
-			}
-			for _, point := range series.Histograms {
-				if point.T == timeMilliseconds(iq.evalTime) {
-					vec = append(vec, promql.Sample{Metric: series.Metric, T: point.T, H: point.H})
-					break
-				}
+func (t *test) runInstantQuery(iq atModifierTestCase, cmd *evalCmd, engine promql.QueryEngine) error {
+	q, err := engine.NewInstantQuery(t.context, t.storage, nil, iq.expr, iq.evalTime)
+	if err != nil {
+		return fmt.Errorf("error creating instant query for %q (line %d): %w", cmd.expr, cmd.line, err)
+	}
+	defer q.Close()
+	res := q.Exec(t.context)
+	countWarnings, _ := res.Warnings.CountWarningsAndInfo()
+	if !cmd.warn && countWarnings > 0 {
+		return fmt.Errorf("unexpected warnings evaluating query %q (line %d): %v", iq.expr, cmd.line, res.Warnings)
+	}
+	if cmd.warn && countWarnings == 0 {
+		return fmt.Errorf("expected warnings evaluating query %q (line %d) but got none", iq.expr, cmd.line)
+	}
+	if res.Err != nil {
+		if cmd.fail {
+			if err := cmd.checkExpectedFailure(res.Err); err != nil {
+				return err
 			}
+
+			return nil
 		}
-		if _, ok := res.Value.(promql.Scalar); ok {
-			err = cmd.compareResult(promql.Scalar{V: vec[0].F})
-		} else {
-			err = cmd.compareResult(vec)
-		}
-		if err != nil {
-			return fmt.Errorf("error in %s %s (line %d) range mode: %w", cmd, iq.expr, cmd.line, err)
-		}
+		return fmt.Errorf("error evaluating query %q (line %d): %w", iq.expr, cmd.line, res.Err)
+	}
+	if res.Err == nil && cmd.fail {
+		return fmt.Errorf("expected error evaluating query %q (line %d) but got none", iq.expr, cmd.line)
+	}
+	err = cmd.compareResult(res.Value)
+	if err != nil {
+		return fmt.Errorf("error in %s %s (line %d): %w", cmd, iq.expr, cmd.line, err)
 	}
 
+	// Check query returns same result in range mode,
+	// by checking against the middle step.
+	q, err = engine.NewRangeQuery(t.context, t.storage, nil, iq.expr, iq.evalTime.Add(-time.Minute), iq.evalTime.Add(time.Minute), time.Minute)
+	if err != nil {
+		return fmt.Errorf("error creating range query for %q (line %d): %w", cmd.expr, cmd.line, err)
+	}
+	rangeRes := q.Exec(t.context)
+	if rangeRes.Err != nil {
+		return fmt.Errorf("error evaluating query %q (line %d) in range mode: %w", iq.expr, cmd.line, rangeRes.Err)
+	}
+	defer q.Close()
+	if cmd.ordered {
+		// Range queries are always sorted by labels, so skip this test case that expects results in a particular order.
+		return nil
+	}
+	mat := rangeRes.Value.(promql.Matrix)
+	if err := assertMatrixSorted(mat); err != nil {
+		return err
+	}
+
+	vec := make(promql.Vector, 0, len(mat))
+	for _, series := range mat {
+		// We expect either Floats or Histograms.
+		for _, point := range series.Floats {
+			if point.T == timeMilliseconds(iq.evalTime) {
+				vec = append(vec, promql.Sample{Metric: series.Metric, T: point.T, F: point.F})
+				break
+			}
+		}
+		for _, point := range series.Histograms {
+			if point.T == timeMilliseconds(iq.evalTime) {
+				vec = append(vec, promql.Sample{Metric: series.Metric, T: point.T, H: point.H})
+				break
+			}
+		}
+	}
+	if _, ok := res.Value.(promql.Scalar); ok {
+		err = cmd.compareResult(promql.Scalar{V: vec[0].F})
+	} else {
+		err = cmd.compareResult(vec)
+	}
+	if err != nil {
+		return fmt.Errorf("error in %s %s (line %d) range mode: %w", cmd, iq.expr, cmd.line, err)
+	}
 	return nil
 }
 
@@ -1014,7 +1204,7 @@ func (ll *LazyLoader) parse(input string) error {
 		if len(l) == 0 {
 			continue
 		}
-		if strings.ToLower(patSpace.Split(l, 2)[0]) == "load" {
+		if strings.HasPrefix(strings.ToLower(patSpace.Split(l, 2)[0]), "load") {
 			_, cmd, err := parseLoad(lines, i)
 			if err != nil {
 				return err

promql/promqltest/testdata/aggregators.test

@@ -399,7 +399,7 @@ eval instant at 1m quantile without(point)((scalar(foo)), data)
 	{test="three samples"} 1.6
 	{test="uneven samples"} 2.8
 
-eval instant at 1m quantile without(point)(NaN, data)
+eval_warn instant at 1m quantile without(point)(NaN, data)
     {test="two samples"} NaN
     {test="three samples"} NaN
     {test="uneven samples"} NaN

promql/promqltest/testdata/functions.test

@@ -838,17 +838,17 @@ eval instant at 1m quantile_over_time(1, data[1m])
 	{test="three samples"} 2
 	{test="uneven samples"} 4
 
-eval instant at 1m quantile_over_time(-1, data[1m])
+eval_warn instant at 1m quantile_over_time(-1, data[1m])
 	{test="two samples"} -Inf
 	{test="three samples"} -Inf
 	{test="uneven samples"} -Inf
 
-eval instant at 1m quantile_over_time(2, data[1m])
+eval_warn instant at 1m quantile_over_time(2, data[1m])
 	{test="two samples"} +Inf
 	{test="three samples"} +Inf
 	{test="uneven samples"} +Inf
 
-eval instant at 1m (quantile_over_time(2, (data[1m])))
+eval_warn instant at 1m (quantile_over_time(2, (data[1m])))
 	{test="two samples"} +Inf
 	{test="three samples"} +Inf
 	{test="uneven samples"} +Inf

promql/promqltest/testdata/histograms.test

@@ -5,7 +5,7 @@
 # server has to cope with it.
 
 # Test histogram.
-load 5m
+load_with_nhcb 5m
 	testhistogram_bucket{le="0.1", start="positive"}	0+5x10
 	testhistogram_bucket{le=".2", start="positive"}		0+7x10
 	testhistogram_bucket{le="1e0", start="positive"}	0+11x10
@@ -18,15 +18,33 @@ load 5m
 # Another test histogram, where q(1/6), q(1/2), and q(5/6) are each in
 # the middle of a bucket and should therefore be 1, 3, and 5,
 # respectively.
-load 5m
+load_with_nhcb 5m
 	testhistogram2_bucket{le="0"}	 0+0x10
 	testhistogram2_bucket{le="2"}	 0+1x10
 	testhistogram2_bucket{le="4"}    0+2x10
 	testhistogram2_bucket{le="6"}	 0+3x10
 	testhistogram2_bucket{le="+Inf"} 0+3x10
 
+# Another test histogram, this time without any observations in the +Inf bucket.
+# This enables a meaningful calculation of standard deviation and variance.
+load_with_nhcb 5m
+	testhistogram3_bucket{le="0", start="positive"}	0+0x10
+	testhistogram3_bucket{le="0.1", start="positive"}	0+5x10
+	testhistogram3_bucket{le=".2", start="positive"}		0+7x10
+	testhistogram3_bucket{le="1e0", start="positive"}	0+11x10
+	testhistogram3_bucket{le="+Inf", start="positive"}	0+11x10
+	testhistogram3_sum{start="positive"}	0+33x10
+	testhistogram3_count{start="positive"}	0+11x10
+	testhistogram3_bucket{le="-.25", start="negative"}	0+0x10
+	testhistogram3_bucket{le="-.2", start="negative"}	0+1x10
+	testhistogram3_bucket{le="-0.1", start="negative"}	0+2x10
+	testhistogram3_bucket{le="0.3", start="negative"}	0+2x10
+	testhistogram3_bucket{le="+Inf", start="negative"}	0+2x10
+	testhistogram3_sum{start="negative"}	0+8x10
+	testhistogram3_count{start="negative"}	0+2x10
+
 # Now a more realistic histogram per job and instance to test aggregation.
-load 5m
+load_with_nhcb 5m
 	request_duration_seconds_bucket{job="job1", instance="ins1", le="0.1"}	0+1x10
 	request_duration_seconds_bucket{job="job1", instance="ins1", le="0.2"}	0+3x10
 	request_duration_seconds_bucket{job="job1", instance="ins1", le="+Inf"}	0+4x10
@@ -41,7 +59,7 @@ load 5m
 	request_duration_seconds_bucket{job="job2", instance="ins2", le="+Inf"}	0+9x10
 
 # Different le representations in one histogram.
-load 5m
+load_with_nhcb 5m
 	mixed_bucket{job="job1", instance="ins1", le="0.1"}	0+1x10
 	mixed_bucket{job="job1", instance="ins1", le="0.2"}	0+1x10
 	mixed_bucket{job="job1", instance="ins1", le="2e-1"}	0+1x10
@@ -50,27 +68,81 @@ load 5m
 	mixed_bucket{job="job1", instance="ins2", le="+inf"}	0+0x10
 	mixed_bucket{job="job1", instance="ins2", le="+Inf"}	0+0x10
 
+# Test histogram_count.
+eval instant at 50m histogram_count(testhistogram3)
+	{start="positive"} 110
+	{start="negative"} 20
+
+# Test histogram_sum.
+eval instant at 50m histogram_sum(testhistogram3)
+	{start="positive"} 330
+	{start="negative"} 80
+
+# Test histogram_avg.
+eval instant at 50m histogram_avg(testhistogram3)
+	{start="positive"} 3
+	{start="negative"} 4
+
+# Test histogram_stddev.
+eval instant at 50m histogram_stddev(testhistogram3)
+	{start="positive"} 2.8189265757336734
+	{start="negative"} 4.182715937754936
+
+# Test histogram_stdvar.
+eval instant at 50m histogram_stdvar(testhistogram3)
+	{start="positive"} 7.946347039377573
+	{start="negative"} 17.495112615949154
+
+# Test histogram_fraction.
+
+eval instant at 50m histogram_fraction(0, 0.2, testhistogram3)
+	{start="positive"} 0.6363636363636364
+	{start="negative"} 0
+	
+eval instant at 50m histogram_fraction(0, 0.2, rate(testhistogram3[5m]))
+	{start="positive"} 0.6363636363636364
+	{start="negative"} 0
+
+# Test histogram_quantile.
+
+eval instant at 50m histogram_quantile(0, testhistogram3_bucket)
+	{start="positive"} 0
+	{start="negative"} -0.25
+
+eval instant at 50m histogram_quantile(0.25, testhistogram3_bucket)
+	{start="positive"} 0.055
+	{start="negative"} -0.225
+
+eval instant at 50m histogram_quantile(0.5, testhistogram3_bucket)
+	{start="positive"} 0.125
+	{start="negative"} -0.2
+
+eval instant at 50m histogram_quantile(0.75, testhistogram3_bucket)
+	{start="positive"} 0.45
+	{start="negative"} -0.15
+
+eval instant at 50m histogram_quantile(1, testhistogram3_bucket)
+	{start="positive"} 1
+	{start="negative"} -0.1
+
 # Quantile too low.
-eval instant at 50m histogram_quantile(-0.1, testhistogram_bucket)
+eval_warn instant at 50m histogram_quantile(-0.1, testhistogram_bucket)
 	{start="positive"} -Inf
 	{start="negative"} -Inf
 
 # Quantile too high.
-eval instant at 50m histogram_quantile(1.01, testhistogram_bucket)
+eval_warn instant at 50m histogram_quantile(1.01, testhistogram_bucket)
 	{start="positive"} +Inf
 	{start="negative"} +Inf
 
 # Quantile invalid.
-eval instant at 50m histogram_quantile(NaN, testhistogram_bucket)
+eval_warn instant at 50m histogram_quantile(NaN, testhistogram_bucket)
 	{start="positive"} NaN
 	{start="negative"} NaN
 
-# Quantile value in lowest bucket, which is positive.
-eval instant at 50m histogram_quantile(0, testhistogram_bucket{start="positive"})
+# Quantile value in lowest bucket.
+eval instant at 50m histogram_quantile(0, testhistogram_bucket)
 	{start="positive"} 0
-
-# Quantile value in lowest bucket, which is negative.
-eval instant at 50m histogram_quantile(0, testhistogram_bucket{start="negative"})
 	{start="negative"} -0.2
 
 # Quantile value in highest bucket.
@@ -83,7 +155,6 @@ eval instant at 50m histogram_quantile(0.2, testhistogram_bucket)
 	{start="positive"} 0.048
 	{start="negative"} -0.2
 
-
 eval instant at 50m histogram_quantile(0.5, testhistogram_bucket)
 	{start="positive"} 0.15
 	{start="negative"} -0.15
@@ -182,6 +253,9 @@ eval instant at 50m histogram_quantile(0.5, rate(request_duration_seconds_bucket
 	{instance="ins1", job="job2"} 0.1
 	{instance="ins2", job="job2"} 0.11666666666666667
 
+eval instant at 50m sum(request_duration_seconds)
+	{} {{schema:-53 count:250 custom_values:[0.1 0.2] buckets:[100 90 60]}}
+
 # A histogram with nonmonotonic bucket counts. This may happen when recording
 # rule evaluation or federation races scrape ingestion, causing some buckets
 # counts to be derived from fewer samples.
@@ -209,6 +283,10 @@ eval instant at 50m histogram_quantile(0.5, rate(mixed_bucket[5m]))
 	{instance="ins1", job="job1"} 0.15
 	{instance="ins2", job="job1"} NaN
 
+eval instant at 50m histogram_quantile(0.5, rate(mixed[5m]))
+	{instance="ins1", job="job1"} 0.2
+	{instance="ins2", job="job1"} NaN
+
 eval instant at 50m histogram_quantile(0.75, rate(mixed_bucket[5m]))
 	{instance="ins1", job="job1"} 0.2
 	{instance="ins2", job="job1"} NaN
@@ -217,7 +295,7 @@ eval instant at 50m histogram_quantile(1, rate(mixed_bucket[5m]))
 	{instance="ins1", job="job1"} 0.2
 	{instance="ins2", job="job1"} NaN
 
-load 5m
+load_with_nhcb 5m
 	empty_bucket{le="0.1", job="job1", instance="ins1"}    0x10
 	empty_bucket{le="0.2", job="job1", instance="ins1"}    0x10
 	empty_bucket{le="+Inf", job="job1", instance="ins1"}   0x10
@@ -227,9 +305,9 @@ eval instant at 50m histogram_quantile(0.2, rate(empty_bucket[5m]))
 
 # Load a duplicate histogram with a different name to test failure scenario on multiple histograms with the same label set
 # https://github.com/prometheus/prometheus/issues/9910
-load 5m
+load_with_nhcb 5m
 	request_duration_seconds2_bucket{job="job1", instance="ins1", le="0.1"}	0+1x10
 	request_duration_seconds2_bucket{job="job1", instance="ins1", le="0.2"}	0+3x10
 	request_duration_seconds2_bucket{job="job1", instance="ins1", le="+Inf"}	0+4x10
 
-eval_fail instant at 50m histogram_quantile(0.99, {__name__=~"request_duration.*"})
+eval_fail instant at 50m histogram_quantile(0.99, {__name__=~"request_duration_seconds\\d*_bucket$"})

promql/promqltest/testdata/native_histograms.test

@@ -364,7 +364,7 @@ eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_7)
 load 10m
     histogram_quantile_1 {{schema:0 count:12 sum:100 z_bucket:2 z_bucket_w:0.001 buckets:[2 3 0 1 4]}}x1
 
-eval instant at 10m histogram_quantile(1.001, histogram_quantile_1)
+eval_warn instant at 10m histogram_quantile(1.001, histogram_quantile_1)
     {} Inf
 
 eval instant at 10m histogram_quantile(1, histogram_quantile_1)
@@ -388,14 +388,14 @@ eval instant at 10m histogram_quantile(0.1, histogram_quantile_1)
 eval instant at 10m histogram_quantile(0, histogram_quantile_1)
     {} 0
 
-eval instant at 10m histogram_quantile(-1, histogram_quantile_1)
+eval_warn instant at 10m histogram_quantile(-1, histogram_quantile_1)
     {} -Inf
 
 # Apply quantile function to histogram with all negative buckets with zero bucket.
 load 10m
     histogram_quantile_2 {{schema:0 count:12 sum:100 z_bucket:2 z_bucket_w:0.001 n_buckets:[2 3 0 1 4]}}x1
 
-eval instant at 10m histogram_quantile(1.001, histogram_quantile_2)
+eval_warn instant at 10m histogram_quantile(1.001, histogram_quantile_2)
     {} Inf
 
 eval instant at 10m histogram_quantile(1, histogram_quantile_2)
@@ -416,14 +416,14 @@ eval instant at 10m histogram_quantile(0.1, histogram_quantile_2)
 eval instant at 10m histogram_quantile(0, histogram_quantile_2)
     {} -16
 
-eval instant at 10m histogram_quantile(-1, histogram_quantile_2)
+eval_warn instant at 10m histogram_quantile(-1, histogram_quantile_2)
     {} -Inf
 
 # Apply quantile function to histogram with both positive and negative buckets with zero bucket.
 load 10m
     histogram_quantile_3 {{schema:0 count:24 sum:100 z_bucket:4 z_bucket_w:0.001 buckets:[2 3 0 1 4] n_buckets:[2 3 0 1 4]}}x1
 
-eval instant at 10m histogram_quantile(1.001, histogram_quantile_3)
+eval_warn instant at 10m histogram_quantile(1.001, histogram_quantile_3)
     {} Inf
 
 eval instant at 10m histogram_quantile(1, histogram_quantile_3)
@@ -459,7 +459,7 @@ eval instant at 10m histogram_quantile(0.01, histogram_quantile_3)
 eval instant at 10m histogram_quantile(0, histogram_quantile_3)
     {} -16
 
-eval instant at 10m histogram_quantile(-1, histogram_quantile_3)
+eval_warn instant at 10m histogram_quantile(-1, histogram_quantile_3)
     {} -Inf
 
 # Apply fraction function to empty histogram.
@@ -731,3 +731,17 @@ eval instant at 10m histogram_count(increase(reset_in_bucket[15m]))
 eval instant at 10m histogram_sum(increase(reset_in_bucket[15m]))
      {} 10.5
 
+clear
+
+# Test native histograms with custom buckets.
+load 5m
+    custom_buckets_histogram {{schema:-53 sum:5 count:4 custom_values:[5 10] buckets:[1 2 1]}}x10
+
+eval instant at 5m histogram_fraction(5, 10, custom_buckets_histogram)
+    {} 0.5
+
+eval instant at 5m histogram_quantile(0.5, custom_buckets_histogram)
+    {} 7.5
+
+eval instant at 5m sum(custom_buckets_histogram)
+    {} {{schema:-53 sum:5 count:4 custom_values:[5 10] buckets:[1 2 1]}}

promql/quantile.go

@@ -206,12 +206,15 @@ func histogramQuantile(q float64, h *histogram.FloatHistogram) float64 {
 
 	for it.Next() {
 		bucket = it.At()
+		if bucket.Count == 0 {
+			continue
+		}
 		count += bucket.Count
 		if count >= rank {
 			break
 		}
 	}
-	if bucket.Lower < 0 && bucket.Upper > 0 {
+	if !h.UsesCustomBuckets() && bucket.Lower < 0 && bucket.Upper > 0 {
 		switch {
 		case len(h.NegativeBuckets) == 0 && len(h.PositiveBuckets) > 0:
 			// The result is in the zero bucket and the histogram has only
@@ -222,6 +225,17 @@ func histogramQuantile(q float64, h *histogram.FloatHistogram) float64 {
 			// negative buckets. So we consider 0 to be the upper bound.
 			bucket.Upper = 0
 		}
+	} else if h.UsesCustomBuckets() {
+		if bucket.Lower == math.Inf(-1) {
+			// first bucket, with lower bound -Inf
+			if bucket.Upper <= 0 {
+				return bucket.Upper
+			}
+			bucket.Lower = 0
+		} else if bucket.Upper == math.Inf(1) {
+			// last bucket, with upper bound +Inf
+			return bucket.Lower
+		}
 	}
 	// Due to numerical inaccuracies, we could end up with a higher count
 	// than h.Count. Thus, make sure count is never higher than h.Count.

rules/manager_test.go

@@ -1455,7 +1455,8 @@ func TestNativeHistogramsInRecordingRules(t *testing.T) {
 
 	expHist := hists[0].ToFloat(nil)
 	for _, h := range hists[1:] {
-		expHist = expHist.Add(h.ToFloat(nil))
+		expHist, err = expHist.Add(h.ToFloat(nil))
+		require.NoError(t, err)
 	}
 
 	it := s.Iterator(nil)

scrape/scrape.go

@@ -663,7 +663,7 @@ func appender(app storage.Appender, sampleLimit, bucketLimit int, maxSchema int3
 		}
 	}
 
-	if maxSchema < nativeHistogramMaxSchema {
+	if maxSchema < histogram.ExponentialSchemaMax {
 		app = &maxSchemaAppender{
 			Appender:  app,
 			maxSchema: maxSchema,
@@ -1978,10 +1978,10 @@ func pickSchema(bucketFactor float64) int32 {
 	}
 	floor := math.Floor(-math.Log2(math.Log2(bucketFactor)))
 	switch {
-	case floor >= float64(nativeHistogramMaxSchema):
-		return nativeHistogramMaxSchema
-	case floor <= float64(nativeHistogramMinSchema):
-		return nativeHistogramMinSchema
+	case floor >= float64(histogram.ExponentialSchemaMax):
+		return histogram.ExponentialSchemaMax
+	case floor <= float64(histogram.ExponentialSchemaMin):
+		return histogram.ExponentialSchemaMin
 	default:
 		return int32(floor)
 	}

scrape/scrape_test.go

@@ -511,7 +511,7 @@ func TestScrapePoolAppender(t *testing.T) {
 	appl, ok := loop.(*scrapeLoop)
 	require.True(t, ok, "Expected scrapeLoop but got %T", loop)
 
-	wrapped := appender(appl.appender(context.Background()), 0, 0, nativeHistogramMaxSchema)
+	wrapped := appender(appl.appender(context.Background()), 0, 0, histogram.ExponentialSchemaMax)
 
 	tl, ok := wrapped.(*timeLimitAppender)
 	require.True(t, ok, "Expected timeLimitAppender but got %T", wrapped)
@@ -527,7 +527,7 @@ func TestScrapePoolAppender(t *testing.T) {
 	appl, ok = loop.(*scrapeLoop)
 	require.True(t, ok, "Expected scrapeLoop but got %T", loop)
 
-	wrapped = appender(appl.appender(context.Background()), sampleLimit, 0, nativeHistogramMaxSchema)
+	wrapped = appender(appl.appender(context.Background()), sampleLimit, 0, histogram.ExponentialSchemaMax)
 
 	sl, ok := wrapped.(*limitAppender)
 	require.True(t, ok, "Expected limitAppender but got %T", wrapped)
@@ -538,7 +538,7 @@ func TestScrapePoolAppender(t *testing.T) {
 	_, ok = tl.Appender.(nopAppender)
 	require.True(t, ok, "Expected base appender but got %T", tl.Appender)
 
-	wrapped = appender(appl.appender(context.Background()), sampleLimit, 100, nativeHistogramMaxSchema)
+	wrapped = appender(appl.appender(context.Background()), sampleLimit, 100, histogram.ExponentialSchemaMax)
 
 	bl, ok := wrapped.(*bucketLimitAppender)
 	require.True(t, ok, "Expected bucketLimitAppender but got %T", wrapped)
@@ -670,7 +670,7 @@ func newBasicScrapeLoop(t testing.TB, ctx context.Context, scraper scraper, app
 		true,
 		false,
 		true,
-		0, 0, nativeHistogramMaxSchema,
+		0, 0, histogram.ExponentialSchemaMax,
 		nil,
 		interval,
 		time.Hour,
@@ -812,7 +812,7 @@ func TestScrapeLoopRun(t *testing.T) {
 		true,
 		false,
 		true,
-		0, 0, nativeHistogramMaxSchema,
+		0, 0, histogram.ExponentialSchemaMax,
 		nil,
 		time.Second,
 		time.Hour,
@@ -956,7 +956,7 @@ func TestScrapeLoopMetadata(t *testing.T) {
 		true,
 		false,
 		true,
-		0, 0, nativeHistogramMaxSchema,
+		0, 0, histogram.ExponentialSchemaMax,
 		nil,
 		0,
 		0,

scrape/target.go

@@ -365,16 +365,26 @@ type bucketLimitAppender struct {
 
 func (app *bucketLimitAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) {
 	if h != nil {
+		// Return with an early error if the histogram has too many buckets and the
+		// schema is not exponential, in which case we can't reduce the resolution.
+		if len(h.PositiveBuckets)+len(h.NegativeBuckets) > app.limit && !histogram.IsExponentialSchema(h.Schema) {
+			return 0, errBucketLimit
+		}
 		for len(h.PositiveBuckets)+len(h.NegativeBuckets) > app.limit {
-			if h.Schema == -4 {
+			if h.Schema <= histogram.ExponentialSchemaMin {
 				return 0, errBucketLimit
 			}
 			h = h.ReduceResolution(h.Schema - 1)
 		}
 	}
 	if fh != nil {
+		// Return with an early error if the histogram has too many buckets and the
+		// schema is not exponential, in which case we can't reduce the resolution.
+		if len(fh.PositiveBuckets)+len(fh.NegativeBuckets) > app.limit && !histogram.IsExponentialSchema(fh.Schema) {
+			return 0, errBucketLimit
+		}
 		for len(fh.PositiveBuckets)+len(fh.NegativeBuckets) > app.limit {
-			if fh.Schema == -4 {
+			if fh.Schema <= histogram.ExponentialSchemaMin {
 				return 0, errBucketLimit
 			}
 			fh = fh.ReduceResolution(fh.Schema - 1)
@@ -387,11 +397,6 @@ func (app *bucketLimitAppender) AppendHistogram(ref storage.SeriesRef, lset labe
 	return ref, nil
 }
 
-const (
-	nativeHistogramMaxSchema int32 = 8
-	nativeHistogramMinSchema int32 = -4
-)
-
 type maxSchemaAppender struct {
 	storage.Appender
 
@@ -400,12 +405,12 @@ type maxSchemaAppender struct {
 
 func (app *maxSchemaAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) {
 	if h != nil {
-		if h.Schema > app.maxSchema {
+		if histogram.IsExponentialSchema(h.Schema) && h.Schema > app.maxSchema {
 			h = h.ReduceResolution(app.maxSchema)
 		}
 	}
 	if fh != nil {
-		if fh.Schema > app.maxSchema {
+		if histogram.IsExponentialSchema(fh.Schema) && fh.Schema > app.maxSchema {
 			fh = fh.ReduceResolution(app.maxSchema)
 		}
 	}

scrape/target_test.go

@@ -474,6 +474,17 @@ func TestBucketLimitAppender(t *testing.T) {
 		PositiveBuckets: []int64{1, 0}, // 1, 1
 	}
 
+	customBuckets := histogram.Histogram{
+		Schema: histogram.CustomBucketsSchema,
+		Count:  9,
+		Sum:    33,
+		PositiveSpans: []histogram.Span{
+			{Offset: 0, Length: 3},
+		},
+		PositiveBuckets: []int64{3, 0, 0},
+		CustomValues:    []float64{1, 2, 3},
+	}
+
 	cases := []struct {
 		h                 histogram.Histogram
 		limit             int
@@ -507,6 +518,18 @@ func TestBucketLimitAppender(t *testing.T) {
 			expectBucketCount: 1,
 			expectSchema:      -2,
 		},
+		{
+			h:           customBuckets,
+			limit:       2,
+			expectError: true,
+		},
+		{
+			h:                 customBuckets,
+			limit:             3,
+			expectError:       false,
+			expectBucketCount: 3,
+			expectSchema:      histogram.CustomBucketsSchema,
+		},
 	}
 
 	resApp := &collectResultAppender{}
@@ -562,6 +585,17 @@ func TestMaxSchemaAppender(t *testing.T) {
 		NegativeBuckets: []int64{3, 0, 0},
 	}
 
+	customBuckets := histogram.Histogram{
+		Schema: histogram.CustomBucketsSchema,
+		Count:  9,
+		Sum:    33,
+		PositiveSpans: []histogram.Span{
+			{Offset: 0, Length: 3},
+		},
+		PositiveBuckets: []int64{3, 0, 0},
+		CustomValues:    []float64{1, 2, 3},
+	}
+
 	cases := []struct {
 		h            histogram.Histogram
 		maxSchema    int32
@@ -577,6 +611,11 @@ func TestMaxSchemaAppender(t *testing.T) {
 			maxSchema:    0,
 			expectSchema: 0,
 		},
+		{
+			h:            customBuckets,
+			maxSchema:    -1,
+			expectSchema: histogram.CustomBucketsSchema,
+		},
 	}
 
 	resApp := &collectResultAppender{}

tsdb/chunkenc/float_histogram.go

@@ -76,6 +76,7 @@ func (c *FloatHistogramChunk) NumSamples() int {
 func (c *FloatHistogramChunk) Layout() (
 	schema int32, zeroThreshold float64,
 	negativeSpans, positiveSpans []histogram.Span,
+	customValues []float64,
 	err error,
 ) {
 	if c.NumSamples() == 0 {
@@ -133,17 +134,18 @@ func (c *FloatHistogramChunk) Appender() (Appender, error) {
 	a := &FloatHistogramAppender{
 		b: &c.b,
 
-		schema:     it.schema,
-		zThreshold: it.zThreshold,
-		pSpans:     it.pSpans,
-		nSpans:     it.nSpans,
-		t:          it.t,
-		tDelta:     it.tDelta,
-		cnt:        it.cnt,
-		zCnt:       it.zCnt,
-		pBuckets:   pBuckets,
-		nBuckets:   nBuckets,
-		sum:        it.sum,
+		schema:       it.schema,
+		zThreshold:   it.zThreshold,
+		pSpans:       it.pSpans,
+		nSpans:       it.nSpans,
+		customValues: it.customValues,
+		t:            it.t,
+		tDelta:       it.tDelta,
+		cnt:          it.cnt,
+		zCnt:         it.zCnt,
+		pBuckets:     pBuckets,
+		nBuckets:     nBuckets,
+		sum:          it.sum,
 	}
 	if it.numTotal == 0 {
 		a.sum.leading = 0xff
@@ -191,6 +193,7 @@ type FloatHistogramAppender struct {
 	schema         int32
 	zThreshold     float64
 	pSpans, nSpans []histogram.Span
+	customValues   []float64
 
 	t, tDelta          int64
 	sum, cnt, zCnt     xorValue
@@ -222,6 +225,7 @@ func (a *FloatHistogramAppender) Append(int64, float64) {
 //
 // The chunk is not appendable in the following cases:
 //   - The schema has changed.
+//   - The custom bounds have changed if the current schema is custom buckets.
 //   - The threshold for the zero bucket has changed.
 //   - Any buckets have disappeared.
 //   - There was a counter reset in the count of observations or in any bucket, including the zero bucket.
@@ -263,6 +267,11 @@ func (a *FloatHistogramAppender) appendable(h *histogram.FloatHistogram) (
 		return
 	}
 
+	if histogram.IsCustomBucketsSchema(h.Schema) && !histogram.FloatBucketsMatch(h.CustomValues, a.customValues) {
+		counterReset = true
+		return
+	}
+
 	if h.ZeroCount < a.zCnt.value {
 		// There has been a counter reset since ZeroThreshold didn't change.
 		counterReset = true
@@ -303,6 +312,7 @@ func (a *FloatHistogramAppender) appendable(h *histogram.FloatHistogram) (
 //
 // The chunk is not appendable in the following cases:
 //   - The schema has changed.
+//   - The custom bounds have changed if the current schema is custom buckets.
 //   - The threshold for the zero bucket has changed.
 //   - The last sample in the chunk was stale while the current sample is not stale.
 func (a *FloatHistogramAppender) appendableGauge(h *histogram.FloatHistogram) (
@@ -329,6 +339,10 @@ func (a *FloatHistogramAppender) appendableGauge(h *histogram.FloatHistogram) (
 		return
 	}
 
+	if histogram.IsCustomBucketsSchema(h.Schema) && !histogram.FloatBucketsMatch(h.CustomValues, a.customValues) {
+		return
+	}
+
 	positiveInserts, backwardPositiveInserts, positiveSpans = expandSpansBothWays(a.pSpans, h.PositiveSpans)
 	negativeInserts, backwardNegativeInserts, negativeSpans = expandSpansBothWays(a.nSpans, h.NegativeSpans)
 	okToAppend = true
@@ -422,7 +436,7 @@ func (a *FloatHistogramAppender) appendFloatHistogram(t int64, h *histogram.Floa
 	if num == 0 {
 		// The first append gets the privilege to dictate the layout
 		// but it's also responsible for encoding it into the chunk!
-		writeHistogramChunkLayout(a.b, h.Schema, h.ZeroThreshold, h.PositiveSpans, h.NegativeSpans)
+		writeHistogramChunkLayout(a.b, h.Schema, h.ZeroThreshold, h.PositiveSpans, h.NegativeSpans, h.CustomValues)
 		a.schema = h.Schema
 		a.zThreshold = h.ZeroThreshold
 
@@ -438,6 +452,12 @@ func (a *FloatHistogramAppender) appendFloatHistogram(t int64, h *histogram.Floa
 		} else {
 			a.nSpans = nil
 		}
+		if len(h.CustomValues) > 0 {
+			a.customValues = make([]float64, len(h.CustomValues))
+			copy(a.customValues, h.CustomValues)
+		} else {
+			a.customValues = nil
+		}
 
 		numPBuckets, numNBuckets := countSpans(h.PositiveSpans), countSpans(h.NegativeSpans)
 		if numPBuckets > 0 {
@@ -693,6 +713,7 @@ type floatHistogramIterator struct {
 	schema         int32
 	zThreshold     float64
 	pSpans, nSpans []histogram.Span
+	customValues   []float64
 
 	// For the fields that are tracked as deltas and ultimately dod's.
 	t      int64
@@ -753,6 +774,7 @@ func (it *floatHistogramIterator) AtFloatHistogram(fh *histogram.FloatHistogram)
 			NegativeSpans:    it.nSpans,
 			PositiveBuckets:  it.pBuckets,
 			NegativeBuckets:  it.nBuckets,
+			CustomValues:     it.customValues,
 		}
 	}
 
@@ -775,6 +797,9 @@ func (it *floatHistogramIterator) AtFloatHistogram(fh *histogram.FloatHistogram)
 	fh.NegativeBuckets = resize(fh.NegativeBuckets, len(it.nBuckets))
 	copy(fh.NegativeBuckets, it.nBuckets)
 
+	fh.CustomValues = resize(fh.CustomValues, len(it.customValues))
+	copy(fh.CustomValues, it.customValues)
+
 	return it.t, fh
 }
 
@@ -819,7 +844,7 @@ func (it *floatHistogramIterator) Next() ValueType {
 		// The first read is responsible for reading the chunk layout
 		// and for initializing fields that depend on it. We give
 		// counter reset info at chunk level, hence we discard it here.
-		schema, zeroThreshold, posSpans, negSpans, err := readHistogramChunkLayout(&it.br)
+		schema, zeroThreshold, posSpans, negSpans, customValues, err := readHistogramChunkLayout(&it.br)
 		if err != nil {
 			it.err = err
 			return ValNone
@@ -827,6 +852,7 @@ func (it *floatHistogramIterator) Next() ValueType {
 		it.schema = schema
 		it.zThreshold = zeroThreshold
 		it.pSpans, it.nSpans = posSpans, negSpans
+		it.customValues = customValues
 		numPBuckets, numNBuckets := countSpans(posSpans), countSpans(negSpans)
 		// Allocate bucket slices as needed, recycling existing slices
 		// in case this iterator was reset and already has slices of a

tsdb/chunkenc/float_histogram_test.go

@@ -280,7 +280,38 @@ func TestFloatHistogramChunkBucketChanges(t *testing.T) {
 }
 
 func TestFloatHistogramChunkAppendable(t *testing.T) {
-	setup := func() (Chunk, *FloatHistogramAppender, int64, *histogram.FloatHistogram) {
+	eh := &histogram.FloatHistogram{
+		Count:         5,
+		ZeroCount:     2,
+		Sum:           18.4,
+		ZeroThreshold: 1e-125,
+		Schema:        1,
+		PositiveSpans: []histogram.Span{
+			{Offset: 0, Length: 2},
+			{Offset: 2, Length: 1},
+			{Offset: 3, Length: 2},
+			{Offset: 3, Length: 1},
+			{Offset: 1, Length: 1},
+		},
+		PositiveBuckets: []float64{6, 3, 3, 2, 4, 5, 1},
+	}
+
+	cbh := &histogram.FloatHistogram{
+		Count:  24,
+		Sum:    18.4,
+		Schema: histogram.CustomBucketsSchema,
+		PositiveSpans: []histogram.Span{
+			{Offset: 0, Length: 2},
+			{Offset: 2, Length: 1},
+			{Offset: 3, Length: 2},
+			{Offset: 3, Length: 1},
+			{Offset: 1, Length: 1},
+		},
+		PositiveBuckets: []float64{6, 3, 3, 2, 4, 5, 1},
+		CustomValues:    []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20},
+	}
+
+	setup := func(h *histogram.FloatHistogram) (Chunk, *FloatHistogramAppender, int64, *histogram.FloatHistogram) {
 		c := Chunk(NewFloatHistogramChunk())
 
 		// Create fresh appender and add the first histogram.
@@ -289,32 +320,17 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
 		require.Equal(t, 0, c.NumSamples())
 
 		ts := int64(1234567890)
-		h1 := &histogram.FloatHistogram{
-			Count:         5,
-			ZeroCount:     2,
-			Sum:           18.4,
-			ZeroThreshold: 1e-125,
-			Schema:        1,
-			PositiveSpans: []histogram.Span{
-				{Offset: 0, Length: 2},
-				{Offset: 2, Length: 1},
-				{Offset: 3, Length: 2},
-				{Offset: 3, Length: 1},
-				{Offset: 1, Length: 1},
-			},
-			PositiveBuckets: []float64{6, 3, 3, 2, 4, 5, 1},
-		}
 
-		chk, _, app, err := app.AppendFloatHistogram(nil, ts, h1.Copy(), false)
+		chk, _, app, err := app.AppendFloatHistogram(nil, ts, h.Copy(), false)
 		require.NoError(t, err)
 		require.Nil(t, chk)
 		require.Equal(t, 1, c.NumSamples())
 		require.Equal(t, UnknownCounterReset, c.(*FloatHistogramChunk).GetCounterResetHeader())
-		return c, app.(*FloatHistogramAppender), ts, h1
+		return c, app.(*FloatHistogramAppender), ts, h
 	}
 
 	{ // Schema change.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.Schema++
 		_, _, ok, _ := hApp.appendable(h2)
@@ -324,7 +340,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
 	}
 
 	{ // Zero threshold change.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.ZeroThreshold += 0.1
 		_, _, ok, _ := hApp.appendable(h2)
@@ -334,7 +350,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
 	}
 
 	{ // New histogram that has more buckets.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.PositiveSpans = []histogram.Span{
 			{Offset: 0, Length: 3},
@@ -357,7 +373,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
 	}
 
 	{ // New histogram that has a bucket missing.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 
 		h2 := h1.Copy()
 		h2.PositiveSpans = []histogram.Span{
@@ -379,7 +395,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
 	}
 
 	{ // New histogram that has a counter reset while buckets are same.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.Sum = 23
 		h2.PositiveBuckets = []float64{6, 2, 3, 2, 4, 5, 1}
@@ -394,7 +410,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
 	}
 
 	{ // New histogram that has a counter reset while new buckets were added.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.PositiveSpans = []histogram.Span{
 			{Offset: 0, Length: 3},
@@ -415,7 +431,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
 	}
 
 	{
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		// New histogram that has a counter reset while new buckets were
 		// added before the first bucket and reset on first bucket.  (to
 		// catch the edge case where the new bucket should be forwarded
@@ -442,7 +458,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
 	}
 
 	{ // New histogram that has an explicit counter reset.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.CounterResetHint = histogram.CounterReset
 
@@ -450,7 +466,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
 	}
 
 	{ // Start new chunk explicitly, and append a new histogram that is considered appendable to the previous chunk.
-		_, hApp, ts, h1 := setup()
+		_, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy() // Identity is appendable.
 
 		nextChunk := NewFloatHistogramChunk()
@@ -466,7 +482,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
 	}
 
 	{ // Start new chunk explicitly, and append a new histogram that is not considered appendable to the previous chunk.
-		_, hApp, ts, h1 := setup()
+		_, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.Count-- // Make this not appendable due to counter reset.
 
@@ -483,7 +499,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
 	}
 
 	{ // Start new chunk explicitly, and append a new histogram that would need recoding if we added it to the chunk.
-		_, hApp, ts, h1 := setup()
+		_, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.PositiveSpans = []histogram.Span{
 			{Offset: 0, Length: 3},
@@ -507,6 +523,72 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
 		assertSampleCount(t, nextChunk, 1, ValFloatHistogram)
 		require.Equal(t, NotCounterReset, nextChunk.GetCounterResetHeader())
 	}
+
+	{ // Custom buckets, no change.
+		c, hApp, ts, h1 := setup(cbh)
+		h2 := h1.Copy()
+		_, _, ok, _ := hApp.appendable(h2)
+		require.True(t, ok)
+
+		assertNoNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
+	}
+
+	{ // Custom buckets, increase in bucket counts but no change in layout.
+		c, hApp, ts, h1 := setup(cbh)
+		h2 := h1.Copy()
+		h2.Count++
+		h2.PositiveBuckets = []float64{6, 3, 3, 2, 4, 5, 2}
+		_, _, ok, _ := hApp.appendable(h2)
+		require.True(t, ok)
+
+		assertNoNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
+	}
+
+	{ // Custom buckets, decrease in bucket counts but no change in layout.
+		c, hApp, ts, h1 := setup(cbh)
+		h2 := h1.Copy()
+		h2.Count--
+		h2.PositiveBuckets = []float64{6, 3, 3, 2, 4, 5, 0}
+		_, _, ok, _ := hApp.appendable(h2)
+		require.False(t, ok)
+
+		assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
+	}
+
+	{ // Custom buckets, change only in custom bounds.
+		c, hApp, ts, h1 := setup(cbh)
+		h2 := h1.Copy()
+		h2.CustomValues = []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21}
+		_, _, ok, _ := hApp.appendable(h2)
+		require.False(t, ok)
+
+		assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
+	}
+
+	{ // Custom buckets, with more buckets.
+		c, hApp, ts, h1 := setup(cbh)
+		h2 := h1.Copy()
+		h2.PositiveSpans = []histogram.Span{
+			{Offset: 0, Length: 3},
+			{Offset: 1, Length: 1},
+			{Offset: 1, Length: 4},
+			{Offset: 3, Length: 3},
+		}
+		h2.Count += 6
+		h2.Sum = 30
+		// Existing histogram should get values converted from the above to:
+		//   6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between)
+		// so the new histogram should have new counts >= these per-bucket counts, e.g.:
+		h2.PositiveBuckets = []float64{7, 5, 1, 3, 1, 0, 2, 5, 5, 0, 1} // (total 30)
+
+		posInterjections, negInterjections, ok, cr := hApp.appendable(h2)
+		require.NotEmpty(t, posInterjections)
+		require.Empty(t, negInterjections)
+		require.True(t, ok) // Only new buckets came in.
+		require.False(t, cr)
+
+		assertRecodedFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
+	}
 }
 
 func assertNewFloatHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *FloatHistogramAppender, ts int64, h *histogram.FloatHistogram, expectHeader CounterResetHeader) {
@@ -526,7 +608,7 @@ func assertNewFloatHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *Fl
 func assertNoNewFloatHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *FloatHistogramAppender, ts int64, h *histogram.FloatHistogram, expectHeader CounterResetHeader) {
 	oldChunkBytes := oldChunk.Bytes()
 	newChunk, recoded, newAppender, err := hApp.AppendFloatHistogram(nil, ts, h, false)
-	require.NotEqual(t, oldChunkBytes, oldChunk.Bytes()) // Sanity check that previous chunk is untouched.
+	require.Greater(t, len(oldChunk.Bytes()), len(oldChunkBytes)) // Check that current chunk is bigger than previously.
 	require.NoError(t, err)
 	require.Nil(t, newChunk)
 	require.False(t, recoded)
@@ -715,6 +797,32 @@ func TestFloatHistogramChunkAppendableWithEmptySpan(t *testing.T) {
 				NegativeBuckets: []float64{1, 4, 2, 7, 5, 5, 2},
 			},
 		},
+		"empty span in old and new custom buckets histogram": {
+			h1: &histogram.FloatHistogram{
+				Schema: histogram.CustomBucketsSchema,
+				Count:  7,
+				Sum:    1234.5,
+				PositiveSpans: []histogram.Span{
+					{Offset: 0, Length: 4},
+					{Offset: 0, Length: 0},
+					{Offset: 0, Length: 3},
+				},
+				PositiveBuckets: []float64{1, 2, 1, 1, 1, 1, 1},
+				CustomValues:    []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
+			},
+			h2: &histogram.FloatHistogram{
+				Schema: histogram.CustomBucketsSchema,
+				Count:  10,
+				Sum:    2345.6,
+				PositiveSpans: []histogram.Span{
+					{Offset: 0, Length: 4},
+					{Offset: 0, Length: 0},
+					{Offset: 0, Length: 3},
+				},
+				PositiveBuckets: []float64{1, 3, 1, 2, 1, 1, 1},
+				CustomValues:    []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
+			},
+		},
 	}
 
 	for name, tc := range tests {
@@ -741,7 +849,40 @@ func TestFloatHistogramChunkAppendableWithEmptySpan(t *testing.T) {
 }
 
 func TestFloatHistogramChunkAppendableGauge(t *testing.T) {
-	setup := func() (Chunk, *FloatHistogramAppender, int64, *histogram.FloatHistogram) {
+	eh := &histogram.FloatHistogram{
+		CounterResetHint: histogram.GaugeType,
+		Count:            5,
+		ZeroCount:        2,
+		Sum:              18.4,
+		ZeroThreshold:    1e-125,
+		Schema:           1,
+		PositiveSpans: []histogram.Span{
+			{Offset: 0, Length: 2},
+			{Offset: 2, Length: 1},
+			{Offset: 3, Length: 2},
+			{Offset: 3, Length: 1},
+			{Offset: 1, Length: 1},
+		},
+		PositiveBuckets: []float64{6, 3, 3, 2, 4, 5, 1},
+	}
+
+	cbh := &histogram.FloatHistogram{
+		CounterResetHint: histogram.GaugeType,
+		Count:            24,
+		Sum:              18.4,
+		Schema:           histogram.CustomBucketsSchema,
+		PositiveSpans: []histogram.Span{
+			{Offset: 0, Length: 2},
+			{Offset: 2, Length: 1},
+			{Offset: 3, Length: 2},
+			{Offset: 3, Length: 1},
+			{Offset: 1, Length: 1},
+		},
+		PositiveBuckets: []float64{6, 3, 3, 2, 4, 5, 1},
+		CustomValues:    []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20},
+	}
+
+	setup := func(h *histogram.FloatHistogram) (Chunk, *FloatHistogramAppender, int64, *histogram.FloatHistogram) {
 		c := Chunk(NewFloatHistogramChunk())
 
 		// Create fresh appender and add the first histogram.
@@ -750,33 +891,17 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) {
 		require.Equal(t, 0, c.NumSamples())
 
 		ts := int64(1234567890)
-		h1 := &histogram.FloatHistogram{
-			CounterResetHint: histogram.GaugeType,
-			Count:            5,
-			ZeroCount:        2,
-			Sum:              18.4,
-			ZeroThreshold:    1e-125,
-			Schema:           1,
-			PositiveSpans: []histogram.Span{
-				{Offset: 0, Length: 2},
-				{Offset: 2, Length: 1},
-				{Offset: 3, Length: 2},
-				{Offset: 3, Length: 1},
-				{Offset: 1, Length: 1},
-			},
-			PositiveBuckets: []float64{6, 3, 3, 2, 4, 5, 1},
-		}
 
-		chk, _, app, err := app.AppendFloatHistogram(nil, ts, h1.Copy(), false)
+		chk, _, app, err := app.AppendFloatHistogram(nil, ts, h.Copy(), false)
 		require.NoError(t, err)
 		require.Nil(t, chk)
 		require.Equal(t, 1, c.NumSamples())
 		require.Equal(t, GaugeType, c.(*FloatHistogramChunk).GetCounterResetHeader())
-		return c, app.(*FloatHistogramAppender), ts, h1
+		return c, app.(*FloatHistogramAppender), ts, h
 	}
 
 	{ // Schema change.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.Schema++
 		_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
@@ -786,7 +911,7 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) {
 	}
 
 	{ // Zero threshold change.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.ZeroThreshold += 0.1
 		_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
@@ -796,7 +921,7 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) {
 	}
 
 	{ // New histogram that has more buckets.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.PositiveSpans = []histogram.Span{
 			{Offset: 0, Length: 3},
@@ -820,7 +945,7 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) {
 	}
 
 	{ // New histogram that has buckets missing.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.PositiveSpans = []histogram.Span{
 			{Offset: 0, Length: 2},
@@ -844,7 +969,7 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) {
 	}
 
 	{ // New histogram that has a bucket missing and new buckets.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.PositiveSpans = []histogram.Span{
 			{Offset: 0, Length: 2},
@@ -866,7 +991,7 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) {
 	}
 
 	{ // New histogram that has a counter reset while buckets are same.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.Sum = 23
 		h2.PositiveBuckets = []float64{6, 2, 3, 2, 4, 5, 1}
@@ -882,7 +1007,7 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) {
 	}
 
 	{ // New histogram that has a counter reset while new buckets were added.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.PositiveSpans = []histogram.Span{
 			{Offset: 0, Length: 3},
@@ -906,7 +1031,7 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) {
 	{
 		// New histogram that has a counter reset while new buckets were
 		// added before the first bucket and reset on first bucket.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.PositiveSpans = []histogram.Span{
 			{Offset: -3, Length: 2},
@@ -928,6 +1053,73 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) {
 
 		assertRecodedFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
 	}
+
+	{ // Custom buckets, no change.
+		c, hApp, ts, h1 := setup(cbh)
+		h2 := h1.Copy()
+		_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
+		require.True(t, ok)
+
+		assertNoNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
+	}
+
+	{ // Custom buckets, increase in bucket counts but no change in layout.
+		c, hApp, ts, h1 := setup(cbh)
+		h2 := h1.Copy()
+		h2.Count++
+		h2.PositiveBuckets = []float64{6, 3, 3, 2, 4, 5, 2}
+		_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
+		require.True(t, ok)
+
+		assertNoNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
+	}
+
+	{ // Custom buckets, decrease in bucket counts but no change in layout.
+		c, hApp, ts, h1 := setup(cbh)
+		h2 := h1.Copy()
+		h2.Count--
+		h2.PositiveBuckets = []float64{6, 3, 3, 2, 4, 5, 0}
+		_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
+		require.True(t, ok)
+
+		assertNoNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
+	}
+
+	{ // Custom buckets, change only in custom bounds.
+		c, hApp, ts, h1 := setup(cbh)
+		h2 := h1.Copy()
+		h2.CustomValues = []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21}
+		_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
+		require.False(t, ok)
+
+		assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
+	}
+
+	{ // Custom buckets, with more buckets.
+		c, hApp, ts, h1 := setup(cbh)
+		h2 := h1.Copy()
+		h2.PositiveSpans = []histogram.Span{
+			{Offset: 0, Length: 3},
+			{Offset: 1, Length: 1},
+			{Offset: 1, Length: 4},
+			{Offset: 3, Length: 3},
+		}
+		h2.Count += 6
+		h2.Sum = 30
+		// Existing histogram should get values converted from the above to:
+		//   6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between)
+		// so the new histogram should have new counts >= these per-bucket counts, e.g.:
+		h2.PositiveBuckets = []float64{7, 5, 1, 3, 1, 0, 2, 5, 5, 0, 1} // (total 30)
+
+		posInterjections, negInterjections, pBackwardI, nBackwardI, _, _, ok := hApp.appendableGauge(h2)
+		require.NotEmpty(t, posInterjections)
+		require.Empty(t, negInterjections)
+		require.Empty(t, pBackwardI)
+		require.Empty(t, nBackwardI)
+		require.True(t, ok) // Only new buckets came in.
+
+		assertRecodedFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
+	}
 }
 
 func TestFloatHistogramAppendOnlyErrors(t *testing.T) {
@@ -975,4 +1167,26 @@ func TestFloatHistogramAppendOnlyErrors(t *testing.T) {
 		require.False(t, isRecoded)
 		require.EqualError(t, err, "float histogram counter reset")
 	})
+	t.Run("counter reset error with custom buckets", func(t *testing.T) {
+		c := Chunk(NewFloatHistogramChunk())
+
+		// Create fresh appender and add the first histogram.
+		app, err := c.Appender()
+		require.NoError(t, err)
+
+		h := tsdbutil.GenerateTestCustomBucketsFloatHistogram(0)
+		var isRecoded bool
+		c, isRecoded, app, err = app.AppendFloatHistogram(nil, 1, h, true)
+		require.Nil(t, c)
+		require.False(t, isRecoded)
+		require.NoError(t, err)
+
+		// Add erroring histogram.
+		h2 := h.Copy()
+		h2.CustomValues = []float64{0, 1, 2, 3, 4, 5, 6, 7}
+		c, isRecoded, _, err = app.AppendFloatHistogram(nil, 2, h2, true)
+		require.Nil(t, c)
+		require.False(t, isRecoded)
+		require.EqualError(t, err, "float histogram counter reset")
+	})
 }

tsdb/chunkenc/histogram.go

@@ -69,6 +69,7 @@ func (c *HistogramChunk) NumSamples() int {
 func (c *HistogramChunk) Layout() (
 	schema int32, zeroThreshold float64,
 	negativeSpans, positiveSpans []histogram.Span,
+	customValues []float64,
 	err error,
 ) {
 	if c.NumSamples() == 0 {
@@ -131,6 +132,7 @@ func (c *HistogramChunk) Appender() (Appender, error) {
 		zThreshold:    it.zThreshold,
 		pSpans:        it.pSpans,
 		nSpans:        it.nSpans,
+		customValues:  it.customValues,
 		t:             it.t,
 		cnt:           it.cnt,
 		zCnt:          it.zCnt,
@@ -198,6 +200,7 @@ type HistogramAppender struct {
 	schema         int32
 	zThreshold     float64
 	pSpans, nSpans []histogram.Span
+	customValues   []float64
 
 	// Although we intend to start new chunks on counter resets, we still
 	// have to handle negative deltas for gauge histograms. Therefore, even
@@ -241,6 +244,7 @@ func (a *HistogramAppender) Append(int64, float64) {
 // The chunk is not appendable in the following cases:
 //
 //   - The schema has changed.
+//   - The custom bounds have changed if the current schema is custom buckets.
 //   - The threshold for the zero bucket has changed.
 //   - Any buckets have disappeared.
 //   - There was a counter reset in the count of observations or in any bucket,
@@ -283,6 +287,11 @@ func (a *HistogramAppender) appendable(h *histogram.Histogram) (
 		return
 	}
 
+	if histogram.IsCustomBucketsSchema(h.Schema) && !histogram.FloatBucketsMatch(h.CustomValues, a.customValues) {
+		counterReset = true
+		return
+	}
+
 	if h.ZeroCount < a.zCnt {
 		// There has been a counter reset since ZeroThreshold didn't change.
 		counterReset = true
@@ -323,6 +332,7 @@ func (a *HistogramAppender) appendable(h *histogram.Histogram) (
 //
 // The chunk is not appendable in the following cases:
 //   - The schema has changed.
+//   - The custom bounds have changed if the current schema is custom buckets.
 //   - The threshold for the zero bucket has changed.
 //   - The last sample in the chunk was stale while the current sample is not stale.
 func (a *HistogramAppender) appendableGauge(h *histogram.Histogram) (
@@ -349,6 +359,10 @@ func (a *HistogramAppender) appendableGauge(h *histogram.Histogram) (
 		return
 	}
 
+	if histogram.IsCustomBucketsSchema(h.Schema) && !histogram.FloatBucketsMatch(h.CustomValues, a.customValues) {
+		return
+	}
+
 	positiveInserts, backwardPositiveInserts, positiveSpans = expandSpansBothWays(a.pSpans, h.PositiveSpans)
 	negativeInserts, backwardNegativeInserts, negativeSpans = expandSpansBothWays(a.nSpans, h.NegativeSpans)
 	okToAppend = true
@@ -442,7 +456,7 @@ func (a *HistogramAppender) appendHistogram(t int64, h *histogram.Histogram) {
 	if num == 0 {
 		// The first append gets the privilege to dictate the layout
 		// but it's also responsible for encoding it into the chunk!
-		writeHistogramChunkLayout(a.b, h.Schema, h.ZeroThreshold, h.PositiveSpans, h.NegativeSpans)
+		writeHistogramChunkLayout(a.b, h.Schema, h.ZeroThreshold, h.PositiveSpans, h.NegativeSpans, h.CustomValues)
 		a.schema = h.Schema
 		a.zThreshold = h.ZeroThreshold
 
@@ -458,6 +472,12 @@ func (a *HistogramAppender) appendHistogram(t int64, h *histogram.Histogram) {
 		} else {
 			a.nSpans = nil
 		}
+		if len(h.CustomValues) > 0 {
+			a.customValues = make([]float64, len(h.CustomValues))
+			copy(a.customValues, h.CustomValues)
+		} else {
+			a.customValues = nil
+		}
 
 		numPBuckets, numNBuckets := countSpans(h.PositiveSpans), countSpans(h.NegativeSpans)
 		if numPBuckets > 0 {
@@ -741,6 +761,7 @@ type histogramIterator struct {
 	schema         int32
 	zThreshold     float64
 	pSpans, nSpans []histogram.Span
+	customValues   []float64
 
 	// For the fields that are tracked as deltas and ultimately dod's.
 	t                            int64
@@ -797,6 +818,7 @@ func (it *histogramIterator) AtHistogram(h *histogram.Histogram) (int64, *histog
 			NegativeSpans:    it.nSpans,
 			PositiveBuckets:  it.pBuckets,
 			NegativeBuckets:  it.nBuckets,
+			CustomValues:     it.customValues,
 		}
 	}
 
@@ -819,6 +841,9 @@ func (it *histogramIterator) AtHistogram(h *histogram.Histogram) (int64, *histog
 	h.NegativeBuckets = resize(h.NegativeBuckets, len(it.nBuckets))
 	copy(h.NegativeBuckets, it.nBuckets)
 
+	h.CustomValues = resize(h.CustomValues, len(it.customValues))
+	copy(h.CustomValues, it.customValues)
+
 	return it.t, h
 }
 
@@ -839,6 +864,7 @@ func (it *histogramIterator) AtFloatHistogram(fh *histogram.FloatHistogram) (int
 			NegativeSpans:    it.nSpans,
 			PositiveBuckets:  it.pFloatBuckets,
 			NegativeBuckets:  it.nFloatBuckets,
+			CustomValues:     it.customValues,
 		}
 	}
 
@@ -869,6 +895,9 @@ func (it *histogramIterator) AtFloatHistogram(fh *histogram.FloatHistogram) (int
 		fh.NegativeBuckets[i] = currentNegative
 	}
 
+	fh.CustomValues = resize(fh.CustomValues, len(it.customValues))
+	copy(fh.CustomValues, it.customValues)
+
 	return it.t, fh
 }
 
@@ -927,7 +956,7 @@ func (it *histogramIterator) Next() ValueType {
 		// The first read is responsible for reading the chunk layout
 		// and for initializing fields that depend on it. We give
 		// counter reset info at chunk level, hence we discard it here.
-		schema, zeroThreshold, posSpans, negSpans, err := readHistogramChunkLayout(&it.br)
+		schema, zeroThreshold, posSpans, negSpans, customValues, err := readHistogramChunkLayout(&it.br)
 		if err != nil {
 			it.err = err
 			return ValNone
@@ -935,6 +964,7 @@ func (it *histogramIterator) Next() ValueType {
 		it.schema = schema
 		it.zThreshold = zeroThreshold
 		it.pSpans, it.nSpans = posSpans, negSpans
+		it.customValues = customValues
 		numPBuckets, numNBuckets := countSpans(posSpans), countSpans(negSpans)
 		// The code below recycles existing slices in case this iterator
 		// was reset and already has slices of a sufficient capacity.

tsdb/chunkenc/histogram_meta.go

@@ -21,17 +21,21 @@ import (
 
 func writeHistogramChunkLayout(
 	b *bstream, schema int32, zeroThreshold float64,
-	positiveSpans, negativeSpans []histogram.Span,
+	positiveSpans, negativeSpans []histogram.Span, customValues []float64,
 ) {
 	putZeroThreshold(b, zeroThreshold)
 	putVarbitInt(b, int64(schema))
 	putHistogramChunkLayoutSpans(b, positiveSpans)
 	putHistogramChunkLayoutSpans(b, negativeSpans)
+	if histogram.IsCustomBucketsSchema(schema) {
+		putHistogramChunkLayoutCustomBounds(b, customValues)
+	}
 }
 
 func readHistogramChunkLayout(b *bstreamReader) (
 	schema int32, zeroThreshold float64,
 	positiveSpans, negativeSpans []histogram.Span,
+	customValues []float64,
 	err error,
 ) {
 	zeroThreshold, err = readZeroThreshold(b)
@@ -55,6 +59,13 @@ func readHistogramChunkLayout(b *bstreamReader) (
 		return
 	}
 
+	if histogram.IsCustomBucketsSchema(schema) {
+		customValues, err = readHistogramChunkLayoutCustomBounds(b)
+		if err != nil {
+			return
+		}
+	}
+
 	return
 }
 
@@ -91,6 +102,30 @@ func readHistogramChunkLayoutSpans(b *bstreamReader) ([]histogram.Span, error) {
 	return spans, nil
 }
 
+func putHistogramChunkLayoutCustomBounds(b *bstream, customValues []float64) {
+	putVarbitUint(b, uint64(len(customValues)))
+	for _, bound := range customValues {
+		putCustomBound(b, bound)
+	}
+}
+
+func readHistogramChunkLayoutCustomBounds(b *bstreamReader) ([]float64, error) {
+	var customValues []float64
+	num, err := readVarbitUint(b)
+	if err != nil {
+		return nil, err
+	}
+	for i := 0; i < int(num); i++ {
+		bound, err := readCustomBound(b)
+		if err != nil {
+			return nil, err
+		}
+
+		customValues = append(customValues, bound)
+	}
+	return customValues, nil
+}
+
 // putZeroThreshold writes the zero threshold to the bstream. It stores typical
 // values in just one byte, but needs 9 bytes for other values. In detail:
 //   - If the threshold is 0, store a single zero byte.
@@ -139,6 +174,59 @@ func readZeroThreshold(br *bstreamReader) (float64, error) {
 	}
 }
 
+// isWholeWhenMultiplied checks to see if the number when multiplied by 1000 can
+// be converted into an integer without losing precision.
+func isWholeWhenMultiplied(in float64) bool {
+	i := uint(math.Round(in * 1000))
+	out := float64(i) / 1000
+	return in == out
+}
+
+// putCustomBound writes a custom bound to the bstream. It stores values from
+// 0 to 33554.430 (inclusive) that are multiples of 0.001 in unsigned varbit
+// encoding of up to 4 bytes, but needs 1 bit + 8 bytes for other values like
+// negative numbers, numbers greater than 33554.430, or numbers that are not
+// a multiple of 0.001, on the assumption that they are less common. In detail:
+//   - Multiply the bound by 1000, without rounding.
+//   - If the multiplied bound is >= 0, <= 33554430 and a whole number,
+//     add 1 and store it in unsigned varbit encoding. All these numbers are
+//     greater than 0, so the leading bit of the varbit is always 1!
+//   - Otherwise, store a 0 bit, followed by the 8 bytes of the original
+//     bound as a float64.
+//
+// When reading the values, we can first decode a value as unsigned varbit,
+// if it's 0, then we read the next 8 bytes as a float64, otherwise
+// we can convert the value to a float64 by subtracting 1 and dividing by 1000.
+func putCustomBound(b *bstream, f float64) {
+	tf := f * 1000
+	// 33554431-1 comes from the maximum that can be stored in a varbit in 4
+	// bytes, other values are stored in 8 bytes anyway.
+	if tf < 0 || tf > 33554430 || !isWholeWhenMultiplied(f) {
+		b.writeBit(zero)
+		b.writeBits(math.Float64bits(f), 64)
+		return
+	}
+	putVarbitUint(b, uint64(math.Round(tf))+1)
+}
+
+// readCustomBound reads the custom bound written with putCustomBound.
+func readCustomBound(br *bstreamReader) (float64, error) {
+	b, err := readVarbitUint(br)
+	if err != nil {
+		return 0, err
+	}
+	switch b {
+	case 0:
+		v, err := br.readBits(64)
+		if err != nil {
+			return 0, err
+		}
+		return math.Float64frombits(v), nil
+	default:
+		return float64(b-1) / 1000, nil
+	}
+}
+
 type bucketIterator struct {
 	spans  []histogram.Span
 	span   int // Span position of last yielded bucket.

tsdb/chunkenc/histogram_meta_test.go

@@ -373,6 +373,7 @@ func TestWriteReadHistogramChunkLayout(t *testing.T) {
 		schema                       int32
 		zeroThreshold                float64
 		positiveSpans, negativeSpans []histogram.Span
+		customValues                 []float64
 	}{
 		{
 			schema:        3,
@@ -422,23 +423,48 @@ func TestWriteReadHistogramChunkLayout(t *testing.T) {
 			positiveSpans: nil,
 			negativeSpans: nil,
 		},
+		{
+			schema:        histogram.CustomBucketsSchema,
+			positiveSpans: []histogram.Span{{Offset: -4, Length: 3}, {Offset: 2, Length: 42}},
+			negativeSpans: nil,
+			customValues:  []float64{-5, -2.5, 0, 0.1, 0.25, 0.5, 1, 2, 5, 10, 25, 50, 100, 255, 500, 1000, 50000, 1e7},
+		},
+		{
+			schema:        histogram.CustomBucketsSchema,
+			positiveSpans: []histogram.Span{{Offset: -4, Length: 3}, {Offset: 2, Length: 42}},
+			negativeSpans: nil,
+			customValues:  []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 25.0, 50.0, 100.0},
+		},
+		{
+			schema:        histogram.CustomBucketsSchema,
+			positiveSpans: []histogram.Span{{Offset: -4, Length: 3}, {Offset: 2, Length: 42}},
+			negativeSpans: nil,
+			customValues:  []float64{0.001, 0.002, 0.004, 0.008, 0.016, 0.032, 0.064, 0.128, 0.256, 0.512, 1.024, 2.048, 4.096, 8.192},
+		},
+		{
+			schema:        histogram.CustomBucketsSchema,
+			positiveSpans: []histogram.Span{{Offset: -4, Length: 3}, {Offset: 2, Length: 42}},
+			negativeSpans: nil,
+			customValues:  []float64{1.001, 1.023, 2.01, 4.007, 4.095, 8.001, 8.19, 16.24},
+		},
 	}
 
 	bs := bstream{}
 
 	for _, l := range layouts {
-		writeHistogramChunkLayout(&bs, l.schema, l.zeroThreshold, l.positiveSpans, l.negativeSpans)
+		writeHistogramChunkLayout(&bs, l.schema, l.zeroThreshold, l.positiveSpans, l.negativeSpans, l.customValues)
 	}
 
 	bsr := newBReader(bs.bytes())
 
 	for _, want := range layouts {
-		gotSchema, gotZeroThreshold, gotPositiveSpans, gotNegativeSpans, err := readHistogramChunkLayout(&bsr)
+		gotSchema, gotZeroThreshold, gotPositiveSpans, gotNegativeSpans, gotCustomBounds, err := readHistogramChunkLayout(&bsr)
 		require.NoError(t, err)
 		require.Equal(t, want.schema, gotSchema)
 		require.Equal(t, want.zeroThreshold, gotZeroThreshold)
 		require.Equal(t, want.positiveSpans, gotPositiveSpans)
 		require.Equal(t, want.negativeSpans, gotNegativeSpans)
+		require.Equal(t, want.customValues, gotCustomBounds)
 	}
 }
 

tsdb/chunkenc/histogram_test.go

@@ -294,7 +294,38 @@ func TestHistogramChunkBucketChanges(t *testing.T) {
 }
 
 func TestHistogramChunkAppendable(t *testing.T) {
-	setup := func() (Chunk, *HistogramAppender, int64, *histogram.Histogram) {
+	eh := &histogram.Histogram{
+		Count:         5,
+		ZeroCount:     2,
+		Sum:           18.4,
+		ZeroThreshold: 1e-125,
+		Schema:        1,
+		PositiveSpans: []histogram.Span{
+			{Offset: 0, Length: 2},
+			{Offset: 2, Length: 1},
+			{Offset: 3, Length: 2},
+			{Offset: 3, Length: 1},
+			{Offset: 1, Length: 1},
+		},
+		PositiveBuckets: []int64{6, -3, 0, -1, 2, 1, -4}, // counts: 6, 3, 3, 2, 4, 5, 1 (total 24)
+	}
+
+	cbh := &histogram.Histogram{
+		Count:  24,
+		Sum:    18.4,
+		Schema: histogram.CustomBucketsSchema,
+		PositiveSpans: []histogram.Span{
+			{Offset: 0, Length: 2},
+			{Offset: 2, Length: 1},
+			{Offset: 3, Length: 2},
+			{Offset: 3, Length: 1},
+			{Offset: 1, Length: 1},
+		},
+		PositiveBuckets: []int64{6, -3, 0, -1, 2, 1, -4}, // counts: 6, 3, 3, 2, 4, 5, 1 (total 24)
+		CustomValues:    []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20},
+	}
+
+	setup := func(h *histogram.Histogram) (Chunk, *HistogramAppender, int64, *histogram.Histogram) {
 		c := Chunk(NewHistogramChunk())
 
 		// Create fresh appender and add the first histogram.
@@ -303,32 +334,17 @@ func TestHistogramChunkAppendable(t *testing.T) {
 		require.Equal(t, 0, c.NumSamples())
 
 		ts := int64(1234567890)
-		h1 := &histogram.Histogram{
-			Count:         5,
-			ZeroCount:     2,
-			Sum:           18.4,
-			ZeroThreshold: 1e-125,
-			Schema:        1,
-			PositiveSpans: []histogram.Span{
-				{Offset: 0, Length: 2},
-				{Offset: 2, Length: 1},
-				{Offset: 3, Length: 2},
-				{Offset: 3, Length: 1},
-				{Offset: 1, Length: 1},
-			},
-			PositiveBuckets: []int64{6, -3, 0, -1, 2, 1, -4}, // counts: 6, 3, 3, 2, 4, 5, 1 (total 24)
-		}
 
-		chk, _, app, err := app.AppendHistogram(nil, ts, h1.Copy(), false)
+		chk, _, app, err := app.AppendHistogram(nil, ts, h.Copy(), false)
 		require.NoError(t, err)
 		require.Nil(t, chk)
 		require.Equal(t, 1, c.NumSamples())
 		require.Equal(t, UnknownCounterReset, c.(*HistogramChunk).GetCounterResetHeader())
-		return c, app.(*HistogramAppender), ts, h1
+		return c, app.(*HistogramAppender), ts, h
 	}
 
 	{ // Schema change.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.Schema++
 		_, _, ok, _ := hApp.appendable(h2)
@@ -338,7 +354,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
 	}
 
 	{ // Zero threshold change.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.ZeroThreshold += 0.1
 		_, _, ok, _ := hApp.appendable(h2)
@@ -348,7 +364,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
 	}
 
 	{ // New histogram that has more buckets.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.PositiveSpans = []histogram.Span{
 			{Offset: 0, Length: 3},
@@ -374,7 +390,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
 	}
 
 	{ // New histogram that has a bucket missing.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.PositiveSpans = []histogram.Span{
 			{Offset: 0, Length: 2},
@@ -395,7 +411,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
 	}
 
 	{ // New histogram that has a counter reset while buckets are same.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.Sum = 23
 		h2.PositiveBuckets = []int64{6, -4, 1, -1, 2, 1, -4} // counts: 6, 2, 3, 2, 4, 5, 1 (total 23)
@@ -410,7 +426,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
 	}
 
 	{ // New histogram that has a counter reset while new buckets were added.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.PositiveSpans = []histogram.Span{
 			{Offset: 0, Length: 3},
@@ -438,7 +454,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
 		// added before the first bucket and reset on first bucket.  (to
 		// catch the edge case where the new bucket should be forwarded
 		// ahead until first old bucket at start)
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.PositiveSpans = []histogram.Span{
 			{Offset: -3, Length: 2},
@@ -464,7 +480,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
 	}
 
 	{ // New histogram that has an explicit counter reset.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.CounterResetHint = histogram.CounterReset
 
@@ -472,7 +488,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
 	}
 
 	{ // Start new chunk explicitly, and append a new histogram that is considered appendable to the previous chunk.
-		_, hApp, ts, h1 := setup()
+		_, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy() // Identity is appendable.
 
 		nextChunk := NewHistogramChunk()
@@ -488,7 +504,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
 	}
 
 	{ // Start new chunk explicitly, and append a new histogram that is not considered appendable to the previous chunk.
-		_, hApp, ts, h1 := setup()
+		_, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.Count-- // Make this not appendable due to counter reset.
 
@@ -505,7 +521,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
 	}
 
 	{ // Start new chunk explicitly, and append a new histogram that would need recoding if we added it to the chunk.
-		_, hApp, ts, h1 := setup()
+		_, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.PositiveSpans = []histogram.Span{
 			{Offset: 0, Length: 3},
@@ -532,6 +548,72 @@ func TestHistogramChunkAppendable(t *testing.T) {
 		assertSampleCount(t, nextChunk, 1, ValHistogram)
 		require.Equal(t, NotCounterReset, nextChunk.GetCounterResetHeader())
 	}
+
+	{ // Custom buckets, no change.
+		c, hApp, ts, h1 := setup(cbh)
+		h2 := h1.Copy()
+		_, _, ok, _ := hApp.appendable(h2)
+		require.True(t, ok)
+
+		assertNoNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
+	}
+
+	{ // Custom buckets, increase in bucket counts but no change in layout.
+		c, hApp, ts, h1 := setup(cbh)
+		h2 := h1.Copy()
+		h2.Count++
+		h2.PositiveBuckets = []int64{6, -3, 0, -1, 2, 1, -3}
+		_, _, ok, _ := hApp.appendable(h2)
+		require.True(t, ok)
+
+		assertNoNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
+	}
+
+	{ // Custom buckets, decrease in bucket counts but no change in layout.
+		c, hApp, ts, h1 := setup(cbh)
+		h2 := h1.Copy()
+		h2.Count--
+		h2.PositiveBuckets = []int64{6, -3, 0, -1, 2, 1, -5}
+		_, _, ok, _ := hApp.appendable(h2)
+		require.False(t, ok)
+
+		assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
+	}
+
+	{ // Custom buckets, change only in custom bounds.
+		c, hApp, ts, h1 := setup(cbh)
+		h2 := h1.Copy()
+		h2.CustomValues = []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21}
+		_, _, ok, _ := hApp.appendable(h2)
+		require.False(t, ok)
+
+		assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
+	}
+
+	{ // Custom buckets, with more buckets.
+		c, hApp, ts, h1 := setup(cbh)
+		h2 := h1.Copy()
+		h2.PositiveSpans = []histogram.Span{
+			{Offset: 0, Length: 3},
+			{Offset: 1, Length: 1},
+			{Offset: 1, Length: 4},
+			{Offset: 3, Length: 3},
+		}
+		h2.Count += 6
+		h2.Sum = 30
+		// Existing histogram should get values converted from the above to:
+		//   6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between)
+		// so the new histogram should have new counts >= these per-bucket counts, e.g.:
+		h2.PositiveBuckets = []int64{7, -2, -4, 2, -2, -1, 2, 3, 0, -5, 1} // 7 5 1 3 1 0 2 5 5 0 1 (total 30)
+
+		posInterjections, negInterjections, ok, cr := hApp.appendable(h2)
+		require.NotEmpty(t, posInterjections)
+		require.Empty(t, negInterjections)
+		require.True(t, ok) // Only new buckets came in.
+		require.False(t, cr)
+
+		assertRecodedHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
+	}
 }
 
 func assertNewHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *HistogramAppender, ts int64, h *histogram.Histogram, expectHeader CounterResetHeader) {
@@ -548,6 +630,19 @@ func assertNewHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *Histogr
 	assertSampleCount(t, newChunk, 1, ValHistogram)
 }
 
+func assertNoNewHistogramChunkOnAppend(t *testing.T, currChunk Chunk, hApp *HistogramAppender, ts int64, h *histogram.Histogram, expectHeader CounterResetHeader) {
+	prevChunkBytes := currChunk.Bytes()
+	newChunk, recoded, newAppender, err := hApp.AppendHistogram(nil, ts, h, false)
+	require.Greater(t, len(currChunk.Bytes()), len(prevChunkBytes)) // Check that current chunk is bigger than previously.
+	require.NoError(t, err)
+	require.Nil(t, newChunk)
+	require.False(t, recoded)
+	require.Equal(t, expectHeader, currChunk.(*HistogramChunk).GetCounterResetHeader())
+	require.NotNil(t, newAppender)
+	require.Equal(t, hApp, newAppender)
+	assertSampleCount(t, currChunk, 2, ValHistogram)
+}
+
 func assertRecodedHistogramChunkOnAppend(t *testing.T, prevChunk Chunk, hApp *HistogramAppender, ts int64, h *histogram.Histogram, expectHeader CounterResetHeader) {
 	prevChunkBytes := prevChunk.Bytes()
 	newChunk, recoded, newAppender, err := hApp.AppendHistogram(nil, ts, h, false)
@@ -738,6 +833,32 @@ func TestHistogramChunkAppendableWithEmptySpan(t *testing.T) {
 				NegativeBuckets: []int64{1, 3, -2, 5, -2, 0, -3},
 			},
 		},
+		"empty span in old and new custom buckets histogram": {
+			h1: &histogram.Histogram{
+				Schema: histogram.CustomBucketsSchema,
+				Count:  7,
+				Sum:    1234.5,
+				PositiveSpans: []histogram.Span{
+					{Offset: 0, Length: 4},
+					{Offset: 0, Length: 0},
+					{Offset: 0, Length: 3},
+				},
+				PositiveBuckets: []int64{1, 1, -1, 0, 0, 0, 0},
+				CustomValues:    []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
+			},
+			h2: &histogram.Histogram{
+				Schema: histogram.CustomBucketsSchema,
+				Count:  10,
+				Sum:    2345.6,
+				PositiveSpans: []histogram.Span{
+					{Offset: 0, Length: 4},
+					{Offset: 0, Length: 0},
+					{Offset: 0, Length: 3},
+				},
+				PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 0},
+				CustomValues:    []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
+			},
+		},
 	}
 
 	for name, tc := range tests {
@@ -905,7 +1026,40 @@ func TestAtFloatHistogram(t *testing.T) {
 }
 
 func TestHistogramChunkAppendableGauge(t *testing.T) {
-	setup := func() (Chunk, *HistogramAppender, int64, *histogram.Histogram) {
+	eh := &histogram.Histogram{
+		CounterResetHint: histogram.GaugeType,
+		Count:            5,
+		ZeroCount:        2,
+		Sum:              18.4,
+		ZeroThreshold:    1e-125,
+		Schema:           1,
+		PositiveSpans: []histogram.Span{
+			{Offset: 0, Length: 2},
+			{Offset: 2, Length: 1},
+			{Offset: 3, Length: 2},
+			{Offset: 3, Length: 1},
+			{Offset: 1, Length: 1},
+		},
+		PositiveBuckets: []int64{6, -3, 0, -1, 2, 1, -4}, // {6, 3, 3, 2, 4, 5, 1}
+	}
+
+	cbh := &histogram.Histogram{
+		CounterResetHint: histogram.GaugeType,
+		Count:            24,
+		Sum:              18.4,
+		Schema:           histogram.CustomBucketsSchema,
+		PositiveSpans: []histogram.Span{
+			{Offset: 0, Length: 2},
+			{Offset: 2, Length: 1},
+			{Offset: 3, Length: 2},
+			{Offset: 3, Length: 1},
+			{Offset: 1, Length: 1},
+		},
+		PositiveBuckets: []int64{6, -3, 0, -1, 2, 1, -4}, // {6, 3, 3, 2, 4, 5, 1}
+		CustomValues:    []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20},
+	}
+
+	setup := func(h *histogram.Histogram) (Chunk, *HistogramAppender, int64, *histogram.Histogram) {
 		c := Chunk(NewHistogramChunk())
 
 		// Create fresh appender and add the first histogram.
@@ -914,66 +1068,38 @@ func TestHistogramChunkAppendableGauge(t *testing.T) {
 		require.Equal(t, 0, c.NumSamples())
 
 		ts := int64(1234567890)
-		h1 := &histogram.Histogram{
-			CounterResetHint: histogram.GaugeType,
-			Count:            5,
-			ZeroCount:        2,
-			Sum:              18.4,
-			ZeroThreshold:    1e-125,
-			Schema:           1,
-			PositiveSpans: []histogram.Span{
-				{Offset: 0, Length: 2},
-				{Offset: 2, Length: 1},
-				{Offset: 3, Length: 2},
-				{Offset: 3, Length: 1},
-				{Offset: 1, Length: 1},
-			},
-			PositiveBuckets: []int64{6, -3, 0, -1, 2, 1, -4}, // {6, 3, 3, 2, 4, 5, 1}
-		}
 
-		chk, _, app, err := app.AppendHistogram(nil, ts, h1.Copy(), false)
+		chk, _, app, err := app.AppendHistogram(nil, ts, h.Copy(), false)
 		require.NoError(t, err)
 		require.Nil(t, chk)
 		require.Equal(t, 1, c.NumSamples())
 		require.Equal(t, GaugeType, c.(*HistogramChunk).GetCounterResetHeader())
 
-		return c, app.(*HistogramAppender), ts, h1
+		return c, app.(*HistogramAppender), ts, h
 	}
 
 	{ // Schema change.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.Schema++
 		_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
 		require.False(t, ok)
 
-		newc, recoded, _, err := hApp.AppendHistogram(nil, ts+1, h2, false)
-		require.NoError(t, err)
-		require.NotNil(t, newc)
-		require.False(t, recoded)
-		require.NotEqual(t, c, newc)
-		require.Equal(t, GaugeType, c.(*HistogramChunk).GetCounterResetHeader())
-		require.Equal(t, GaugeType, newc.(*HistogramChunk).GetCounterResetHeader())
+		assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
 	}
 
 	{ // Zero threshold change.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.ZeroThreshold += 0.1
 		_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
 		require.False(t, ok)
 
-		newc, recoded, _, err := hApp.AppendHistogram(nil, ts+1, h2, false)
-		require.NoError(t, err)
-		require.NotNil(t, newc)
-		require.False(t, recoded)
-		require.NotEqual(t, c, newc)
-		require.Equal(t, GaugeType, c.(*HistogramChunk).GetCounterResetHeader())
-		require.Equal(t, GaugeType, newc.(*HistogramChunk).GetCounterResetHeader())
+		assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
 	}
 
 	{ // New histogram that has more buckets.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.PositiveSpans = []histogram.Span{
 			{Offset: 0, Length: 3},
@@ -993,15 +1119,11 @@ func TestHistogramChunkAppendableGauge(t *testing.T) {
 		require.Empty(t, nBackwardI)
 		require.True(t, ok)
 
-		newc, recoded, _, err := hApp.AppendHistogram(nil, ts+1, h2, false)
-		require.NoError(t, err)
-		require.NotNil(t, newc)
-		require.True(t, recoded)
-		require.Equal(t, GaugeType, c.(*HistogramChunk).GetCounterResetHeader())
+		assertRecodedHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
 	}
 
 	{ // New histogram that has buckets missing.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.PositiveSpans = []histogram.Span{
 			{Offset: 0, Length: 2},
@@ -1021,15 +1143,11 @@ func TestHistogramChunkAppendableGauge(t *testing.T) {
 		require.Empty(t, nBackwardI)
 		require.True(t, ok)
 
-		newc, recoded, _, err := hApp.AppendHistogram(nil, ts+1, h2, false)
-		require.NoError(t, err)
-		require.Nil(t, newc)
-		require.False(t, recoded)
-		require.Equal(t, GaugeType, c.(*HistogramChunk).GetCounterResetHeader())
+		assertNoNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
 	}
 
 	{ // New histogram that has a bucket missing and new buckets.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.PositiveSpans = []histogram.Span{
 			{Offset: 0, Length: 2},
@@ -1047,15 +1165,11 @@ func TestHistogramChunkAppendableGauge(t *testing.T) {
 		require.Empty(t, nBackwardI)
 		require.True(t, ok)
 
-		newc, recoded, _, err := hApp.AppendHistogram(nil, ts+1, h2, false)
-		require.NoError(t, err)
-		require.NotNil(t, newc)
-		require.True(t, recoded)
-		require.Equal(t, GaugeType, c.(*HistogramChunk).GetCounterResetHeader())
+		assertRecodedHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
 	}
 
 	{ // New histogram that has a counter reset while buckets are same.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.Sum = 23
 		h2.PositiveBuckets = []int64{6, -4, 1, -1, 2, 1, -4} // {6, 2, 3, 2, 4, 5, 1}
@@ -1067,15 +1181,11 @@ func TestHistogramChunkAppendableGauge(t *testing.T) {
 		require.Empty(t, nBackwardI)
 		require.True(t, ok)
 
-		newc, recoded, _, err := hApp.AppendHistogram(nil, ts+1, h2, false)
-		require.NoError(t, err)
-		require.Nil(t, newc)
-		require.False(t, recoded)
-		require.Equal(t, GaugeType, c.(*HistogramChunk).GetCounterResetHeader())
+		assertNoNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
 	}
 
 	{ // New histogram that has a counter reset while new buckets were added.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.PositiveSpans = []histogram.Span{
 			{Offset: 0, Length: 3},
@@ -1093,17 +1203,13 @@ func TestHistogramChunkAppendableGauge(t *testing.T) {
 		require.Empty(t, nBackwardI)
 		require.True(t, ok)
 
-		newc, recoded, _, err := hApp.AppendHistogram(nil, ts+1, h2, false)
-		require.NoError(t, err)
-		require.NotNil(t, newc)
-		require.True(t, recoded)
-		require.Equal(t, GaugeType, c.(*HistogramChunk).GetCounterResetHeader())
+		assertRecodedHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
 	}
 
 	{
 		// New histogram that has a counter reset while new buckets were
 		// added before the first bucket and reset on first bucket.
-		c, hApp, ts, h1 := setup()
+		c, hApp, ts, h1 := setup(eh)
 		h2 := h1.Copy()
 		h2.PositiveSpans = []histogram.Span{
 			{Offset: -3, Length: 2},
@@ -1123,11 +1229,74 @@ func TestHistogramChunkAppendableGauge(t *testing.T) {
 		require.Empty(t, nBackwardI)
 		require.True(t, ok)
 
-		newc, recoded, _, err := hApp.AppendHistogram(nil, ts+1, h2, false)
-		require.NoError(t, err)
-		require.NotNil(t, newc)
-		require.True(t, recoded)
-		require.Equal(t, GaugeType, c.(*HistogramChunk).GetCounterResetHeader())
+		assertRecodedHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
+	}
+
+	{ // Custom buckets, no change.
+		c, hApp, ts, h1 := setup(cbh)
+		h2 := h1.Copy()
+		_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
+		require.True(t, ok)
+
+		assertNoNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
+	}
+
+	{ // Custom buckets, increase in bucket counts but no change in layout.
+		c, hApp, ts, h1 := setup(cbh)
+		h2 := h1.Copy()
+		h2.Count++
+		h2.PositiveBuckets = []int64{6, -3, 0, -1, 2, 1, -3}
+		_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
+		require.True(t, ok)
+
+		assertNoNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
+	}
+
+	{ // Custom buckets, decrease in bucket counts but no change in layout.
+		c, hApp, ts, h1 := setup(cbh)
+		h2 := h1.Copy()
+		h2.Count--
+		h2.PositiveBuckets = []int64{6, -3, 0, -1, 2, 1, -5}
+		_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
+		require.True(t, ok)
+
+		assertNoNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
+	}
+
+	{ // Custom buckets, change only in custom bounds.
+		c, hApp, ts, h1 := setup(cbh)
+		h2 := h1.Copy()
+		h2.CustomValues = []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21}
+		_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
+		require.False(t, ok)
+
+		assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
+	}
+
+	{ // Custom buckets, with more buckets.
+		c, hApp, ts, h1 := setup(cbh)
+		h2 := h1.Copy()
+		h2.PositiveSpans = []histogram.Span{
+			{Offset: 0, Length: 3},
+			{Offset: 1, Length: 1},
+			{Offset: 1, Length: 4},
+			{Offset: 3, Length: 3},
+		}
+		h2.Count += 6
+		h2.Sum = 30
+		// Existing histogram should get values converted from the above to:
+		//   6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between)
+		// so the new histogram should have new counts >= these per-bucket counts, e.g.:
+		h2.PositiveBuckets = []int64{7, -2, -4, 2, -2, -1, 2, 3, 0, -5, 1} // 7 5 1 3 1 0 2 5 5 0 1 (total 30)
+
+		posInterjections, negInterjections, pBackwardI, nBackwardI, _, _, ok := hApp.appendableGauge(h2)
+		require.NotEmpty(t, posInterjections)
+		require.Empty(t, negInterjections)
+		require.Empty(t, pBackwardI)
+		require.Empty(t, nBackwardI)
+		require.True(t, ok) // Only new buckets came in.
+
+		assertRecodedHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
 	}
 }
 
@@ -1176,4 +1345,26 @@ func TestHistogramAppendOnlyErrors(t *testing.T) {
 		require.False(t, isRecoded)
 		require.EqualError(t, err, "histogram counter reset")
 	})
+	t.Run("counter reset error with custom buckets", func(t *testing.T) {
+		c := Chunk(NewHistogramChunk())
+
+		// Create fresh appender and add the first histogram.
+		app, err := c.Appender()
+		require.NoError(t, err)
+
+		h := tsdbutil.GenerateTestCustomBucketsHistogram(0)
+		var isRecoded bool
+		c, isRecoded, app, err = app.AppendHistogram(nil, 1, h, true)
+		require.Nil(t, c)
+		require.False(t, isRecoded)
+		require.NoError(t, err)
+
+		// Add erroring histogram.
+		h2 := h.Copy()
+		h2.CustomValues = []float64{0, 1, 2, 3, 4, 5, 6, 7}
+		c, isRecoded, _, err = app.AppendHistogram(nil, 2, h2, true)
+		require.Nil(t, c)
+		require.False(t, isRecoded)
+		require.EqualError(t, err, "histogram counter reset")
+	})
 }

tsdb/tsdbutil/histogram.go

@@ -59,6 +59,20 @@ func GenerateTestHistogram(i int) *histogram.Histogram {
 	}
 }
 
+func GenerateTestCustomBucketsHistogram(i int) *histogram.Histogram {
+	return &histogram.Histogram{
+		Count:  5 + uint64(i*4),
+		Sum:    18.4 * float64(i+1),
+		Schema: histogram.CustomBucketsSchema,
+		PositiveSpans: []histogram.Span{
+			{Offset: 0, Length: 2},
+			{Offset: 1, Length: 2},
+		},
+		PositiveBuckets: []int64{int64(i + 1), 1, -1, 0},
+		CustomValues:    []float64{0, 1, 2, 3, 4},
+	}
+}
+
 func GenerateTestGaugeHistograms(n int) (r []*histogram.Histogram) {
 	for x := 0; x < n; x++ {
 		i := int(math.Sin(float64(x))*100) + 100
@@ -105,6 +119,20 @@ func GenerateTestFloatHistogram(i int) *histogram.FloatHistogram {
 	}
 }
 
+func GenerateTestCustomBucketsFloatHistogram(i int) *histogram.FloatHistogram {
+	return &histogram.FloatHistogram{
+		Count:  5 + float64(i*4),
+		Sum:    18.4 * float64(i+1),
+		Schema: histogram.CustomBucketsSchema,
+		PositiveSpans: []histogram.Span{
+			{Offset: 0, Length: 2},
+			{Offset: 1, Length: 2},
+		},
+		PositiveBuckets: []float64{float64(i + 1), float64(i + 2), float64(i + 1), float64(i + 1)},
+		CustomValues:    []float64{0, 1, 2, 3, 4},
+	}
+}
+
 func GenerateTestGaugeFloatHistograms(n int) (r []*histogram.FloatHistogram) {
 	for x := 0; x < n; x++ {
 		i := int(math.Sin(float64(x))*100) + 100

util/annotations/annotations.go

@@ -94,6 +94,19 @@ func (a Annotations) AsStrings(query string, maxAnnos int) []string {
 	return arr
 }
 
+func (a Annotations) CountWarningsAndInfo() (int, int) {
+	var countWarnings, countInfo int
+	for _, err := range a {
+		if errors.Is(err, PromQLWarning) {
+			countWarnings++
+		}
+		if errors.Is(err, PromQLInfo) {
+			countInfo++
+		}
+	}
+	return countWarnings, countInfo
+}
+
 //nolint:revive // error-naming.
 var (
 	// Currently there are only 2 types, warnings and info.
@@ -103,12 +116,14 @@ var (
 	PromQLInfo    = errors.New("PromQL info")
 	PromQLWarning = errors.New("PromQL warning")
 
-	InvalidQuantileWarning              = fmt.Errorf("%w: quantile value should be between 0 and 1", PromQLWarning)
-	BadBucketLabelWarning               = fmt.Errorf("%w: bucket label %q is missing or has a malformed value", PromQLWarning, model.BucketLabel)
-	MixedFloatsHistogramsWarning        = fmt.Errorf("%w: encountered a mix of histograms and floats for", PromQLWarning)
-	MixedClassicNativeHistogramsWarning = fmt.Errorf("%w: vector contains a mix of classic and native histograms for metric name", PromQLWarning)
-	NativeHistogramNotCounterWarning    = fmt.Errorf("%w: this native histogram metric is not a counter:", PromQLWarning)
-	NativeHistogramNotGaugeWarning      = fmt.Errorf("%w: this native histogram metric is not a gauge:", PromQLWarning)
+	InvalidQuantileWarning                     = fmt.Errorf("%w: quantile value should be between 0 and 1", PromQLWarning)
+	BadBucketLabelWarning                      = fmt.Errorf("%w: bucket label %q is missing or has a malformed value", PromQLWarning, model.BucketLabel)
+	MixedFloatsHistogramsWarning               = fmt.Errorf("%w: encountered a mix of histograms and floats for", PromQLWarning)
+	MixedClassicNativeHistogramsWarning        = fmt.Errorf("%w: vector contains a mix of classic and native histograms for metric name", PromQLWarning)
+	NativeHistogramNotCounterWarning           = fmt.Errorf("%w: this native histogram metric is not a counter:", PromQLWarning)
+	NativeHistogramNotGaugeWarning             = fmt.Errorf("%w: this native histogram metric is not a gauge:", PromQLWarning)
+	MixedExponentialCustomHistogramsWarning    = fmt.Errorf("%w: vector contains a mix of histograms with exponential and custom buckets schemas for metric name", PromQLWarning)
+	IncompatibleCustomBucketsHistogramsWarning = fmt.Errorf("%w: vector contains histograms with incompatible custom buckets for metric name", PromQLWarning)
 
 	PossibleNonCounterInfo                  = fmt.Errorf("%w: metric might not be a counter, name does not end in _total/_sum/_count/_bucket:", PromQLInfo)
 	HistogramQuantileForcedMonotonicityInfo = fmt.Errorf("%w: input to histogram_quantile needed to be fixed for monotonicity (see https://prometheus.io/docs/prometheus/latest/querying/functions/#histogram_quantile) for metric name", PromQLInfo)
@@ -195,6 +210,24 @@ func NewNativeHistogramNotGaugeWarning(metricName string, pos posrange.PositionR
 	}
 }
 
+// NewMixedExponentialCustomHistogramsWarning is used when the queried series includes
+// histograms with both exponential and custom buckets schemas.
+func NewMixedExponentialCustomHistogramsWarning(metricName string, pos posrange.PositionRange) error {
+	return annoErr{
+		PositionRange: pos,
+		Err:           fmt.Errorf("%w %q", MixedExponentialCustomHistogramsWarning, metricName),
+	}
+}
+
+// NewIncompatibleCustomBucketsHistogramsWarning is used when the queried series includes
+// custom buckets histograms with incompatible custom bounds.
+func NewIncompatibleCustomBucketsHistogramsWarning(metricName string, pos posrange.PositionRange) error {
+	return annoErr{
+		PositionRange: pos,
+		Err:           fmt.Errorf("%w %q", IncompatibleCustomBucketsHistogramsWarning, metricName),
+	}
+}
+
 // NewPossibleNonCounterInfo is used when a named counter metric with only float samples does not
 // have the suffixes _total, _sum, _count, or _bucket.
 func NewPossibleNonCounterInfo(metricName string, pos posrange.PositionRange) error {