mirror of
https://github.com/prometheus/prometheus.git
synced 2025-03-05 20:59:13 -08:00
Style cleanup of all the changes in sparsehistogram so far
A lot of this code was hacked together, literally during a hackathon. This commit intends not to change the code substantially, but just make the code obey the usual style practices. A (possibly incomplete) list of areas: * Generally address linter warnings. * The `pgk` directory is deprecated as per dev-summit. No new packages should be added to it. I moved the new `pkg/histogram` package to `model` anticipating what's proposed in #9478. * Make the naming of the Sparse Histogram more consistent. Including abbreviations, there were just too many names for it: SparseHistogram, Histogram, Histo, hist, his, shs, h. The idea is to call it "Histogram" in general. Only add "Sparse" if it is needed to avoid confusion with conventional Histograms (which is rare because the TSDB really has no notion of conventional Histograms). Use abbreviations only in local scope, and then really abbreviate (not just removing three out of seven letters like in "Histo"). This is in the spirit of https://github.com/golang/go/wiki/CodeReviewComments#variable-names * Several other minor name changes. * A lot of formatting of doc comments. For one, following https://github.com/golang/go/wiki/CodeReviewComments#comment-sentences , but also layout question, anticipating how things will look like when rendered by `godoc` (even where `godoc` doesn't render them right now because they are for unexported types or not a doc comment at all but just a normal code comment - consistency is queen!). * Re-enabled `TestQueryLog` and `TestEndopints` (they pass now, leaving them disabled was presumably an oversight). * Bucket iterator for histogram.Histogram is now created with a method. * HistogramChunk.iterator now allows iterator recycling. (I think @dieterbe only commented it out because he was confused by the question in the comment.) * HistogramAppender.Append panics now because we decided to treat staleness marker differently. Signed-off-by: beorn7 <beorn@grafana.com>
This commit is contained in:
beorn7
parent
fd5ea4e0b5
commit
7a8bb8222c
|
|
@ -58,9 +58,9 @@ import (
|
|||
"github.com/prometheus/prometheus/config"
|
||||
"github.com/prometheus/prometheus/discovery"
|
||||
_ "github.com/prometheus/prometheus/discovery/install" // Register service discovery implementations.
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/notifier"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/pkg/logging"
|
||||
"github.com/prometheus/prometheus/pkg/relabel"
|
||||
|
|
@ -1190,7 +1190,7 @@ func (n notReadyAppender) AppendExemplar(ref uint64, l labels.Labels, e exemplar
|
|||
return 0, tsdb.ErrNotReady
|
||||
}
|
||||
|
||||
func (n notReadyAppender) AppendHistogram(ref uint64, l labels.Labels, t int64, sh histogram.SparseHistogram) (uint64, error) {
|
||||
func (n notReadyAppender) AppendHistogram(ref uint64, l labels.Labels, t int64, sh histogram.Histogram) (uint64, error) {
|
||||
return 0, tsdb.ErrNotReady
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -405,7 +405,6 @@ func readQueryLog(t *testing.T, path string) []queryLogLine {
|
|||
}
|
||||
|
||||
func TestQueryLog(t *testing.T) {
|
||||
t.Skip()
|
||||
if testing.Short() {
|
||||
t.Skip("skipping test in short mode.")
|
||||
}
|
||||
|
|
|
|||
2
go.mod
2
go.mod
|
|
@ -62,7 +62,7 @@ require (
|
|||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
|
||||
golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34
|
||||
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac
|
||||
golang.org/x/tools v0.1.5
|
||||
golang.org/x/tools v0.1.7
|
||||
google.golang.org/api v0.56.0
|
||||
google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83
|
||||
google.golang.org/protobuf v1.27.1
|
||||
|
|
|
|||
6
go.sum
6
go.sum
|
|
@ -1308,6 +1308,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
|
|||
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
|
||||
github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA=
|
||||
github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg=
|
||||
|
|
@ -1497,6 +1498,7 @@ golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qx
|
|||
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210903162142-ad29c8ab022f h1:w6wWR0H+nyVpbSAQbzVEIACVyr/h8l/BEkY6Sokc7Eg=
|
||||
golang.org/x/net v0.0.0-20210903162142-ad29c8ab022f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
|
|
@ -1634,6 +1636,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
|
|||
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34 h1:GkvMjFtXUmahfDtashnc1mnrCtuBVcwse5QV2lUk/tI=
|
||||
golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
|
|
@ -1739,8 +1742,9 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
|||
golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA=
|
||||
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ=
|
||||
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
|
|
|
|||
|
|
@ -17,55 +17,90 @@ import (
|
|||
"math"
|
||||
)
|
||||
|
||||
// SparseHistogram encodes a sparse histogram
|
||||
// full details: https://docs.google.com/document/d/1cLNv3aufPZb3fNfaJgdaRBZsInZKKIHo9E6HinJVbpM/edit#
|
||||
// the most tricky bit is how bucket indices represent real bucket boundaries
|
||||
// Histogram encodes a sparse, high-resolution histogram. See the design
|
||||
// document for full details:
|
||||
// https://docs.google.com/document/d/1cLNv3aufPZb3fNfaJgdaRBZsInZKKIHo9E6HinJVbpM/edit#
|
||||
//
|
||||
// an example for schema 0 (which doubles the size of consecutive buckets):
|
||||
// The most tricky bit is how bucket indices represent real bucket boundaries.
|
||||
// An example for schema 0 (by which each bucket is twice as wide as the
|
||||
// previous bucket):
|
||||
//
|
||||
// buckets syntax (LE,GE) (-2,-1) (-1,-0.5) (-0.5,-0.25) ... (-0.001-0.001) ... (0.25-0.5)(0.5-1) (1-2) ....
|
||||
// ^
|
||||
// zero bucket (here width a width of 0.001) ZB
|
||||
// pos bucket idx ... -1 0 1 2 3
|
||||
// neg bucket idx 3 2 1 0 -1 ...
|
||||
// actively used bucket indices themselves are represented by the spans
|
||||
type SparseHistogram struct {
|
||||
Schema int32
|
||||
ZeroThreshold float64
|
||||
ZeroCount, Count uint64
|
||||
Sum float64
|
||||
PositiveSpans, NegativeSpans []Span
|
||||
// Bucket boundaries → [-2,-1) [-1,-0.5) [-0.5,-0.25) ... [-0.001,0.001] ... (0.25,0.5] (0.5,1] (1,2] ....
|
||||
// ↑ ↑ ↑ ↑ ↑ ↑ ↑
|
||||
// Zero bucket (width e.g. 0.001) → | | | ZB | | |
|
||||
// Positive bucket indices → | | | ... -1 0 1 2 3
|
||||
// Negative bucket indices → 3 2 1 0 -1 ...
|
||||
//
|
||||
// Wich bucket indices are actually used is determined by the spans.
|
||||
type Histogram struct {
|
||||
// Currently valid schema numbers are -4 <= n <= 8. They are all for
|
||||
// base-2 bucket schemas, where 1 is a bucket boundary in each case, and
|
||||
// then each power of two is divided into 2^n logarithmic buckets. Or
|
||||
// in other words, each bucket boundary is the previous boundary times
|
||||
// 2^(2^-n).
|
||||
Schema int32
|
||||
// Width of the zero bucket.
|
||||
ZeroThreshold float64
|
||||
// Observations falling into the zero bucket.
|
||||
ZeroCount uint64
|
||||
// Total number of observations.
|
||||
Count uint64
|
||||
// Sum of observations.
|
||||
Sum float64
|
||||
// Spans for positive and negative buckets (see Span below).
|
||||
PositiveSpans, NegativeSpans []Span
|
||||
// Observation counts in buckets. The first element is an absolute
|
||||
// count. All following ones are deltas relative to the previous
|
||||
// element.
|
||||
PositiveBuckets, NegativeBuckets []int64
|
||||
}
|
||||
|
||||
// A Span defines a continuous sequence of buckets.
|
||||
type Span struct {
|
||||
// Gap to previous span (always positive), or starting index for the 1st
|
||||
// span (which can be negative).
|
||||
Offset int32
|
||||
// Length of the span.
|
||||
Length uint32
|
||||
}
|
||||
|
||||
func (s SparseHistogram) Copy() SparseHistogram {
|
||||
c := s
|
||||
// Copy returns a deep copy of the Histogram.
|
||||
func (h Histogram) Copy() Histogram {
|
||||
c := h
|
||||
|
||||
if s.PositiveSpans != nil {
|
||||
c.PositiveSpans = make([]Span, len(s.PositiveSpans))
|
||||
copy(c.PositiveSpans, s.PositiveSpans)
|
||||
if h.PositiveSpans != nil {
|
||||
c.PositiveSpans = make([]Span, len(h.PositiveSpans))
|
||||
copy(c.PositiveSpans, h.PositiveSpans)
|
||||
}
|
||||
if s.NegativeSpans != nil {
|
||||
c.NegativeSpans = make([]Span, len(s.NegativeSpans))
|
||||
copy(c.NegativeSpans, s.NegativeSpans)
|
||||
if h.NegativeSpans != nil {
|
||||
c.NegativeSpans = make([]Span, len(h.NegativeSpans))
|
||||
copy(c.NegativeSpans, h.NegativeSpans)
|
||||
}
|
||||
if s.PositiveBuckets != nil {
|
||||
c.PositiveBuckets = make([]int64, len(s.PositiveBuckets))
|
||||
copy(c.PositiveBuckets, s.PositiveBuckets)
|
||||
if h.PositiveBuckets != nil {
|
||||
c.PositiveBuckets = make([]int64, len(h.PositiveBuckets))
|
||||
copy(c.PositiveBuckets, h.PositiveBuckets)
|
||||
}
|
||||
if s.NegativeBuckets != nil {
|
||||
c.NegativeBuckets = make([]int64, len(s.NegativeBuckets))
|
||||
copy(c.NegativeBuckets, s.NegativeBuckets)
|
||||
if h.NegativeBuckets != nil {
|
||||
c.NegativeBuckets = make([]int64, len(h.NegativeBuckets))
|
||||
copy(c.NegativeBuckets, h.NegativeBuckets)
|
||||
}
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
// CumulativeBucketIterator returns a BucketIterator to iterate over a
|
||||
// cumulative view of the buckets. This method currently only supports
|
||||
// Histograms without negative buckets and panics if the Histogram has negative
|
||||
// buckets. It is currently only used for testing.
|
||||
func (h Histogram) CumulativeBucketIterator() BucketIterator {
|
||||
if len(h.NegativeBuckets) > 0 {
|
||||
panic("CumulativeIterator called on Histogram with negative buckets")
|
||||
}
|
||||
return &cumulativeBucketIterator{h: h, posSpansIdx: -1}
|
||||
}
|
||||
|
||||
// BucketIterator iterates over the buckets of a Histogram, returning decoded
|
||||
// buckets.
|
||||
type BucketIterator interface {
|
||||
// Next advances the iterator by one.
|
||||
Next() bool
|
||||
|
|
@ -76,28 +111,23 @@ type BucketIterator interface {
|
|||
Err() error
|
||||
}
|
||||
|
||||
// Bucket represents a bucket (currently only a cumulative one with an upper
|
||||
// inclusive bound and a cumulative count).
|
||||
type Bucket struct {
|
||||
Le float64
|
||||
Upper float64
|
||||
Count uint64
|
||||
}
|
||||
|
||||
// CumulativeExpandSparseHistogram expands the given histogram to produce cumulative buckets.
|
||||
// It assumes that the total length of spans matches the number of buckets for pos and neg respectively.
|
||||
// TODO: supports only positive buckets, also do for negative.
|
||||
func CumulativeExpandSparseHistogram(h SparseHistogram) BucketIterator {
|
||||
return &cumulativeBucketIterator{h: h, posSpansIdx: -1}
|
||||
}
|
||||
|
||||
type cumulativeBucketIterator struct {
|
||||
h SparseHistogram
|
||||
h Histogram
|
||||
|
||||
posSpansIdx int // Index in h.PositiveSpans we are in. -1 means 0 bucket.
|
||||
posBucketsIdx int // Index in h.PositiveBuckets
|
||||
posBucketsIdx int // Index in h.PositiveBuckets.
|
||||
idxInSpan uint32 // Index in the current span. 0 <= idxInSpan < span.Length.
|
||||
|
||||
initialised bool
|
||||
currIdx int32 // The actual bucket index after decoding from spans.
|
||||
currLe float64 // The upper boundary of the current bucket.
|
||||
currUpper float64 // The upper boundary of the current bucket.
|
||||
currCount int64 // Current non-cumulative count for the current bucket. Does not apply for empty bucket.
|
||||
currCumulativeCount uint64 // Current "cumulative" count for the current bucket.
|
||||
|
||||
|
|
@ -116,7 +146,7 @@ func (c *cumulativeBucketIterator) Next() bool {
|
|||
return c.Next()
|
||||
}
|
||||
|
||||
c.currLe = c.h.ZeroThreshold
|
||||
c.currUpper = c.h.ZeroThreshold
|
||||
c.currCount = int64(c.h.ZeroCount)
|
||||
c.currCumulativeCount = uint64(c.currCount)
|
||||
return true
|
||||
|
|
@ -128,7 +158,7 @@ func (c *cumulativeBucketIterator) Next() bool {
|
|||
|
||||
if c.emptyBucketCount > 0 {
|
||||
// We are traversing through empty buckets at the moment.
|
||||
c.currLe = getLe(c.currIdx, c.h.Schema)
|
||||
c.currUpper = getUpper(c.currIdx, c.h.Schema)
|
||||
c.currIdx++
|
||||
c.emptyBucketCount--
|
||||
return true
|
||||
|
|
@ -145,7 +175,7 @@ func (c *cumulativeBucketIterator) Next() bool {
|
|||
|
||||
c.currCount += c.h.PositiveBuckets[c.posBucketsIdx]
|
||||
c.currCumulativeCount += uint64(c.currCount)
|
||||
c.currLe = getLe(c.currIdx, c.h.Schema)
|
||||
c.currUpper = getUpper(c.currIdx, c.h.Schema)
|
||||
|
||||
c.posBucketsIdx++
|
||||
c.idxInSpan++
|
||||
|
|
@ -163,24 +193,26 @@ func (c *cumulativeBucketIterator) Next() bool {
|
|||
}
|
||||
func (c *cumulativeBucketIterator) At() Bucket {
|
||||
return Bucket{
|
||||
Le: c.currLe,
|
||||
Upper: c.currUpper,
|
||||
Count: c.currCumulativeCount,
|
||||
}
|
||||
}
|
||||
func (c *cumulativeBucketIterator) Err() error { return nil }
|
||||
|
||||
func getLe(idx, schema int32) float64 {
|
||||
func getUpper(idx, schema int32) float64 {
|
||||
if schema < 0 {
|
||||
return math.Ldexp(1, int(idx)<<(-schema))
|
||||
}
|
||||
|
||||
fracIdx := idx & ((1 << schema) - 1)
|
||||
frac := sparseBounds[schema][fracIdx]
|
||||
frac := exponentialBounds[schema][fracIdx]
|
||||
exp := (int(idx) >> schema) + 1
|
||||
return math.Ldexp(frac, exp)
|
||||
}
|
||||
|
||||
var sparseBounds = [][]float64{
|
||||
// exponentialBounds is a precalculated table of bucket bounds in the interval
|
||||
// [0.5,1) in schema 0 to 8.
|
||||
var exponentialBounds = [][]float64{
|
||||
// Schema "0":
|
||||
{0.5},
|
||||
// Schema 1:
|
||||
165
model/histogram/histogram_test.go
Normal file
165
model/histogram/histogram_test.go
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
// Copyright 2021 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package histogram
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCumulativeBucketIterator(t *testing.T) {
|
||||
cases := []struct {
|
||||
histogram Histogram
|
||||
expectedCumulativeBuckets []Bucket
|
||||
}{
|
||||
{
|
||||
histogram: Histogram{
|
||||
Schema: 0,
|
||||
PositiveSpans: []Span{
|
||||
{Offset: 0, Length: 2},
|
||||
{Offset: 1, Length: 2},
|
||||
},
|
||||
PositiveBuckets: []int64{1, 1, -1, 0},
|
||||
},
|
||||
expectedCumulativeBuckets: []Bucket{
|
||||
{Upper: 1, Count: 1},
|
||||
{Upper: 2, Count: 3},
|
||||
|
||||
{Upper: 4, Count: 3},
|
||||
|
||||
{Upper: 8, Count: 4},
|
||||
{Upper: 16, Count: 5},
|
||||
},
|
||||
},
|
||||
{
|
||||
histogram: Histogram{
|
||||
Schema: 0,
|
||||
PositiveSpans: []Span{
|
||||
{Offset: 0, Length: 5},
|
||||
{Offset: 1, Length: 1},
|
||||
},
|
||||
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0},
|
||||
},
|
||||
expectedCumulativeBuckets: []Bucket{
|
||||
{Upper: 1, Count: 1},
|
||||
{Upper: 2, Count: 4},
|
||||
{Upper: 4, Count: 5},
|
||||
{Upper: 8, Count: 7},
|
||||
|
||||
{Upper: 16, Count: 8},
|
||||
|
||||
{Upper: 32, Count: 8},
|
||||
{Upper: 64, Count: 9},
|
||||
},
|
||||
},
|
||||
{
|
||||
histogram: Histogram{
|
||||
Schema: 0,
|
||||
PositiveSpans: []Span{
|
||||
{Offset: 0, Length: 7},
|
||||
},
|
||||
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 0},
|
||||
},
|
||||
expectedCumulativeBuckets: []Bucket{
|
||||
{Upper: 1, Count: 1},
|
||||
{Upper: 2, Count: 4},
|
||||
{Upper: 4, Count: 5},
|
||||
{Upper: 8, Count: 7},
|
||||
{Upper: 16, Count: 8},
|
||||
{Upper: 32, Count: 9},
|
||||
{Upper: 64, Count: 10},
|
||||
},
|
||||
},
|
||||
{
|
||||
histogram: Histogram{
|
||||
Schema: 3,
|
||||
PositiveSpans: []Span{
|
||||
{Offset: -5, Length: 2}, // -5 -4
|
||||
{Offset: 2, Length: 3}, // -1 0 1
|
||||
{Offset: 2, Length: 2}, // 4 5
|
||||
},
|
||||
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 3},
|
||||
},
|
||||
expectedCumulativeBuckets: []Bucket{
|
||||
{Upper: 0.6484197773255048, Count: 1}, // -5
|
||||
{Upper: 0.7071067811865475, Count: 4}, // -4
|
||||
|
||||
{Upper: 0.7711054127039704, Count: 4}, // -3
|
||||
{Upper: 0.8408964152537144, Count: 4}, // -2
|
||||
|
||||
{Upper: 0.9170040432046711, Count: 5}, // -1
|
||||
{Upper: 1, Count: 7}, // 1
|
||||
{Upper: 1.0905077326652577, Count: 8}, // 0
|
||||
|
||||
{Upper: 1.189207115002721, Count: 8}, // 1
|
||||
{Upper: 1.2968395546510096, Count: 8}, // 2
|
||||
|
||||
{Upper: 1.414213562373095, Count: 9}, // 3
|
||||
{Upper: 1.5422108254079407, Count: 13}, // 4
|
||||
},
|
||||
},
|
||||
{
|
||||
histogram: Histogram{
|
||||
Schema: -2,
|
||||
PositiveSpans: []Span{
|
||||
{Offset: -2, Length: 4}, // -2 -1 0 1
|
||||
{Offset: 2, Length: 2}, // 4 5
|
||||
},
|
||||
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0},
|
||||
},
|
||||
expectedCumulativeBuckets: []Bucket{
|
||||
{Upper: 0.00390625, Count: 1}, // -2
|
||||
{Upper: 0.0625, Count: 4}, // -1
|
||||
{Upper: 1, Count: 5}, // 0
|
||||
{Upper: 16, Count: 7}, // 1
|
||||
|
||||
{Upper: 256, Count: 7}, // 2
|
||||
{Upper: 4096, Count: 7}, // 3
|
||||
|
||||
{Upper: 65536, Count: 8}, // 4
|
||||
{Upper: 1048576, Count: 9}, // 5
|
||||
},
|
||||
},
|
||||
{
|
||||
histogram: Histogram{
|
||||
Schema: -1,
|
||||
PositiveSpans: []Span{
|
||||
{Offset: -2, Length: 5}, // -2 -1 0 1 2
|
||||
},
|
||||
PositiveBuckets: []int64{1, 2, -2, 1, -1},
|
||||
},
|
||||
expectedCumulativeBuckets: []Bucket{
|
||||
{Upper: 0.0625, Count: 1}, // -2
|
||||
{Upper: 0.25, Count: 4}, // -1
|
||||
{Upper: 1, Count: 5}, // 0
|
||||
{Upper: 4, Count: 7}, // 1
|
||||
{Upper: 16, Count: 8}, // 2
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for i, c := range cases {
|
||||
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
|
||||
it := c.histogram.CumulativeBucketIterator()
|
||||
actualBuckets := make([]Bucket, 0, len(c.expectedCumulativeBuckets))
|
||||
for it.Next() {
|
||||
actualBuckets = append(actualBuckets, it.At())
|
||||
}
|
||||
require.NoError(t, it.Err())
|
||||
require.Equal(t, c.expectedCumulativeBuckets, actualBuckets)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -1,165 +0,0 @@
|
|||
// Copyright 2021 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package histogram
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCumulativeExpandSparseHistogram(t *testing.T) {
|
||||
cases := []struct {
|
||||
hist SparseHistogram
|
||||
expBuckets []Bucket
|
||||
}{
|
||||
{
|
||||
hist: SparseHistogram{
|
||||
Schema: 0,
|
||||
PositiveSpans: []Span{
|
||||
{Offset: 0, Length: 2},
|
||||
{Offset: 1, Length: 2},
|
||||
},
|
||||
PositiveBuckets: []int64{1, 1, -1, 0},
|
||||
},
|
||||
expBuckets: []Bucket{
|
||||
{Le: 1, Count: 1},
|
||||
{Le: 2, Count: 3},
|
||||
|
||||
{Le: 4, Count: 3},
|
||||
|
||||
{Le: 8, Count: 4},
|
||||
{Le: 16, Count: 5},
|
||||
},
|
||||
},
|
||||
{
|
||||
hist: SparseHistogram{
|
||||
Schema: 0,
|
||||
PositiveSpans: []Span{
|
||||
{Offset: 0, Length: 5},
|
||||
{Offset: 1, Length: 1},
|
||||
},
|
||||
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0},
|
||||
},
|
||||
expBuckets: []Bucket{
|
||||
{Le: 1, Count: 1},
|
||||
{Le: 2, Count: 4},
|
||||
{Le: 4, Count: 5},
|
||||
{Le: 8, Count: 7},
|
||||
|
||||
{Le: 16, Count: 8},
|
||||
|
||||
{Le: 32, Count: 8},
|
||||
{Le: 64, Count: 9},
|
||||
},
|
||||
},
|
||||
{
|
||||
hist: SparseHistogram{
|
||||
Schema: 0,
|
||||
PositiveSpans: []Span{
|
||||
{Offset: 0, Length: 7},
|
||||
},
|
||||
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 0},
|
||||
},
|
||||
expBuckets: []Bucket{
|
||||
{Le: 1, Count: 1},
|
||||
{Le: 2, Count: 4},
|
||||
{Le: 4, Count: 5},
|
||||
{Le: 8, Count: 7},
|
||||
{Le: 16, Count: 8},
|
||||
{Le: 32, Count: 9},
|
||||
{Le: 64, Count: 10},
|
||||
},
|
||||
},
|
||||
{
|
||||
hist: SparseHistogram{
|
||||
Schema: 3,
|
||||
PositiveSpans: []Span{
|
||||
{Offset: -5, Length: 2}, // -5 -4
|
||||
{Offset: 2, Length: 3}, // -1 0 1
|
||||
{Offset: 2, Length: 2}, // 4 5
|
||||
},
|
||||
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 3},
|
||||
},
|
||||
expBuckets: []Bucket{
|
||||
{Le: 0.6484197773255048, Count: 1}, // -5
|
||||
{Le: 0.7071067811865475, Count: 4}, // -4
|
||||
|
||||
{Le: 0.7711054127039704, Count: 4}, // -3
|
||||
{Le: 0.8408964152537144, Count: 4}, // -2
|
||||
|
||||
{Le: 0.9170040432046711, Count: 5}, // -1
|
||||
{Le: 1, Count: 7}, // 1
|
||||
{Le: 1.0905077326652577, Count: 8}, // 0
|
||||
|
||||
{Le: 1.189207115002721, Count: 8}, // 1
|
||||
{Le: 1.2968395546510096, Count: 8}, // 2
|
||||
|
||||
{Le: 1.414213562373095, Count: 9}, // 3
|
||||
{Le: 1.5422108254079407, Count: 13}, // 4
|
||||
},
|
||||
},
|
||||
{
|
||||
hist: SparseHistogram{
|
||||
Schema: -2,
|
||||
PositiveSpans: []Span{
|
||||
{Offset: -2, Length: 4}, // -2 -1 0 1
|
||||
{Offset: 2, Length: 2}, // 4 5
|
||||
},
|
||||
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0},
|
||||
},
|
||||
expBuckets: []Bucket{
|
||||
{Le: 0.00390625, Count: 1}, // -2
|
||||
{Le: 0.0625, Count: 4}, // -1
|
||||
{Le: 1, Count: 5}, // 0
|
||||
{Le: 16, Count: 7}, // 1
|
||||
|
||||
{Le: 256, Count: 7}, // 2
|
||||
{Le: 4096, Count: 7}, // 3
|
||||
|
||||
{Le: 65536, Count: 8}, // 4
|
||||
{Le: 1048576, Count: 9}, // 5
|
||||
},
|
||||
},
|
||||
{
|
||||
hist: SparseHistogram{
|
||||
Schema: -1,
|
||||
PositiveSpans: []Span{
|
||||
{Offset: -2, Length: 5}, // -2 -1 0 1 2
|
||||
},
|
||||
PositiveBuckets: []int64{1, 2, -2, 1, -1},
|
||||
},
|
||||
expBuckets: []Bucket{
|
||||
{Le: 0.0625, Count: 1}, // -2
|
||||
{Le: 0.25, Count: 4}, // -1
|
||||
{Le: 1, Count: 5}, // 0
|
||||
{Le: 4, Count: 7}, // 1
|
||||
{Le: 16, Count: 8}, // 2
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for i, c := range cases {
|
||||
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
|
||||
it := CumulativeExpandSparseHistogram(c.hist)
|
||||
actBuckets := make([]Bucket, 0, len(c.expBuckets))
|
||||
for it.Next() {
|
||||
actBuckets = append(actBuckets, it.At())
|
||||
}
|
||||
require.NoError(t, it.Err())
|
||||
require.Equal(t, c.expBuckets, actBuckets)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -16,8 +16,8 @@ package textparse
|
|||
import (
|
||||
"mime"
|
||||
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
)
|
||||
|
||||
|
|
@ -29,9 +29,8 @@ type Parser interface {
|
|||
Series() ([]byte, *int64, float64)
|
||||
|
||||
// Histogram returns the bytes of a series with a sparse histogram as a
|
||||
// value, the timestamp if set, and the sparse histogram in the current
|
||||
// sample.
|
||||
Histogram() ([]byte, *int64, histogram.SparseHistogram)
|
||||
// value, the timestamp if set, and the histogram in the current sample.
|
||||
Histogram() ([]byte, *int64, histogram.Histogram)
|
||||
|
||||
// Help returns the metric name and help text in the current entry.
|
||||
// Must only be called after Next returned a help entry.
|
||||
|
|
|
|||
|
|
@ -27,8 +27,8 @@ import (
|
|||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/pkg/value"
|
||||
)
|
||||
|
|
@ -114,10 +114,10 @@ func (p *OpenMetricsParser) Series() ([]byte, *int64, float64) {
|
|||
return p.series, nil, p.val
|
||||
}
|
||||
|
||||
// Histogram always returns (nil, nil, SparseHistogram{}) because OpenMetrics
|
||||
// does not support sparse histograms.
|
||||
func (p *OpenMetricsParser) Histogram() ([]byte, *int64, histogram.SparseHistogram) {
|
||||
return nil, nil, histogram.SparseHistogram{}
|
||||
// Histogram always returns (nil, nil, histogram.Histogram{}) because
|
||||
// OpenMetrics does not support sparse histograms.
|
||||
func (p *OpenMetricsParser) Histogram() ([]byte, *int64, histogram.Histogram) {
|
||||
return nil, nil, histogram.Histogram{}
|
||||
}
|
||||
|
||||
// Help returns the metric name and help text in the current entry.
|
||||
|
|
|
|||
|
|
@ -28,8 +28,8 @@ import (
|
|||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/pkg/value"
|
||||
)
|
||||
|
|
@ -169,10 +169,10 @@ func (p *PromParser) Series() ([]byte, *int64, float64) {
|
|||
return p.series, nil, p.val
|
||||
}
|
||||
|
||||
// Histogram always returns (nil, nil, SparseHistogram{}) because the Prometheus
|
||||
// text format does not support sparse histograms.
|
||||
func (p *PromParser) Histogram() ([]byte, *int64, histogram.SparseHistogram) {
|
||||
return nil, nil, histogram.SparseHistogram{}
|
||||
// Histogram always returns (nil, nil, histogram.Histogram{}) because the
|
||||
// Prometheus text format does not support sparse histograms.
|
||||
func (p *PromParser) Histogram() ([]byte, *int64, histogram.Histogram) {
|
||||
return nil, nil, histogram.Histogram{}
|
||||
}
|
||||
|
||||
// Help returns the metric name and help text in the current entry.
|
||||
|
|
|
|||
|
|
@ -27,8 +27,8 @@ import (
|
|||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/prometheus/common/model"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
|
||||
dto "github.com/prometheus/prometheus/prompb/io/prometheus/client"
|
||||
|
|
@ -65,6 +65,7 @@ type ProtobufParser struct {
|
|||
metricBytes *bytes.Buffer // A somewhat fluid representation of the current metric.
|
||||
}
|
||||
|
||||
// NewProtobufParser returns a parser for the payload in the byte slice.
|
||||
func NewProtobufParser(b []byte) Parser {
|
||||
return &ProtobufParser{
|
||||
in: b,
|
||||
|
|
@ -134,13 +135,13 @@ func (p *ProtobufParser) Series() ([]byte, *int64, float64) {
|
|||
// Histogram returns the bytes of a series with a sparse histogram as a
|
||||
// value, the timestamp if set, and the sparse histogram in the current
|
||||
// sample.
|
||||
func (p *ProtobufParser) Histogram() ([]byte, *int64, histogram.SparseHistogram) {
|
||||
func (p *ProtobufParser) Histogram() ([]byte, *int64, histogram.Histogram) {
|
||||
var (
|
||||
m = p.mf.GetMetric()[p.metricPos]
|
||||
ts = m.GetTimestampMs()
|
||||
h = m.GetHistogram()
|
||||
)
|
||||
sh := histogram.SparseHistogram{
|
||||
sh := histogram.Histogram{
|
||||
Count: h.GetSampleCount(),
|
||||
Sum: h.GetSampleSum(),
|
||||
ZeroThreshold: h.GetSbZeroThreshold(),
|
||||
|
|
|
|||
|
|
@ -22,8 +22,8 @@ import (
|
|||
"github.com/gogo/protobuf/proto"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
|
||||
dto "github.com/prometheus/prometheus/prompb/io/prometheus/client"
|
||||
|
|
@ -266,7 +266,7 @@ metric: <
|
|||
help string
|
||||
unit string
|
||||
comment string
|
||||
shs histogram.SparseHistogram
|
||||
shs histogram.Histogram
|
||||
e []exemplar.Exemplar
|
||||
}{
|
||||
{
|
||||
|
|
@ -332,7 +332,7 @@ metric: <
|
|||
{
|
||||
m: "test_histogram",
|
||||
t: 1234568,
|
||||
shs: histogram.SparseHistogram{
|
||||
shs: histogram.Histogram{
|
||||
Count: 175,
|
||||
ZeroCount: 2,
|
||||
Sum: 0.0008280461746287094,
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ import (
|
|||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/promql/parser"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
|
|
@ -296,8 +296,11 @@ func (ssi *storageSeriesIterator) At() (t int64, v float64) {
|
|||
return p.T, p.V
|
||||
}
|
||||
|
||||
func (ssi *storageSeriesIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
return 0, histogram.SparseHistogram{}
|
||||
// AtHistogram always returns (0, histogram.Histogram{}) because there is no
|
||||
// support for histogram values yet.
|
||||
// TODO(beorn7): Fix that for histogram support in PromQL.
|
||||
func (ssi *storageSeriesIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
return 0, histogram.Histogram{}
|
||||
}
|
||||
|
||||
func (ssi *storageSeriesIterator) ChunkEncoding() chunkenc.Encoding {
|
||||
|
|
|
|||
|
|
@ -17,8 +17,8 @@ import (
|
|||
"context"
|
||||
"math/rand"
|
||||
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
)
|
||||
|
|
@ -35,7 +35,7 @@ func (a nopAppender) Append(uint64, labels.Labels, int64, float64) (uint64, erro
|
|||
func (a nopAppender) AppendExemplar(uint64, labels.Labels, exemplar.Exemplar) (uint64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
func (a nopAppender) AppendHistogram(uint64, labels.Labels, int64, histogram.SparseHistogram) (uint64, error) {
|
||||
func (a nopAppender) AppendHistogram(uint64, labels.Labels, int64, histogram.Histogram) (uint64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
func (a nopAppender) Commit() error { return nil }
|
||||
|
|
@ -47,9 +47,9 @@ type sample struct {
|
|||
v float64
|
||||
}
|
||||
|
||||
type hist struct {
|
||||
h histogram.SparseHistogram
|
||||
type histogramSample struct {
|
||||
t int64
|
||||
h histogram.Histogram
|
||||
}
|
||||
|
||||
// collectResultAppender records all samples that were added through the appender.
|
||||
|
|
@ -61,9 +61,9 @@ type collectResultAppender struct {
|
|||
rolledbackResult []sample
|
||||
pendingExemplars []exemplar.Exemplar
|
||||
resultExemplars []exemplar.Exemplar
|
||||
resultHistograms []hist
|
||||
pendingHistograms []hist
|
||||
rolledbackHistograms []hist
|
||||
resultHistograms []histogramSample
|
||||
pendingHistograms []histogramSample
|
||||
rolledbackHistograms []histogramSample
|
||||
}
|
||||
|
||||
func (a *collectResultAppender) Append(ref uint64, lset labels.Labels, t int64, v float64) (uint64, error) {
|
||||
|
|
@ -96,13 +96,13 @@ func (a *collectResultAppender) AppendExemplar(ref uint64, l labels.Labels, e ex
|
|||
return a.next.AppendExemplar(ref, l, e)
|
||||
}
|
||||
|
||||
func (a *collectResultAppender) AppendHistogram(ref uint64, l labels.Labels, t int64, sh histogram.SparseHistogram) (uint64, error) {
|
||||
a.pendingHistograms = append(a.pendingHistograms, hist{h: sh, t: t})
|
||||
func (a *collectResultAppender) AppendHistogram(ref uint64, l labels.Labels, t int64, h histogram.Histogram) (uint64, error) {
|
||||
a.pendingHistograms = append(a.pendingHistograms, histogramSample{h: h, t: t})
|
||||
if a.next == nil {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
return a.next.AppendHistogram(ref, l, t, sh)
|
||||
return a.next.AppendHistogram(ref, l, t, h)
|
||||
}
|
||||
|
||||
func (a *collectResultAppender) Commit() error {
|
||||
|
|
|
|||
|
|
@ -39,8 +39,8 @@ import (
|
|||
|
||||
"github.com/prometheus/prometheus/config"
|
||||
"github.com/prometheus/prometheus/discovery/targetgroup"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/pkg/pool"
|
||||
"github.com/prometheus/prometheus/pkg/relabel"
|
||||
|
|
@ -1411,7 +1411,7 @@ loop:
|
|||
met []byte
|
||||
parsedTimestamp *int64
|
||||
val float64
|
||||
his histogram.SparseHistogram
|
||||
h histogram.Histogram
|
||||
)
|
||||
if et, err = p.Next(); err != nil {
|
||||
if err == io.EOF {
|
||||
|
|
@ -1439,7 +1439,7 @@ loop:
|
|||
|
||||
t := defTime
|
||||
if isHistogram {
|
||||
met, parsedTimestamp, his = p.Histogram()
|
||||
met, parsedTimestamp, h = p.Histogram()
|
||||
} else {
|
||||
met, parsedTimestamp, val = p.Series()
|
||||
}
|
||||
|
|
@ -1491,7 +1491,7 @@ loop:
|
|||
}
|
||||
|
||||
if isHistogram {
|
||||
ref, err = app.AppendHistogram(ref, lset, t, his)
|
||||
ref, err = app.AppendHistogram(ref, lset, t, h)
|
||||
} else {
|
||||
ref, err = app.Append(ref, lset, t, val)
|
||||
}
|
||||
|
|
@ -1554,7 +1554,6 @@ loop:
|
|||
if err == nil {
|
||||
sl.cache.forEachStale(func(lset labels.Labels) bool {
|
||||
// Series no longer exposed, mark it stale.
|
||||
// TODO(beorn7): Appending staleness markers breaks horribly for histograms.
|
||||
_, err = app.Append(0, lset, defTime, math.Float64frombits(value.StaleNaN))
|
||||
switch errors.Cause(err) {
|
||||
case storage.ErrOutOfOrderSample, storage.ErrDuplicateSampleForTimestamp:
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ package storage
|
|||
import (
|
||||
"math"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
)
|
||||
|
||||
|
|
@ -198,8 +198,11 @@ func (it *sampleRingIterator) At() (int64, float64) {
|
|||
return it.r.at(it.i)
|
||||
}
|
||||
|
||||
func (it *sampleRingIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
return 0, histogram.SparseHistogram{}
|
||||
// AtHistogram always returns (0, histogram.Histogram{}) because there is no
|
||||
// support for histogram values yet.
|
||||
// TODO(beorn7): Fix that for histogram support in PromQL.
|
||||
func (it *sampleRingIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
return 0, histogram.Histogram{}
|
||||
}
|
||||
|
||||
func (it *sampleRingIterator) ChunkEncoding() chunkenc.Encoding {
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ import (
|
|||
"math/rand"
|
||||
"testing"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
|
@ -196,8 +196,8 @@ type mockSeriesIterator struct {
|
|||
|
||||
func (m *mockSeriesIterator) Seek(t int64) bool { return m.seek(t) }
|
||||
func (m *mockSeriesIterator) At() (int64, float64) { return m.at() }
|
||||
func (m *mockSeriesIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
return 0, histogram.SparseHistogram{}
|
||||
func (m *mockSeriesIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
return 0, histogram.Histogram{}
|
||||
}
|
||||
func (m *mockSeriesIterator) ChunkEncoding() chunkenc.Encoding {
|
||||
return chunkenc.EncXOR
|
||||
|
|
@ -219,8 +219,8 @@ func (it *fakeSeriesIterator) At() (int64, float64) {
|
|||
return it.idx * it.step, 123 // value doesn't matter
|
||||
}
|
||||
|
||||
func (it *fakeSeriesIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
return it.idx * it.step, histogram.SparseHistogram{} // value doesn't matter
|
||||
func (it *fakeSeriesIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
return it.idx * it.step, histogram.Histogram{} // value doesn't matter
|
||||
}
|
||||
|
||||
func (it *fakeSeriesIterator) ChunkEncoding() chunkenc.Encoding {
|
||||
|
|
|
|||
|
|
@ -20,8 +20,8 @@ import (
|
|||
"github.com/go-kit/log/level"
|
||||
"github.com/prometheus/common/model"
|
||||
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
tsdb_errors "github.com/prometheus/prometheus/tsdb/errors"
|
||||
)
|
||||
|
|
@ -173,14 +173,14 @@ func (f *fanoutAppender) AppendExemplar(ref uint64, l labels.Labels, e exemplar.
|
|||
return ref, nil
|
||||
}
|
||||
|
||||
func (f *fanoutAppender) AppendHistogram(ref uint64, l labels.Labels, t int64, sh histogram.SparseHistogram) (uint64, error) {
|
||||
ref, err := f.primary.AppendHistogram(ref, l, t, sh)
|
||||
func (f *fanoutAppender) AppendHistogram(ref uint64, l labels.Labels, t int64, h histogram.Histogram) (uint64, error) {
|
||||
ref, err := f.primary.AppendHistogram(ref, l, t, h)
|
||||
if err != nil {
|
||||
return ref, err
|
||||
}
|
||||
|
||||
for _, appender := range f.secondaries {
|
||||
if _, err := appender.AppendHistogram(ref, l, t, sh); err != nil {
|
||||
if _, err := appender.AppendHistogram(ref, l, t, h); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,8 +18,8 @@ import (
|
|||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
"github.com/prometheus/prometheus/tsdb/chunks"
|
||||
|
|
@ -213,17 +213,16 @@ type ExemplarAppender interface {
|
|||
AppendExemplar(ref uint64, l labels.Labels, e exemplar.Exemplar) (uint64, error)
|
||||
}
|
||||
|
||||
// HistogramAppender provides an interface for adding sparse histogram to the Prometheus.
|
||||
// HistogramAppender provides an interface for appending histograms to the storage.
|
||||
type HistogramAppender interface {
|
||||
// AppendHistogram adds a sparse histogram for the given series labels.
|
||||
// An optional reference number can be provided to accelerate calls.
|
||||
// A reference number is returned which can be used to add further
|
||||
// histograms in the same or later transactions.
|
||||
// Returned reference numbers are ephemeral and may be rejected in calls
|
||||
// to Append() at any point. Adding the sample via Append() returns a new
|
||||
// reference number.
|
||||
// AppendHistogram adds a histogram for the given series labels. An
|
||||
// optional reference number can be provided to accelerate calls. A
|
||||
// reference number is returned which can be used to add further
|
||||
// histograms in the same or later transactions. Returned reference
|
||||
// numbers are ephemeral and may be rejected in calls to Append() at any
|
||||
// point. Adding the sample via Append() returns a new reference number.
|
||||
// If the reference is 0 it must not be used for caching.
|
||||
AppendHistogram(ref uint64, l labels.Labels, t int64, sh histogram.SparseHistogram) (uint64, error)
|
||||
AppendHistogram(ref uint64, l labels.Labels, t int64, h histogram.Histogram) (uint64, error)
|
||||
}
|
||||
|
||||
// SeriesSet contains a set of series.
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ import (
|
|||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
"github.com/prometheus/prometheus/tsdb/chunks"
|
||||
|
|
@ -465,7 +465,7 @@ func (c *chainSampleIterator) Seek(t int64) bool {
|
|||
}
|
||||
if len(c.h) > 0 {
|
||||
c.curr = heap.Pop(&c.h).(chunkenc.Iterator)
|
||||
if c.curr.ChunkEncoding() == chunkenc.EncSHS {
|
||||
if c.curr.ChunkEncoding() == chunkenc.EncHistogram {
|
||||
c.lastt, _ = c.curr.AtHistogram()
|
||||
} else {
|
||||
c.lastt, _ = c.curr.At()
|
||||
|
|
@ -483,7 +483,7 @@ func (c *chainSampleIterator) At() (t int64, v float64) {
|
|||
return c.curr.At()
|
||||
}
|
||||
|
||||
func (c *chainSampleIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
func (c *chainSampleIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
if c.curr == nil {
|
||||
panic("chainSampleIterator.AtHistogram() called before first .Next() or after .Next() returned false.")
|
||||
}
|
||||
|
|
@ -517,7 +517,7 @@ func (c *chainSampleIterator) Next() bool {
|
|||
var currt int64
|
||||
for {
|
||||
if c.curr.Next() {
|
||||
if c.curr.ChunkEncoding() == chunkenc.EncSHS {
|
||||
if c.curr.ChunkEncoding() == chunkenc.EncHistogram {
|
||||
currt, _ = c.curr.AtHistogram()
|
||||
} else {
|
||||
currt, _ = c.curr.At()
|
||||
|
|
@ -534,7 +534,7 @@ func (c *chainSampleIterator) Next() bool {
|
|||
|
||||
// Check current iterator with the top of the heap.
|
||||
var nextt int64
|
||||
if c.h[0].ChunkEncoding() == chunkenc.EncSHS {
|
||||
if c.h[0].ChunkEncoding() == chunkenc.EncHistogram {
|
||||
nextt, _ = c.h[0].AtHistogram()
|
||||
} else {
|
||||
nextt, _ = c.h[0].At()
|
||||
|
|
@ -552,7 +552,7 @@ func (c *chainSampleIterator) Next() bool {
|
|||
}
|
||||
|
||||
c.curr = heap.Pop(&c.h).(chunkenc.Iterator)
|
||||
if c.curr.ChunkEncoding() == chunkenc.EncSHS {
|
||||
if c.curr.ChunkEncoding() == chunkenc.EncHistogram {
|
||||
currt, _ = c.curr.AtHistogram()
|
||||
} else {
|
||||
currt, _ = c.curr.At()
|
||||
|
|
@ -581,12 +581,12 @@ func (h samplesIteratorHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
|
|||
|
||||
func (h samplesIteratorHeap) Less(i, j int) bool {
|
||||
var at, bt int64
|
||||
if h[i].ChunkEncoding() == chunkenc.EncSHS {
|
||||
if h[i].ChunkEncoding() == chunkenc.EncHistogram {
|
||||
at, _ = h[i].AtHistogram()
|
||||
} else {
|
||||
at, _ = h[i].At()
|
||||
}
|
||||
if h[j].ChunkEncoding() == chunkenc.EncSHS {
|
||||
if h[j].ChunkEncoding() == chunkenc.EncHistogram {
|
||||
bt, _ = h[j].AtHistogram()
|
||||
} else {
|
||||
bt, _ = h[j].At()
|
||||
|
|
|
|||
|
|
@ -26,8 +26,8 @@ import (
|
|||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/common/model"
|
||||
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/pkg/textparse"
|
||||
"github.com/prometheus/prometheus/prompb"
|
||||
|
|
@ -370,8 +370,11 @@ func (c *concreteSeriesIterator) At() (t int64, v float64) {
|
|||
return s.Timestamp, s.Value
|
||||
}
|
||||
|
||||
func (c *concreteSeriesIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
return 0, histogram.SparseHistogram{}
|
||||
// AtHistogram always returns (0, histogram.Histogram{}) because there is no
|
||||
// support for histogram values yet.
|
||||
// TODO(beorn7): Fix that for histogram support in remote storage.
|
||||
func (c *concreteSeriesIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
return 0, histogram.Histogram{}
|
||||
}
|
||||
|
||||
func (c *concreteSeriesIterator) ChunkEncoding() chunkenc.Encoding {
|
||||
|
|
|
|||
|
|
@ -24,8 +24,8 @@ import (
|
|||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
|
||||
"github.com/prometheus/prometheus/config"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
"github.com/prometheus/prometheus/tsdb/wal"
|
||||
|
|
@ -241,7 +241,7 @@ func (t *timestampTracker) AppendExemplar(_ uint64, _ labels.Labels, _ exemplar.
|
|||
return 0, nil
|
||||
}
|
||||
|
||||
func (t *timestampTracker) AppendHistogram(_ uint64, _ labels.Labels, ts int64, _ histogram.SparseHistogram) (uint64, error) {
|
||||
func (t *timestampTracker) AppendHistogram(_ uint64, _ labels.Labels, ts int64, _ histogram.Histogram) (uint64, error) {
|
||||
t.histograms++
|
||||
if ts > t.highestTimestamp {
|
||||
t.highestTimestamp = ts
|
||||
|
|
|
|||
|
|
@ -25,8 +25,8 @@ import (
|
|||
"github.com/go-kit/log"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/prompb"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
|
|
@ -188,7 +188,7 @@ func (m *mockAppendable) AppendExemplar(_ uint64, l labels.Labels, e exemplar.Ex
|
|||
return 0, nil
|
||||
}
|
||||
|
||||
func (*mockAppendable) AppendHistogram(ref uint64, l labels.Labels, t int64, sh histogram.SparseHistogram) (uint64, error) {
|
||||
// noop until we implement sparse histograms over remote write
|
||||
func (*mockAppendable) AppendHistogram(ref uint64, l labels.Labels, t int64, h histogram.Histogram) (uint64, error) {
|
||||
// TODO(beorn7): Noop until we implement sparse histograms over remote write.
|
||||
return 0, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ import (
|
|||
"math"
|
||||
"sort"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
"github.com/prometheus/prometheus/tsdb/chunks"
|
||||
|
|
@ -91,8 +91,10 @@ func (it *listSeriesIterator) At() (int64, float64) {
|
|||
return s.T(), s.V()
|
||||
}
|
||||
|
||||
func (it *listSeriesIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
return 0, histogram.SparseHistogram{}
|
||||
// AtHistogram always returns (0, histogram.Histogram{}) because there is no
|
||||
// support for histogram values yet.
|
||||
func (it *listSeriesIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
return 0, histogram.Histogram{}
|
||||
}
|
||||
|
||||
func (it *listSeriesIterator) ChunkEncoding() chunkenc.Encoding {
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import (
|
|||
"sync"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
)
|
||||
|
||||
// Encoding is the identifier for a chunk encoding.
|
||||
|
|
@ -30,8 +30,8 @@ func (e Encoding) String() string {
|
|||
return "none"
|
||||
case EncXOR:
|
||||
return "XOR"
|
||||
case EncSHS:
|
||||
return "SHS"
|
||||
case EncHistogram:
|
||||
return "histogram"
|
||||
}
|
||||
return "<unknown>"
|
||||
}
|
||||
|
|
@ -39,7 +39,7 @@ func (e Encoding) String() string {
|
|||
// IsValidEncoding returns true for supported encodings.
|
||||
func IsValidEncoding(e Encoding) bool {
|
||||
switch e {
|
||||
case EncXOR, EncSHS:
|
||||
case EncXOR, EncHistogram:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
|
|
@ -49,7 +49,7 @@ func IsValidEncoding(e Encoding) bool {
|
|||
const (
|
||||
EncNone Encoding = iota
|
||||
EncXOR
|
||||
EncSHS
|
||||
EncHistogram
|
||||
)
|
||||
|
||||
// Chunk holds a sequence of sample pairs that can be iterated over and appended to.
|
||||
|
|
@ -82,7 +82,7 @@ type Chunk interface {
|
|||
// Appender adds sample pairs to a chunk.
|
||||
type Appender interface {
|
||||
Append(int64, float64)
|
||||
AppendHistogram(t int64, h histogram.SparseHistogram)
|
||||
AppendHistogram(t int64, h histogram.Histogram)
|
||||
}
|
||||
|
||||
// Iterator is a simple iterator that can only get the next value.
|
||||
|
|
@ -100,7 +100,7 @@ type Iterator interface {
|
|||
At() (int64, float64)
|
||||
// AtHistogram returns the current timestamp/histogram pair.
|
||||
// Before the iterator has advanced AtHistogram behaviour is unspecified.
|
||||
AtHistogram() (int64, histogram.SparseHistogram)
|
||||
AtHistogram() (int64, histogram.Histogram)
|
||||
// Err returns the current error. It should be used only after iterator is
|
||||
// exhausted, that is `Next` or `Seek` returns false.
|
||||
Err() error
|
||||
|
|
@ -117,8 +117,8 @@ type nopIterator struct{}
|
|||
|
||||
func (nopIterator) Seek(int64) bool { return false }
|
||||
func (nopIterator) At() (int64, float64) { return math.MinInt64, 0 }
|
||||
func (nopIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
return math.MinInt64, histogram.SparseHistogram{}
|
||||
func (nopIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
return math.MinInt64, histogram.Histogram{}
|
||||
}
|
||||
func (nopIterator) Next() bool { return false }
|
||||
func (nopIterator) Err() error { return nil }
|
||||
|
|
@ -132,8 +132,8 @@ type Pool interface {
|
|||
|
||||
// pool is a memory pool of chunk objects.
|
||||
type pool struct {
|
||||
xor sync.Pool
|
||||
shs sync.Pool
|
||||
xor sync.Pool
|
||||
histogram sync.Pool
|
||||
}
|
||||
|
||||
// NewPool returns a new pool.
|
||||
|
|
@ -144,9 +144,9 @@ func NewPool() Pool {
|
|||
return &XORChunk{b: bstream{}}
|
||||
},
|
||||
},
|
||||
shs: sync.Pool{
|
||||
histogram: sync.Pool{
|
||||
New: func() interface{} {
|
||||
return &HistoChunk{b: bstream{}}
|
||||
return &HistogramChunk{b: bstream{}}
|
||||
},
|
||||
},
|
||||
}
|
||||
|
|
@ -159,9 +159,9 @@ func (p *pool) Get(e Encoding, b []byte) (Chunk, error) {
|
|||
c.b.stream = b
|
||||
c.b.count = 0
|
||||
return c, nil
|
||||
case EncSHS:
|
||||
case EncHistogram:
|
||||
// TODO: update metadata
|
||||
c := p.shs.Get().(*HistoChunk)
|
||||
c := p.histogram.Get().(*HistogramChunk)
|
||||
c.b.stream = b
|
||||
c.b.count = 0
|
||||
return c, nil
|
||||
|
|
@ -182,9 +182,9 @@ func (p *pool) Put(c Chunk) error {
|
|||
xc.b.stream = nil
|
||||
xc.b.count = 0
|
||||
p.xor.Put(c)
|
||||
case EncSHS:
|
||||
case EncHistogram:
|
||||
// TODO: update metadata
|
||||
sh, ok := c.(*HistoChunk)
|
||||
sh, ok := c.(*HistogramChunk)
|
||||
// This may happen often with wrapped chunks. Nothing we can really do about
|
||||
// it but returning an error would cause a lot of allocations again. Thus,
|
||||
// we just skip it.
|
||||
|
|
@ -193,7 +193,7 @@ func (p *pool) Put(c Chunk) error {
|
|||
}
|
||||
sh.b.stream = nil
|
||||
sh.b.count = 0
|
||||
p.shs.Put(c)
|
||||
p.histogram.Put(c)
|
||||
default:
|
||||
return errors.Errorf("invalid chunk encoding %q", c.Encoding())
|
||||
}
|
||||
|
|
@ -207,9 +207,9 @@ func FromData(e Encoding, d []byte) (Chunk, error) {
|
|||
switch e {
|
||||
case EncXOR:
|
||||
return &XORChunk{b: bstream{count: 0, stream: d}}, nil
|
||||
case EncSHS:
|
||||
case EncHistogram:
|
||||
// TODO: update metadata
|
||||
return &HistoChunk{b: bstream{count: 0, stream: d}}, nil
|
||||
return &HistogramChunk{b: bstream{count: 0, stream: d}}, nil
|
||||
}
|
||||
return nil, errors.Errorf("invalid chunk encoding %q", e)
|
||||
}
|
||||
|
|
@ -219,8 +219,8 @@ func NewEmptyChunk(e Encoding) (Chunk, error) {
|
|||
switch e {
|
||||
case EncXOR:
|
||||
return NewXORChunk(), nil
|
||||
case EncSHS:
|
||||
return NewHistoChunk(), nil
|
||||
case EncHistogram:
|
||||
return NewHistogramChunk(), nil
|
||||
}
|
||||
return nil, errors.Errorf("invalid chunk encoding %q", e)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,943 +0,0 @@
|
|||
// Copyright 2021 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// The code in this file was largely written by Damian Gryski as part of
|
||||
// https://github.com/dgryski/go-tsz and published under the license below.
|
||||
// It was modified to accommodate reading from byte slices without modifying
|
||||
// the underlying bytes, which would panic when reading from mmap'd
|
||||
// read-only byte slices.
|
||||
|
||||
// Copyright (c) 2015,2016 Damian Gryski <damian@gryski.com>
|
||||
// All rights reserved.
|
||||
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
|
||||
// * Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimer.
|
||||
//
|
||||
// * Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
package chunkenc
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"math"
|
||||
"math/bits"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/value"
|
||||
)
|
||||
|
||||
const ()
|
||||
|
||||
// HistoChunk holds sparse histogram encoded sample data.
|
||||
// Appends a histogram sample
|
||||
// * schema defines the resolution (number of buckets per power of 2)
|
||||
// Currently, valid numbers are -4 <= n <= 8.
|
||||
// They are all for base-2 bucket schemas, where 1 is a bucket boundary in each case, and
|
||||
// then each power of two is divided into 2^n logarithmic buckets.
|
||||
// Or in other words, each bucket boundary is the previous boundary times 2^(2^-n).
|
||||
// In the future, more bucket schemas may be added using numbers < -4 or > 8.
|
||||
// The bucket with upper boundary of 1 is always bucket 0.
|
||||
// Then negative numbers for smaller boundaries and positive for uppers.
|
||||
//
|
||||
// fields are stored like so:
|
||||
// field ts count zeroCount sum []posbuckets negbuckets
|
||||
// observation 1 raw raw raw raw []raw []raw
|
||||
// observation 2 delta delta delta xor []delta []delta
|
||||
// observation >2 dod dod dod xor []dod []dod
|
||||
// TODO zerothreshold
|
||||
type HistoChunk struct {
|
||||
b bstream
|
||||
}
|
||||
|
||||
// NewHistoChunk returns a new chunk with Histo encoding of the given size.
|
||||
func NewHistoChunk() *HistoChunk {
|
||||
b := make([]byte, 3, 128)
|
||||
return &HistoChunk{b: bstream{stream: b, count: 0}}
|
||||
}
|
||||
|
||||
// Encoding returns the encoding type.
|
||||
func (c *HistoChunk) Encoding() Encoding {
|
||||
return EncSHS
|
||||
}
|
||||
|
||||
// Bytes returns the underlying byte slice of the chunk.
|
||||
func (c *HistoChunk) Bytes() []byte {
|
||||
return c.b.bytes()
|
||||
}
|
||||
|
||||
// NumSamples returns the number of samples in the chunk.
|
||||
func (c *HistoChunk) NumSamples() int {
|
||||
return int(binary.BigEndian.Uint16(c.Bytes()))
|
||||
}
|
||||
|
||||
// Meta returns the histogram metadata.
|
||||
// callers may only call this on chunks that have at least one sample
|
||||
func (c *HistoChunk) Meta() (int32, float64, []histogram.Span, []histogram.Span, error) {
|
||||
if c.NumSamples() == 0 {
|
||||
panic("HistoChunk.Meta() called on an empty chunk")
|
||||
}
|
||||
b := newBReader(c.Bytes()[2:])
|
||||
return readHistoChunkMeta(&b)
|
||||
}
|
||||
|
||||
// CounterResetHeader defines the first 2 bits of the chunk header.
|
||||
type CounterResetHeader byte
|
||||
|
||||
const (
|
||||
CounterReset CounterResetHeader = 0b10000000
|
||||
NotCounterReset CounterResetHeader = 0b01000000
|
||||
GaugeType CounterResetHeader = 0b11000000
|
||||
UnknownCounterReset CounterResetHeader = 0b00000000
|
||||
)
|
||||
|
||||
// SetCounterResetHeader sets the counter reset header.
|
||||
func (c *HistoChunk) SetCounterResetHeader(h CounterResetHeader) {
|
||||
switch h {
|
||||
case CounterReset, NotCounterReset, GaugeType, UnknownCounterReset:
|
||||
bytes := c.Bytes()
|
||||
bytes[2] = (bytes[2] & 0b00111111) | byte(h)
|
||||
default:
|
||||
panic("invalid CounterResetHeader type")
|
||||
}
|
||||
}
|
||||
|
||||
// GetCounterResetHeader returns the info about the first 2 bits of the chunk header.
|
||||
func (c *HistoChunk) GetCounterResetHeader() CounterResetHeader {
|
||||
return CounterResetHeader(c.Bytes()[2] & 0b11000000)
|
||||
}
|
||||
|
||||
// Compact implements the Chunk interface.
|
||||
func (c *HistoChunk) Compact() {
|
||||
if l := len(c.b.stream); cap(c.b.stream) > l+chunkCompactCapacityThreshold {
|
||||
buf := make([]byte, l)
|
||||
copy(buf, c.b.stream)
|
||||
c.b.stream = buf
|
||||
}
|
||||
}
|
||||
|
||||
// Appender implements the Chunk interface.
|
||||
func (c *HistoChunk) Appender() (Appender, error) {
|
||||
it := c.iterator(nil)
|
||||
|
||||
// To get an appender we must know the state it would have if we had
|
||||
// appended all existing data from scratch.
|
||||
// We iterate through the end and populate via the iterator's state.
|
||||
for it.Next() {
|
||||
}
|
||||
if err := it.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
a := &HistoAppender{
|
||||
b: &c.b,
|
||||
|
||||
schema: it.schema,
|
||||
zeroThreshold: it.zeroThreshold,
|
||||
posSpans: it.posSpans,
|
||||
negSpans: it.negSpans,
|
||||
t: it.t,
|
||||
cnt: it.cnt,
|
||||
zcnt: it.zcnt,
|
||||
tDelta: it.tDelta,
|
||||
cntDelta: it.cntDelta,
|
||||
zcntDelta: it.zcntDelta,
|
||||
posbuckets: it.posbuckets,
|
||||
negbuckets: it.negbuckets,
|
||||
posbucketsDelta: it.posbucketsDelta,
|
||||
negbucketsDelta: it.negbucketsDelta,
|
||||
|
||||
sum: it.sum,
|
||||
leading: it.leading,
|
||||
trailing: it.trailing,
|
||||
|
||||
buf64: make([]byte, binary.MaxVarintLen64),
|
||||
}
|
||||
if binary.BigEndian.Uint16(a.b.bytes()) == 0 {
|
||||
a.leading = 0xff
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func countSpans(spans []histogram.Span) int {
|
||||
var cnt int
|
||||
for _, s := range spans {
|
||||
cnt += int(s.Length)
|
||||
}
|
||||
return cnt
|
||||
}
|
||||
|
||||
func newHistoIterator(b []byte) *histoIterator {
|
||||
it := &histoIterator{
|
||||
br: newBReader(b),
|
||||
numTotal: binary.BigEndian.Uint16(b),
|
||||
t: math.MinInt64,
|
||||
}
|
||||
// The first 2 bytes contain chunk headers.
|
||||
// We skip that for actual samples.
|
||||
_, _ = it.br.readBits(16)
|
||||
return it
|
||||
}
|
||||
|
||||
func (c *HistoChunk) iterator(it Iterator) *histoIterator {
|
||||
// TODO fix this. this is taken from xor.go // dieter not sure what the purpose of this is
|
||||
// Should iterators guarantee to act on a copy of the data so it doesn't lock append?
|
||||
// When using striped locks to guard access to chunks, probably yes.
|
||||
// Could only copy data if the chunk is not completed yet.
|
||||
//if histoIter, ok := it.(*histoIterator); ok {
|
||||
// histoIter.Reset(c.b.bytes())
|
||||
// return histoIter
|
||||
//}
|
||||
return newHistoIterator(c.b.bytes())
|
||||
}
|
||||
|
||||
// Iterator implements the Chunk interface.
|
||||
func (c *HistoChunk) Iterator(it Iterator) Iterator {
|
||||
return c.iterator(it)
|
||||
}
|
||||
|
||||
// HistoAppender is an Appender implementation for sparse histograms.
|
||||
type HistoAppender struct {
|
||||
b *bstream
|
||||
|
||||
// Metadata:
|
||||
schema int32
|
||||
zeroThreshold float64
|
||||
posSpans, negSpans []histogram.Span
|
||||
|
||||
// For the fields that are tracked as dod's. Note that we expect to
|
||||
// handle negative deltas (e.g. resets) by creating new chunks, we still
|
||||
// want to support it in general hence signed integer types.
|
||||
t int64
|
||||
cnt, zcnt uint64
|
||||
tDelta, cntDelta, zcntDelta int64
|
||||
|
||||
posbuckets, negbuckets []int64
|
||||
posbucketsDelta, negbucketsDelta []int64
|
||||
|
||||
// The sum is Gorilla xor encoded.
|
||||
sum float64
|
||||
leading uint8
|
||||
trailing uint8
|
||||
|
||||
buf64 []byte // For working on varint64's.
|
||||
}
|
||||
|
||||
func putVarint(b *bstream, buf []byte, x int64) {
|
||||
for _, byt := range buf[:binary.PutVarint(buf, x)] {
|
||||
b.writeByte(byt)
|
||||
}
|
||||
}
|
||||
|
||||
func putUvarint(b *bstream, buf []byte, x uint64) {
|
||||
for _, byt := range buf[:binary.PutUvarint(buf, x)] {
|
||||
b.writeByte(byt)
|
||||
}
|
||||
}
|
||||
|
||||
// Append implements Appender. This implementation does nothing for now.
|
||||
// TODO(beorn7): Implement in a meaningful way, i.e. we need to support
|
||||
// appending of stale markers, but this should never be used for "real"
|
||||
// samples.
|
||||
func (a *HistoAppender) Append(int64, float64) {}
|
||||
|
||||
// Appendable returns whether the chunk can be appended to, and if so
|
||||
// whether any recoding needs to happen using the provided interjections
|
||||
// (in case of any new buckets, positive or negative range, respectively)
|
||||
// The chunk is not appendable if:
|
||||
// * the schema has changed
|
||||
// * the zerobucket threshold has changed
|
||||
// * any buckets disappeared
|
||||
// * there was a counter reset in the count of observations or in any bucket, including the zero bucket
|
||||
// * the last sample in the chunk was stale while the current sample is not stale
|
||||
// It returns an additional boolean set to true if it is not appendable because of a counter reset.
|
||||
// If the given sample is stale, it will always return true.
|
||||
// If counterReset is true, okToAppend MUST be false.
|
||||
func (a *HistoAppender) Appendable(h histogram.SparseHistogram) (posInterjections []Interjection, negInterjections []Interjection, okToAppend bool, counterReset bool) {
|
||||
if value.IsStaleNaN(h.Sum) {
|
||||
// This is a stale sample whose buckets and spans don't matter.
|
||||
okToAppend = true
|
||||
return
|
||||
}
|
||||
if value.IsStaleNaN(a.sum) {
|
||||
// If the last sample was stale, then we can only accept stale samples in this chunk.
|
||||
return
|
||||
}
|
||||
|
||||
if h.Count < a.cnt {
|
||||
// There has been a counter reset.
|
||||
counterReset = true
|
||||
return
|
||||
}
|
||||
|
||||
if h.Schema != a.schema || h.ZeroThreshold != a.zeroThreshold {
|
||||
return
|
||||
}
|
||||
|
||||
if h.ZeroCount < a.zcnt {
|
||||
// There has been a counter reset since ZeroThreshold didn't change.
|
||||
counterReset = true
|
||||
return
|
||||
}
|
||||
|
||||
var ok bool
|
||||
posInterjections, ok = compareSpans(a.posSpans, h.PositiveSpans)
|
||||
if !ok {
|
||||
counterReset = true
|
||||
return
|
||||
}
|
||||
negInterjections, ok = compareSpans(a.negSpans, h.NegativeSpans)
|
||||
if !ok {
|
||||
counterReset = true
|
||||
return
|
||||
}
|
||||
|
||||
if counterResetInAnyBucket(a.posbuckets, h.PositiveBuckets, a.posSpans, h.PositiveSpans) ||
|
||||
counterResetInAnyBucket(a.negbuckets, h.NegativeBuckets, a.negSpans, h.NegativeSpans) {
|
||||
counterReset, posInterjections, negInterjections = true, nil, nil
|
||||
return
|
||||
}
|
||||
|
||||
okToAppend = true
|
||||
return
|
||||
}
|
||||
|
||||
// counterResetInAnyBucket returns true if there was a counter reset for any bucket.
|
||||
// This should be called only when buckets are same or new buckets were added,
|
||||
// and does not handle the case of buckets missing.
|
||||
func counterResetInAnyBucket(oldBuckets, newBuckets []int64, oldSpans, newSpans []histogram.Span) bool {
|
||||
if len(oldSpans) == 0 || len(oldBuckets) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
oldSpanSliceIdx, newSpanSliceIdx := 0, 0 // Index for the span slices.
|
||||
oldInsideSpanIdx, newInsideSpanIdx := uint32(0), uint32(0) // Index inside a span.
|
||||
oldIdx, newIdx := oldSpans[0].Offset, newSpans[0].Offset
|
||||
|
||||
oldBucketSliceIdx, newBucketSliceIdx := 0, 0 // Index inside bucket slice.
|
||||
oldVal, newVal := oldBuckets[0], newBuckets[0]
|
||||
|
||||
// Since we assume that new spans won't have missing buckets, there will never be a case
|
||||
// where the old index will not find a matching new index.
|
||||
for {
|
||||
if oldIdx == newIdx {
|
||||
if newVal < oldVal {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
if oldIdx <= newIdx {
|
||||
// Moving ahead old bucket and span by 1 index.
|
||||
if oldInsideSpanIdx == oldSpans[oldSpanSliceIdx].Length-1 {
|
||||
// Current span is over.
|
||||
oldSpanSliceIdx++
|
||||
oldInsideSpanIdx = 0
|
||||
if oldSpanSliceIdx >= len(oldSpans) {
|
||||
// All old spans are over.
|
||||
break
|
||||
}
|
||||
oldIdx += 1 + oldSpans[oldSpanSliceIdx].Offset
|
||||
} else {
|
||||
oldInsideSpanIdx++
|
||||
oldIdx++
|
||||
}
|
||||
oldBucketSliceIdx++
|
||||
oldVal += oldBuckets[oldBucketSliceIdx]
|
||||
}
|
||||
|
||||
if oldIdx > newIdx {
|
||||
// Moving ahead new bucket and span by 1 index.
|
||||
if newInsideSpanIdx == newSpans[newSpanSliceIdx].Length-1 {
|
||||
// Current span is over.
|
||||
newSpanSliceIdx++
|
||||
newInsideSpanIdx = 0
|
||||
if newSpanSliceIdx >= len(newSpans) {
|
||||
// All new spans are over.
|
||||
// This should not happen, old spans above should catch this first.
|
||||
panic("new spans over before old spans in counterReset")
|
||||
}
|
||||
newIdx += 1 + newSpans[newSpanSliceIdx].Offset
|
||||
} else {
|
||||
newInsideSpanIdx++
|
||||
newIdx++
|
||||
}
|
||||
newBucketSliceIdx++
|
||||
newVal += newBuckets[newBucketSliceIdx]
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// AppendHistogram appends a SparseHistogram to the chunk. We assume the
|
||||
// histogram is properly structured. E.g. that the number of pos/neg buckets
|
||||
// used corresponds to the number conveyed by the pos/neg span structures.
|
||||
// callers must call Appendable() first and act accordingly!
|
||||
func (a *HistoAppender) AppendHistogram(t int64, h histogram.SparseHistogram) {
|
||||
var tDelta, cntDelta, zcntDelta int64
|
||||
num := binary.BigEndian.Uint16(a.b.bytes())
|
||||
|
||||
if value.IsStaleNaN(h.Sum) {
|
||||
// Emptying out other fields to write no buckets, and an empty meta in case of
|
||||
// first histogram in the chunk.
|
||||
h = histogram.SparseHistogram{Sum: h.Sum}
|
||||
}
|
||||
|
||||
switch num {
|
||||
case 0:
|
||||
// the first append gets the privilege to dictate the metadata
|
||||
// but it's also responsible for encoding it into the chunk!
|
||||
|
||||
writeHistoChunkMeta(a.b, h.Schema, h.ZeroThreshold, h.PositiveSpans, h.NegativeSpans)
|
||||
a.schema = h.Schema
|
||||
a.zeroThreshold = h.ZeroThreshold
|
||||
a.posSpans, a.negSpans = h.PositiveSpans, h.NegativeSpans
|
||||
numPosBuckets, numNegBuckets := countSpans(h.PositiveSpans), countSpans(h.NegativeSpans)
|
||||
a.posbuckets = make([]int64, numPosBuckets)
|
||||
a.negbuckets = make([]int64, numNegBuckets)
|
||||
a.posbucketsDelta = make([]int64, numPosBuckets)
|
||||
a.negbucketsDelta = make([]int64, numNegBuckets)
|
||||
|
||||
// now store actual data
|
||||
putVarint(a.b, a.buf64, t)
|
||||
putUvarint(a.b, a.buf64, h.Count)
|
||||
putUvarint(a.b, a.buf64, h.ZeroCount)
|
||||
a.b.writeBits(math.Float64bits(h.Sum), 64)
|
||||
for _, buck := range h.PositiveBuckets {
|
||||
putVarint(a.b, a.buf64, buck)
|
||||
}
|
||||
for _, buck := range h.NegativeBuckets {
|
||||
putVarint(a.b, a.buf64, buck)
|
||||
}
|
||||
case 1:
|
||||
tDelta = t - a.t
|
||||
cntDelta = int64(h.Count) - int64(a.cnt)
|
||||
zcntDelta = int64(h.ZeroCount) - int64(a.zcnt)
|
||||
|
||||
if value.IsStaleNaN(h.Sum) {
|
||||
cntDelta, zcntDelta = 0, 0
|
||||
}
|
||||
|
||||
putVarint(a.b, a.buf64, tDelta)
|
||||
putVarint(a.b, a.buf64, cntDelta)
|
||||
putVarint(a.b, a.buf64, zcntDelta)
|
||||
|
||||
a.writeSumDelta(h.Sum)
|
||||
|
||||
for i, buck := range h.PositiveBuckets {
|
||||
delta := buck - a.posbuckets[i]
|
||||
putVarint(a.b, a.buf64, delta)
|
||||
a.posbucketsDelta[i] = delta
|
||||
}
|
||||
for i, buck := range h.NegativeBuckets {
|
||||
delta := buck - a.negbuckets[i]
|
||||
putVarint(a.b, a.buf64, delta)
|
||||
a.negbucketsDelta[i] = delta
|
||||
}
|
||||
|
||||
default:
|
||||
tDelta = t - a.t
|
||||
cntDelta = int64(h.Count) - int64(a.cnt)
|
||||
zcntDelta = int64(h.ZeroCount) - int64(a.zcnt)
|
||||
|
||||
tDod := tDelta - a.tDelta
|
||||
cntDod := cntDelta - a.cntDelta
|
||||
zcntDod := zcntDelta - a.zcntDelta
|
||||
|
||||
if value.IsStaleNaN(h.Sum) {
|
||||
cntDod, zcntDod = 0, 0
|
||||
}
|
||||
|
||||
putInt64VBBucket(a.b, tDod)
|
||||
putInt64VBBucket(a.b, cntDod)
|
||||
putInt64VBBucket(a.b, zcntDod)
|
||||
|
||||
a.writeSumDelta(h.Sum)
|
||||
|
||||
for i, buck := range h.PositiveBuckets {
|
||||
delta := buck - a.posbuckets[i]
|
||||
dod := delta - a.posbucketsDelta[i]
|
||||
putInt64VBBucket(a.b, dod)
|
||||
a.posbucketsDelta[i] = delta
|
||||
}
|
||||
for i, buck := range h.NegativeBuckets {
|
||||
delta := buck - a.negbuckets[i]
|
||||
dod := delta - a.negbucketsDelta[i]
|
||||
putInt64VBBucket(a.b, dod)
|
||||
a.negbucketsDelta[i] = delta
|
||||
}
|
||||
}
|
||||
|
||||
binary.BigEndian.PutUint16(a.b.bytes(), num+1)
|
||||
|
||||
a.t = t
|
||||
a.cnt = h.Count
|
||||
a.zcnt = h.ZeroCount
|
||||
a.tDelta = tDelta
|
||||
a.cntDelta = cntDelta
|
||||
a.zcntDelta = zcntDelta
|
||||
|
||||
a.posbuckets, a.negbuckets = h.PositiveBuckets, h.NegativeBuckets
|
||||
// note that the bucket deltas were already updated above
|
||||
a.sum = h.Sum
|
||||
}
|
||||
|
||||
// Recode converts the current chunk to accommodate an expansion of the set of
|
||||
// (positive and/or negative) buckets used, according to the provided
|
||||
// interjections, resulting in the honoring of the provided new posSpans and
|
||||
// negSpans.
|
||||
func (a *HistoAppender) Recode(posInterjections, negInterjections []Interjection, posSpans, negSpans []histogram.Span) (Chunk, Appender) {
|
||||
// TODO(beorn7): This currently just decodes everything and then encodes
|
||||
// it again with the new span layout. This can probably be done in-place
|
||||
// by editing the chunk. But let's first see how expensive it is in the
|
||||
// big picture.
|
||||
byts := a.b.bytes()
|
||||
it := newHistoIterator(byts)
|
||||
hc := NewHistoChunk()
|
||||
app, err := hc.Appender()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
numPosBuckets, numNegBuckets := countSpans(posSpans), countSpans(negSpans)
|
||||
|
||||
for it.Next() {
|
||||
tOld, hOld := it.AtHistogram()
|
||||
|
||||
// We have to newly allocate slices for the modified buckets
|
||||
// here because they are kept by the appender until the next
|
||||
// append.
|
||||
// TODO(beorn7): We might be able to optimize this.
|
||||
posBuckets := make([]int64, numPosBuckets)
|
||||
negBuckets := make([]int64, numNegBuckets)
|
||||
|
||||
// Save the modified histogram to the new chunk.
|
||||
hOld.PositiveSpans, hOld.NegativeSpans = posSpans, negSpans
|
||||
if len(posInterjections) > 0 {
|
||||
hOld.PositiveBuckets = interject(hOld.PositiveBuckets, posBuckets, posInterjections)
|
||||
}
|
||||
if len(negInterjections) > 0 {
|
||||
hOld.NegativeBuckets = interject(hOld.NegativeBuckets, negBuckets, negInterjections)
|
||||
}
|
||||
app.AppendHistogram(tOld, hOld)
|
||||
}
|
||||
|
||||
// Set the flags.
|
||||
hc.SetCounterResetHeader(CounterResetHeader(byts[2] & 0b11000000))
|
||||
return hc, app
|
||||
}
|
||||
|
||||
func (a *HistoAppender) writeSumDelta(v float64) {
|
||||
vDelta := math.Float64bits(v) ^ math.Float64bits(a.sum)
|
||||
|
||||
if vDelta == 0 {
|
||||
a.b.writeBit(zero)
|
||||
return
|
||||
}
|
||||
a.b.writeBit(one)
|
||||
|
||||
leading := uint8(bits.LeadingZeros64(vDelta))
|
||||
trailing := uint8(bits.TrailingZeros64(vDelta))
|
||||
|
||||
// Clamp number of leading zeros to avoid overflow when encoding.
|
||||
if leading >= 32 {
|
||||
leading = 31
|
||||
}
|
||||
|
||||
if a.leading != 0xff && leading >= a.leading && trailing >= a.trailing {
|
||||
a.b.writeBit(zero)
|
||||
a.b.writeBits(vDelta>>a.trailing, 64-int(a.leading)-int(a.trailing))
|
||||
} else {
|
||||
a.leading, a.trailing = leading, trailing
|
||||
|
||||
a.b.writeBit(one)
|
||||
a.b.writeBits(uint64(leading), 5)
|
||||
|
||||
// Note that if leading == trailing == 0, then sigbits == 64. But that value doesn't actually fit into the 6 bits we have.
|
||||
// Luckily, we never need to encode 0 significant bits, since that would put us in the other case (vdelta == 0).
|
||||
// So instead we write out a 0 and adjust it back to 64 on unpacking.
|
||||
sigbits := 64 - leading - trailing
|
||||
a.b.writeBits(uint64(sigbits), 6)
|
||||
a.b.writeBits(vDelta>>trailing, int(sigbits))
|
||||
}
|
||||
}
|
||||
|
||||
type histoIterator struct {
|
||||
br bstreamReader
|
||||
numTotal uint16
|
||||
numRead uint16
|
||||
|
||||
// Metadata:
|
||||
schema int32
|
||||
zeroThreshold float64
|
||||
posSpans, negSpans []histogram.Span
|
||||
|
||||
// For the fields that are tracked as dod's.
|
||||
t int64
|
||||
cnt, zcnt uint64
|
||||
tDelta, cntDelta, zcntDelta int64
|
||||
|
||||
posbuckets, negbuckets []int64
|
||||
posbucketsDelta, negbucketsDelta []int64
|
||||
|
||||
// The sum is Gorilla xor encoded.
|
||||
sum float64
|
||||
leading uint8
|
||||
trailing uint8
|
||||
|
||||
err error
|
||||
}
|
||||
|
||||
func (it *histoIterator) Seek(t int64) bool {
|
||||
if it.err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
for t > it.t || it.numRead == 0 {
|
||||
if !it.Next() {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (it *histoIterator) At() (int64, float64) {
|
||||
panic("cannot call histoIterator.At().")
|
||||
}
|
||||
|
||||
func (it *histoIterator) ChunkEncoding() Encoding {
|
||||
return EncSHS
|
||||
}
|
||||
|
||||
func (it *histoIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
if value.IsStaleNaN(it.sum) {
|
||||
return it.t, histogram.SparseHistogram{Sum: it.sum}
|
||||
}
|
||||
return it.t, histogram.SparseHistogram{
|
||||
Count: it.cnt,
|
||||
ZeroCount: it.zcnt,
|
||||
Sum: it.sum,
|
||||
ZeroThreshold: it.zeroThreshold,
|
||||
Schema: it.schema,
|
||||
PositiveSpans: it.posSpans,
|
||||
NegativeSpans: it.negSpans,
|
||||
PositiveBuckets: it.posbuckets,
|
||||
NegativeBuckets: it.negbuckets,
|
||||
}
|
||||
}
|
||||
|
||||
func (it *histoIterator) Err() error {
|
||||
return it.err
|
||||
}
|
||||
|
||||
func (it *histoIterator) Reset(b []byte) {
|
||||
// The first 2 bytes contain chunk headers.
|
||||
// We skip that for actual samples.
|
||||
it.br = newBReader(b[2:])
|
||||
it.numTotal = binary.BigEndian.Uint16(b)
|
||||
it.numRead = 0
|
||||
|
||||
it.t, it.cnt, it.zcnt = 0, 0, 0
|
||||
it.tDelta, it.cntDelta, it.zcntDelta = 0, 0, 0
|
||||
|
||||
for i := range it.posbuckets {
|
||||
it.posbuckets[i] = 0
|
||||
it.posbucketsDelta[i] = 0
|
||||
}
|
||||
for i := range it.negbuckets {
|
||||
it.negbuckets[i] = 0
|
||||
it.negbucketsDelta[i] = 0
|
||||
}
|
||||
|
||||
it.sum = 0
|
||||
it.leading = 0
|
||||
it.trailing = 0
|
||||
it.err = nil
|
||||
}
|
||||
|
||||
func (it *histoIterator) Next() bool {
|
||||
if it.err != nil || it.numRead == it.numTotal {
|
||||
return false
|
||||
}
|
||||
|
||||
if it.numRead == 0 {
|
||||
|
||||
// first read is responsible for reading chunk metadata and initializing fields that depend on it
|
||||
// We give counter reset info at chunk level, hence we discard it here.
|
||||
schema, zeroThreshold, posSpans, negSpans, err := readHistoChunkMeta(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.schema = schema
|
||||
it.zeroThreshold = zeroThreshold
|
||||
it.posSpans, it.negSpans = posSpans, negSpans
|
||||
numPosBuckets, numNegBuckets := countSpans(posSpans), countSpans(negSpans)
|
||||
if numPosBuckets > 0 {
|
||||
it.posbuckets = make([]int64, numPosBuckets)
|
||||
it.posbucketsDelta = make([]int64, numPosBuckets)
|
||||
}
|
||||
if numNegBuckets > 0 {
|
||||
it.negbuckets = make([]int64, numNegBuckets)
|
||||
it.negbucketsDelta = make([]int64, numNegBuckets)
|
||||
}
|
||||
|
||||
// now read actual data
|
||||
|
||||
t, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.t = t
|
||||
|
||||
cnt, err := binary.ReadUvarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.cnt = cnt
|
||||
|
||||
zcnt, err := binary.ReadUvarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.zcnt = zcnt
|
||||
|
||||
sum, err := it.br.readBits(64)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.sum = math.Float64frombits(sum)
|
||||
|
||||
for i := range it.posbuckets {
|
||||
v, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.posbuckets[i] = v
|
||||
}
|
||||
for i := range it.negbuckets {
|
||||
v, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.negbuckets[i] = v
|
||||
}
|
||||
|
||||
it.numRead++
|
||||
return true
|
||||
}
|
||||
|
||||
if it.numRead == 1 {
|
||||
tDelta, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.tDelta = tDelta
|
||||
it.t += int64(it.tDelta)
|
||||
|
||||
cntDelta, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.cntDelta = cntDelta
|
||||
it.cnt = uint64(int64(it.cnt) + it.cntDelta)
|
||||
|
||||
zcntDelta, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.zcntDelta = zcntDelta
|
||||
it.zcnt = uint64(int64(it.zcnt) + it.zcntDelta)
|
||||
|
||||
ok := it.readSum()
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
if value.IsStaleNaN(it.sum) {
|
||||
it.numRead++
|
||||
return true
|
||||
}
|
||||
|
||||
for i := range it.posbuckets {
|
||||
delta, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.posbucketsDelta[i] = delta
|
||||
it.posbuckets[i] = it.posbuckets[i] + delta
|
||||
}
|
||||
|
||||
for i := range it.negbuckets {
|
||||
delta, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.negbucketsDelta[i] = delta
|
||||
it.negbuckets[i] = it.negbuckets[i] + delta
|
||||
}
|
||||
|
||||
it.numRead++
|
||||
return true
|
||||
}
|
||||
|
||||
tDod, err := readInt64VBBucket(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.tDelta = it.tDelta + tDod
|
||||
it.t += it.tDelta
|
||||
|
||||
cntDod, err := readInt64VBBucket(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.cntDelta = it.cntDelta + cntDod
|
||||
it.cnt = uint64(int64(it.cnt) + it.cntDelta)
|
||||
|
||||
zcntDod, err := readInt64VBBucket(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.zcntDelta = it.zcntDelta + zcntDod
|
||||
it.zcnt = uint64(int64(it.zcnt) + it.zcntDelta)
|
||||
|
||||
ok := it.readSum()
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
if value.IsStaleNaN(it.sum) {
|
||||
it.numRead++
|
||||
return true
|
||||
}
|
||||
|
||||
for i := range it.posbuckets {
|
||||
dod, err := readInt64VBBucket(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.posbucketsDelta[i] = it.posbucketsDelta[i] + dod
|
||||
it.posbuckets[i] = it.posbuckets[i] + it.posbucketsDelta[i]
|
||||
}
|
||||
|
||||
for i := range it.negbuckets {
|
||||
dod, err := readInt64VBBucket(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.negbucketsDelta[i] = it.negbucketsDelta[i] + dod
|
||||
it.negbuckets[i] = it.negbuckets[i] + it.negbucketsDelta[i]
|
||||
}
|
||||
|
||||
it.numRead++
|
||||
return true
|
||||
}
|
||||
|
||||
func (it *histoIterator) readSum() bool {
|
||||
bit, err := it.br.readBitFast()
|
||||
if err != nil {
|
||||
bit, err = it.br.readBit()
|
||||
}
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
|
||||
if bit == zero {
|
||||
// it.sum = it.sum
|
||||
} else {
|
||||
bit, err := it.br.readBitFast()
|
||||
if err != nil {
|
||||
bit, err = it.br.readBit()
|
||||
}
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
if bit == zero {
|
||||
// reuse leading/trailing zero bits
|
||||
// it.leading, it.trailing = it.leading, it.trailing
|
||||
} else {
|
||||
bits, err := it.br.readBitsFast(5)
|
||||
if err != nil {
|
||||
bits, err = it.br.readBits(5)
|
||||
}
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.leading = uint8(bits)
|
||||
|
||||
bits, err = it.br.readBitsFast(6)
|
||||
if err != nil {
|
||||
bits, err = it.br.readBits(6)
|
||||
}
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
mbits := uint8(bits)
|
||||
// 0 significant bits here means we overflowed and we actually need 64; see comment in encoder
|
||||
if mbits == 0 {
|
||||
mbits = 64
|
||||
}
|
||||
it.trailing = 64 - it.leading - mbits
|
||||
}
|
||||
|
||||
mbits := 64 - it.leading - it.trailing
|
||||
bits, err := it.br.readBitsFast(mbits)
|
||||
if err != nil {
|
||||
bits, err = it.br.readBits(mbits)
|
||||
}
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
vbits := math.Float64bits(it.sum)
|
||||
vbits ^= bits << it.trailing
|
||||
it.sum = math.Float64frombits(vbits)
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
|
@ -1,261 +0,0 @@
|
|||
// Copyright 2021 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package chunkenc
|
||||
|
||||
import (
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
)
|
||||
|
||||
func writeHistoChunkMeta(b *bstream, schema int32, zeroThreshold float64, posSpans, negSpans []histogram.Span) {
|
||||
putInt64VBBucket(b, int64(schema))
|
||||
putFloat64VBBucket(b, zeroThreshold)
|
||||
putHistoChunkMetaSpans(b, posSpans)
|
||||
putHistoChunkMetaSpans(b, negSpans)
|
||||
}
|
||||
|
||||
func putHistoChunkMetaSpans(b *bstream, spans []histogram.Span) {
|
||||
putInt64VBBucket(b, int64(len(spans)))
|
||||
for _, s := range spans {
|
||||
putInt64VBBucket(b, int64(s.Length))
|
||||
putInt64VBBucket(b, int64(s.Offset))
|
||||
}
|
||||
}
|
||||
|
||||
func readHistoChunkMeta(b *bstreamReader) (schema int32, zeroThreshold float64, posSpans []histogram.Span, negSpans []histogram.Span, err error) {
|
||||
_, err = b.ReadByte() // The header.
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
v, err := readInt64VBBucket(b)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
schema = int32(v)
|
||||
|
||||
zeroThreshold, err = readFloat64VBBucket(b)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
posSpans, err = readHistoChunkMetaSpans(b)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
negSpans, err = readHistoChunkMetaSpans(b)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func readHistoChunkMetaSpans(b *bstreamReader) ([]histogram.Span, error) {
|
||||
var spans []histogram.Span
|
||||
num, err := readInt64VBBucket(b)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := 0; i < int(num); i++ {
|
||||
|
||||
length, err := readInt64VBBucket(b)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
offset, err := readInt64VBBucket(b)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
spans = append(spans, histogram.Span{
|
||||
Length: uint32(length),
|
||||
Offset: int32(offset),
|
||||
})
|
||||
}
|
||||
return spans, nil
|
||||
}
|
||||
|
||||
type bucketIterator struct {
|
||||
spans []histogram.Span
|
||||
span int // span position of last yielded bucket
|
||||
bucket int // bucket position within span of last yielded bucket
|
||||
idx int // bucket index (globally across all spans) of last yielded bucket
|
||||
}
|
||||
|
||||
func newBucketIterator(spans []histogram.Span) *bucketIterator {
|
||||
b := bucketIterator{
|
||||
spans: spans,
|
||||
span: 0,
|
||||
bucket: -1,
|
||||
idx: -1,
|
||||
}
|
||||
if len(spans) > 0 {
|
||||
b.idx += int(spans[0].Offset)
|
||||
}
|
||||
return &b
|
||||
}
|
||||
|
||||
func (b *bucketIterator) Next() (int, bool) {
|
||||
// we're already out of bounds
|
||||
if b.span >= len(b.spans) {
|
||||
return 0, false
|
||||
}
|
||||
try:
|
||||
if b.bucket < int(b.spans[b.span].Length-1) { // try to move within same span.
|
||||
b.bucket++
|
||||
b.idx++
|
||||
return b.idx, true
|
||||
} else if b.span < len(b.spans)-1 { // try to move from one span to the next
|
||||
b.span++
|
||||
b.idx += int(b.spans[b.span].Offset + 1)
|
||||
b.bucket = 0
|
||||
if b.spans[b.span].Length == 0 {
|
||||
// pathological case that should never happen. We can't use this span, let's try again.
|
||||
goto try
|
||||
}
|
||||
return b.idx, true
|
||||
}
|
||||
// we're out of options
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// Interjection describes that num new buckets are introduced before processing the pos'th delta from the original slice
|
||||
type Interjection struct {
|
||||
pos int
|
||||
num int
|
||||
}
|
||||
|
||||
// compareSpans returns the interjections to convert a slice of deltas to a new slice representing an expanded set of buckets, or false if incompatible (e.g. if buckets were removed)
|
||||
// For example:
|
||||
// Let's say the old buckets look like this:
|
||||
// span syntax: [offset, length]
|
||||
// spans : [ 0 , 2 ] [2,1] [ 3 , 2 ] [3,1] [1,1]
|
||||
// bucket idx : [0] [1] 2 3 [4] 5 6 7 [8] [9] 10 11 12 [13] 14 [15]
|
||||
// raw values 6 3 3 2 4 5 1
|
||||
// deltas 6 -3 0 -1 2 1 -4
|
||||
|
||||
// But now we introduce a new bucket layout. (carefully chosen example where we have a span appended, one unchanged[*], one prepended, and two merge - in that order)
|
||||
// [*] unchanged in terms of which bucket indices they represent. but to achieve that, their offset needs to change if "disrupted" by spans changing ahead of them
|
||||
// \/ this one is "unchanged"
|
||||
// spans : [ 0 , 3 ] [1,1] [ 1 , 4 ] [ 3 , 3 ]
|
||||
// bucket idx : [0] [1] [2] 3 [4] 5 [6] [7] [8] [9] 10 11 12 [13] [14] [15]
|
||||
// raw values 6 3 0 3 0 0 2 4 5 0 1
|
||||
// deltas 6 -3 -3 3 -3 0 2 2 1 -5 1
|
||||
// delta mods: / \ / \ / \
|
||||
// note that whenever any new buckets are introduced, the subsequent "old" bucket needs to readjust its delta to the new base of 0
|
||||
// thus, for the caller, who wants to transform the set of original deltas to a new set of deltas to match a new span layout that adds buckets, we simply
|
||||
// need to generate a list of interjections
|
||||
// note: within compareSpans we don't have to worry about the changes to the spans themselves,
|
||||
// thanks to the iterators, we get to work with the more useful bucket indices (which of course directly correspond to the buckets we have to adjust)
|
||||
func compareSpans(a, b []histogram.Span) ([]Interjection, bool) {
|
||||
ai := newBucketIterator(a)
|
||||
bi := newBucketIterator(b)
|
||||
|
||||
var interjections []Interjection
|
||||
|
||||
// when inter.num becomes > 0, this becomes a valid interjection that should be yielded when we finish a streak of new buckets
|
||||
var inter Interjection
|
||||
|
||||
av, aok := ai.Next()
|
||||
bv, bok := bi.Next()
|
||||
loop:
|
||||
for {
|
||||
switch {
|
||||
case aok && bok:
|
||||
switch {
|
||||
case av == bv: // Both have an identical value. move on!
|
||||
// Finish WIP interjection and reset.
|
||||
if inter.num > 0 {
|
||||
interjections = append(interjections, inter)
|
||||
}
|
||||
inter.num = 0
|
||||
av, aok = ai.Next()
|
||||
bv, bok = bi.Next()
|
||||
inter.pos++
|
||||
case av < bv: // b misses a value that is in a.
|
||||
return interjections, false
|
||||
case av > bv: // a misses a value that is in b. Forward b and recompare.
|
||||
inter.num++
|
||||
bv, bok = bi.Next()
|
||||
}
|
||||
case aok && !bok: // b misses a value that is in a.
|
||||
return interjections, false
|
||||
case !aok && bok: // a misses a value that is in b. Forward b and recompare.
|
||||
inter.num++
|
||||
bv, bok = bi.Next()
|
||||
default: // Both iterators ran out. We're done.
|
||||
if inter.num > 0 {
|
||||
interjections = append(interjections, inter)
|
||||
}
|
||||
break loop
|
||||
}
|
||||
}
|
||||
|
||||
return interjections, true
|
||||
}
|
||||
|
||||
// interject merges 'in' with the provided interjections and writes them into
|
||||
// 'out', which must already have the appropriate length.
|
||||
func interject(in, out []int64, interjections []Interjection) []int64 {
|
||||
var j int // Position in out.
|
||||
var v int64 // The last value seen.
|
||||
var interj int // The next interjection to process.
|
||||
for i, d := range in {
|
||||
if interj < len(interjections) && i == interjections[interj].pos {
|
||||
|
||||
// We have an interjection!
|
||||
// Add interjection.num new delta values such that their
|
||||
// bucket values equate 0.
|
||||
out[j] = int64(-v)
|
||||
j++
|
||||
for x := 1; x < interjections[interj].num; x++ {
|
||||
out[j] = 0
|
||||
j++
|
||||
}
|
||||
interj++
|
||||
|
||||
// Now save the value from the input. The delta value we
|
||||
// should save is the original delta value + the last
|
||||
// value of the point before the interjection (to undo
|
||||
// the delta that was introduced by the interjection).
|
||||
out[j] = d + v
|
||||
j++
|
||||
v = d + v
|
||||
continue
|
||||
}
|
||||
|
||||
// If there was no interjection, the original delta is still
|
||||
// valid.
|
||||
out[j] = d
|
||||
j++
|
||||
v += d
|
||||
}
|
||||
switch interj {
|
||||
case len(interjections):
|
||||
// All interjections processed. Nothing more to do.
|
||||
case len(interjections) - 1:
|
||||
// One more interjection to process at the end.
|
||||
out[j] = int64(-v)
|
||||
j++
|
||||
for x := 1; x < interjections[interj].num; x++ {
|
||||
out[j] = 0
|
||||
j++
|
||||
}
|
||||
default:
|
||||
panic("unprocessed interjections left")
|
||||
}
|
||||
return out
|
||||
}
|
||||
934
tsdb/chunkenc/histogram.go
Normal file
934
tsdb/chunkenc/histogram.go
Normal file
|
|
@ -0,0 +1,934 @@
|
|||
// Copyright 2021 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package chunkenc
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"math"
|
||||
"math/bits"
|
||||
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/value"
|
||||
)
|
||||
|
||||
const ()
|
||||
|
||||
// HistogramChunk holds encoded sample data for a sparse, high-resolution
|
||||
// histogram.
|
||||
//
|
||||
// TODO(beorn7): Document the layout of chunk metadata.
|
||||
//
|
||||
// Each sample has multiple "fields", stored in the following way (raw = store
|
||||
// number directly, delta = store delta to the previous number, dod = store
|
||||
// delta of the delta to the previous number, xor = what we do for regular
|
||||
// sample values):
|
||||
//
|
||||
// field → ts count zeroCount sum []posbuckets []negbuckets
|
||||
// sample 1 raw raw raw raw []raw []raw
|
||||
// sample 2 delta delta delta xor []delta []delta
|
||||
// sample >2 dod dod dod xor []dod []dod
|
||||
type HistogramChunk struct {
|
||||
b bstream
|
||||
}
|
||||
|
||||
// NewHistogramChunk returns a new chunk with histogram encoding of the given
|
||||
// size.
|
||||
func NewHistogramChunk() *HistogramChunk {
|
||||
b := make([]byte, 3, 128)
|
||||
return &HistogramChunk{b: bstream{stream: b, count: 0}}
|
||||
}
|
||||
|
||||
// Encoding returns the encoding type.
|
||||
func (c *HistogramChunk) Encoding() Encoding {
|
||||
return EncHistogram
|
||||
}
|
||||
|
||||
// Bytes returns the underlying byte slice of the chunk.
|
||||
func (c *HistogramChunk) Bytes() []byte {
|
||||
return c.b.bytes()
|
||||
}
|
||||
|
||||
// NumSamples returns the number of samples in the chunk.
|
||||
func (c *HistogramChunk) NumSamples() int {
|
||||
return int(binary.BigEndian.Uint16(c.Bytes()))
|
||||
}
|
||||
|
||||
// Meta returns the histogram metadata. Only call this on chunks that have at
|
||||
// least one sample.
|
||||
func (c *HistogramChunk) Meta() (
|
||||
schema int32, zeroThreshold float64,
|
||||
negativeSpans, positiveSpans []histogram.Span,
|
||||
err error,
|
||||
) {
|
||||
if c.NumSamples() == 0 {
|
||||
panic("HistoChunk.Meta() called on an empty chunk")
|
||||
}
|
||||
b := newBReader(c.Bytes()[2:])
|
||||
return readHistogramChunkMeta(&b)
|
||||
}
|
||||
|
||||
// CounterResetHeader defines the first 2 bits of the chunk header.
|
||||
type CounterResetHeader byte
|
||||
|
||||
const (
|
||||
CounterReset CounterResetHeader = 0b10000000
|
||||
NotCounterReset CounterResetHeader = 0b01000000
|
||||
GaugeType CounterResetHeader = 0b11000000
|
||||
UnknownCounterReset CounterResetHeader = 0b00000000
|
||||
)
|
||||
|
||||
// SetCounterResetHeader sets the counter reset header.
|
||||
func (c *HistogramChunk) SetCounterResetHeader(h CounterResetHeader) {
|
||||
switch h {
|
||||
case CounterReset, NotCounterReset, GaugeType, UnknownCounterReset:
|
||||
bytes := c.Bytes()
|
||||
bytes[2] = (bytes[2] & 0b00111111) | byte(h)
|
||||
default:
|
||||
panic("invalid CounterResetHeader type")
|
||||
}
|
||||
}
|
||||
|
||||
// GetCounterResetHeader returns the info about the first 2 bits of the chunk
|
||||
// header.
|
||||
func (c *HistogramChunk) GetCounterResetHeader() CounterResetHeader {
|
||||
return CounterResetHeader(c.Bytes()[2] & 0b11000000)
|
||||
}
|
||||
|
||||
// Compact implements the Chunk interface.
|
||||
func (c *HistogramChunk) Compact() {
|
||||
if l := len(c.b.stream); cap(c.b.stream) > l+chunkCompactCapacityThreshold {
|
||||
buf := make([]byte, l)
|
||||
copy(buf, c.b.stream)
|
||||
c.b.stream = buf
|
||||
}
|
||||
}
|
||||
|
||||
// Appender implements the Chunk interface.
|
||||
func (c *HistogramChunk) Appender() (Appender, error) {
|
||||
it := c.iterator(nil)
|
||||
|
||||
// To get an appender, we must know the state it would have if we had
|
||||
// appended all existing data from scratch. We iterate through the end
|
||||
// and populate via the iterator's state.
|
||||
for it.Next() {
|
||||
}
|
||||
if err := it.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
a := &HistogramAppender{
|
||||
b: &c.b,
|
||||
|
||||
schema: it.schema,
|
||||
zThreshold: it.zThreshold,
|
||||
pSpans: it.pSpans,
|
||||
nSpans: it.nSpans,
|
||||
t: it.t,
|
||||
cnt: it.cnt,
|
||||
zCnt: it.zCnt,
|
||||
tDelta: it.tDelta,
|
||||
cntDelta: it.cntDelta,
|
||||
zCntDelta: it.zCntDelta,
|
||||
pBuckets: it.pBuckets,
|
||||
nBuckets: it.nBuckets,
|
||||
pBucketsDelta: it.pBucketsDelta,
|
||||
nBucketsDelta: it.nBucketsDelta,
|
||||
|
||||
sum: it.sum,
|
||||
leading: it.leading,
|
||||
trailing: it.trailing,
|
||||
|
||||
buf64: make([]byte, binary.MaxVarintLen64),
|
||||
}
|
||||
if binary.BigEndian.Uint16(a.b.bytes()) == 0 {
|
||||
a.leading = 0xff
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func countSpans(spans []histogram.Span) int {
|
||||
var cnt int
|
||||
for _, s := range spans {
|
||||
cnt += int(s.Length)
|
||||
}
|
||||
return cnt
|
||||
}
|
||||
|
||||
func newHistogramIterator(b []byte) *histogramIterator {
|
||||
it := &histogramIterator{
|
||||
br: newBReader(b),
|
||||
numTotal: binary.BigEndian.Uint16(b),
|
||||
t: math.MinInt64,
|
||||
}
|
||||
// The first 2 bytes contain chunk headers.
|
||||
// We skip that for actual samples.
|
||||
_, _ = it.br.readBits(16)
|
||||
return it
|
||||
}
|
||||
|
||||
func (c *HistogramChunk) iterator(it Iterator) *histogramIterator {
|
||||
// This commet is copied from XORChunk.iterator:
|
||||
// Should iterators guarantee to act on a copy of the data so it doesn't lock append?
|
||||
// When using striped locks to guard access to chunks, probably yes.
|
||||
// Could only copy data if the chunk is not completed yet.
|
||||
if histogramIter, ok := it.(*histogramIterator); ok {
|
||||
histogramIter.Reset(c.b.bytes())
|
||||
return histogramIter
|
||||
}
|
||||
return newHistogramIterator(c.b.bytes())
|
||||
}
|
||||
|
||||
// Iterator implements the Chunk interface.
|
||||
func (c *HistogramChunk) Iterator(it Iterator) Iterator {
|
||||
return c.iterator(it)
|
||||
}
|
||||
|
||||
// HistogramAppender is an Appender implementation for sparse histograms.
|
||||
type HistogramAppender struct {
|
||||
b *bstream
|
||||
|
||||
// Metadata:
|
||||
schema int32
|
||||
zThreshold float64
|
||||
pSpans, nSpans []histogram.Span
|
||||
|
||||
// Although we intend to start new chunks on counter resets, we still
|
||||
// have to handle negative deltas for gauge histograms. Therefore, even
|
||||
// deltas are signed types here (even for tDelta to not treat that one
|
||||
// specially).
|
||||
t int64
|
||||
cnt, zCnt uint64
|
||||
tDelta, cntDelta, zCntDelta int64
|
||||
pBuckets, nBuckets []int64
|
||||
pBucketsDelta, nBucketsDelta []int64
|
||||
|
||||
// The sum is Gorilla xor encoded.
|
||||
sum float64
|
||||
leading uint8
|
||||
trailing uint8
|
||||
|
||||
buf64 []byte // For working on varint64's.
|
||||
}
|
||||
|
||||
func putVarint(b *bstream, buf []byte, x int64) {
|
||||
for _, byt := range buf[:binary.PutVarint(buf, x)] {
|
||||
b.writeByte(byt)
|
||||
}
|
||||
}
|
||||
|
||||
func putUvarint(b *bstream, buf []byte, x uint64) {
|
||||
for _, byt := range buf[:binary.PutUvarint(buf, x)] {
|
||||
b.writeByte(byt)
|
||||
}
|
||||
}
|
||||
|
||||
// Append implements Appender. This implementation panics because normal float
|
||||
// samples must never be appended to a histogram chunk.
|
||||
func (a *HistogramAppender) Append(int64, float64) {
|
||||
panic("appended a float sample to a histogram chunk")
|
||||
}
|
||||
|
||||
// Appendable returns whether the chunk can be appended to, and if so
|
||||
// whether any recoding needs to happen using the provided interjections
|
||||
// (in case of any new buckets, positive or negative range, respectively).
|
||||
//
|
||||
// The chunk is not appendable in the following cases:
|
||||
//
|
||||
// • The schema has changed.
|
||||
//
|
||||
// • The threshold for the zero bucket has changed.
|
||||
//
|
||||
// • Any buckets have disappeared.
|
||||
//
|
||||
// • There was a counter reset in the count of observations or in any bucket,
|
||||
// including the zero bucket.
|
||||
//
|
||||
// • The last sample in the chunk was stale while the current sample is not stale.
|
||||
//
|
||||
// The method returns an additional boolean set to true if it is not appendable
|
||||
// because of a counter reset. If the given sample is stale, it is always ok to
|
||||
// append. If counterReset is true, okToAppend is always false.
|
||||
func (a *HistogramAppender) Appendable(h histogram.Histogram) (
|
||||
positiveInterjections, negativeInterjections []Interjection,
|
||||
okToAppend bool, counterReset bool,
|
||||
) {
|
||||
if value.IsStaleNaN(h.Sum) {
|
||||
// This is a stale sample whose buckets and spans don't matter.
|
||||
okToAppend = true
|
||||
return
|
||||
}
|
||||
if value.IsStaleNaN(a.sum) {
|
||||
// If the last sample was stale, then we can only accept stale
|
||||
// samples in this chunk.
|
||||
return
|
||||
}
|
||||
|
||||
if h.Count < a.cnt {
|
||||
// There has been a counter reset.
|
||||
counterReset = true
|
||||
return
|
||||
}
|
||||
|
||||
if h.Schema != a.schema || h.ZeroThreshold != a.zThreshold {
|
||||
return
|
||||
}
|
||||
|
||||
if h.ZeroCount < a.zCnt {
|
||||
// There has been a counter reset since ZeroThreshold didn't change.
|
||||
counterReset = true
|
||||
return
|
||||
}
|
||||
|
||||
var ok bool
|
||||
positiveInterjections, ok = compareSpans(a.pSpans, h.PositiveSpans)
|
||||
if !ok {
|
||||
counterReset = true
|
||||
return
|
||||
}
|
||||
negativeInterjections, ok = compareSpans(a.nSpans, h.NegativeSpans)
|
||||
if !ok {
|
||||
counterReset = true
|
||||
return
|
||||
}
|
||||
|
||||
if counterResetInAnyBucket(a.pBuckets, h.PositiveBuckets, a.pSpans, h.PositiveSpans) ||
|
||||
counterResetInAnyBucket(a.nBuckets, h.NegativeBuckets, a.nSpans, h.NegativeSpans) {
|
||||
counterReset, positiveInterjections, negativeInterjections = true, nil, nil
|
||||
return
|
||||
}
|
||||
|
||||
okToAppend = true
|
||||
return
|
||||
}
|
||||
|
||||
// counterResetInAnyBucket returns true if there was a counter reset for any
|
||||
// bucket. This should be called only when the bucket layout is the same or new
|
||||
// buckets were added. It does not handle the case of buckets missing.
|
||||
func counterResetInAnyBucket(oldBuckets, newBuckets []int64, oldSpans, newSpans []histogram.Span) bool {
|
||||
if len(oldSpans) == 0 || len(oldBuckets) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
oldSpanSliceIdx, newSpanSliceIdx := 0, 0 // Index for the span slices.
|
||||
oldInsideSpanIdx, newInsideSpanIdx := uint32(0), uint32(0) // Index inside a span.
|
||||
oldIdx, newIdx := oldSpans[0].Offset, newSpans[0].Offset
|
||||
|
||||
oldBucketSliceIdx, newBucketSliceIdx := 0, 0 // Index inside bucket slice.
|
||||
oldVal, newVal := oldBuckets[0], newBuckets[0]
|
||||
|
||||
// Since we assume that new spans won't have missing buckets, there will never be a case
|
||||
// where the old index will not find a matching new index.
|
||||
for {
|
||||
if oldIdx == newIdx {
|
||||
if newVal < oldVal {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
if oldIdx <= newIdx {
|
||||
// Moving ahead old bucket and span by 1 index.
|
||||
if oldInsideSpanIdx == oldSpans[oldSpanSliceIdx].Length-1 {
|
||||
// Current span is over.
|
||||
oldSpanSliceIdx++
|
||||
oldInsideSpanIdx = 0
|
||||
if oldSpanSliceIdx >= len(oldSpans) {
|
||||
// All old spans are over.
|
||||
break
|
||||
}
|
||||
oldIdx += 1 + oldSpans[oldSpanSliceIdx].Offset
|
||||
} else {
|
||||
oldInsideSpanIdx++
|
||||
oldIdx++
|
||||
}
|
||||
oldBucketSliceIdx++
|
||||
oldVal += oldBuckets[oldBucketSliceIdx]
|
||||
}
|
||||
|
||||
if oldIdx > newIdx {
|
||||
// Moving ahead new bucket and span by 1 index.
|
||||
if newInsideSpanIdx == newSpans[newSpanSliceIdx].Length-1 {
|
||||
// Current span is over.
|
||||
newSpanSliceIdx++
|
||||
newInsideSpanIdx = 0
|
||||
if newSpanSliceIdx >= len(newSpans) {
|
||||
// All new spans are over.
|
||||
// This should not happen, old spans above should catch this first.
|
||||
panic("new spans over before old spans in counterReset")
|
||||
}
|
||||
newIdx += 1 + newSpans[newSpanSliceIdx].Offset
|
||||
} else {
|
||||
newInsideSpanIdx++
|
||||
newIdx++
|
||||
}
|
||||
newBucketSliceIdx++
|
||||
newVal += newBuckets[newBucketSliceIdx]
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// AppendHistogram appends a histogram to the chunk. The caller must ensure that
|
||||
// the histogram is properly structured, e.g. the number of buckets used
|
||||
// corresponds to the number conveyed by the span structures. First call
|
||||
// Appendable() and act accordingly!
|
||||
func (a *HistogramAppender) AppendHistogram(t int64, h histogram.Histogram) {
|
||||
var tDelta, cntDelta, zCntDelta int64
|
||||
num := binary.BigEndian.Uint16(a.b.bytes())
|
||||
|
||||
if value.IsStaleNaN(h.Sum) {
|
||||
// Emptying out other fields to write no buckets, and an empty
|
||||
// meta in case of first histogram in the chunk.
|
||||
h = histogram.Histogram{Sum: h.Sum}
|
||||
}
|
||||
|
||||
switch num {
|
||||
case 0:
|
||||
// The first append gets the privilege to dictate the metadata
|
||||
// but it's also responsible for encoding it into the chunk!
|
||||
writeHistogramChunkMeta(a.b, h.Schema, h.ZeroThreshold, h.PositiveSpans, h.NegativeSpans)
|
||||
a.schema = h.Schema
|
||||
a.zThreshold = h.ZeroThreshold
|
||||
a.pSpans, a.nSpans = h.PositiveSpans, h.NegativeSpans
|
||||
numPBuckets, numNBuckets := countSpans(h.PositiveSpans), countSpans(h.NegativeSpans)
|
||||
a.pBuckets = make([]int64, numPBuckets)
|
||||
a.nBuckets = make([]int64, numNBuckets)
|
||||
a.pBucketsDelta = make([]int64, numPBuckets)
|
||||
a.nBucketsDelta = make([]int64, numNBuckets)
|
||||
|
||||
// Now store the actual data.
|
||||
putVarint(a.b, a.buf64, t)
|
||||
putUvarint(a.b, a.buf64, h.Count) // TODO(beorn7): Use putVarbitInt?
|
||||
putUvarint(a.b, a.buf64, h.ZeroCount) // TODO(beorn7): Use putVarbitInt?
|
||||
a.b.writeBits(math.Float64bits(h.Sum), 64)
|
||||
for _, buck := range h.PositiveBuckets {
|
||||
putVarint(a.b, a.buf64, buck) // TODO(beorn7): Use putVarbitInt?
|
||||
}
|
||||
for _, buck := range h.NegativeBuckets {
|
||||
putVarint(a.b, a.buf64, buck) // TODO(beorn7): Use putVarbitInt?
|
||||
}
|
||||
case 1:
|
||||
tDelta = t - a.t
|
||||
cntDelta = int64(h.Count) - int64(a.cnt)
|
||||
zCntDelta = int64(h.ZeroCount) - int64(a.zCnt)
|
||||
|
||||
if value.IsStaleNaN(h.Sum) {
|
||||
cntDelta, zCntDelta = 0, 0
|
||||
}
|
||||
|
||||
putVarint(a.b, a.buf64, tDelta) // TODO(beorn7): This should probably be putUvarint.
|
||||
putVarint(a.b, a.buf64, cntDelta) // TODO(beorn7): Use putVarbitInt?
|
||||
putVarint(a.b, a.buf64, zCntDelta) // TODO(beorn7): Use putVarbitInt?
|
||||
|
||||
a.writeSumDelta(h.Sum)
|
||||
|
||||
for i, buck := range h.PositiveBuckets {
|
||||
delta := buck - a.pBuckets[i]
|
||||
putVarint(a.b, a.buf64, delta) // TODO(beorn7): Use putVarbitInt?
|
||||
a.pBucketsDelta[i] = delta
|
||||
}
|
||||
for i, buck := range h.NegativeBuckets {
|
||||
delta := buck - a.nBuckets[i]
|
||||
putVarint(a.b, a.buf64, delta) // TODO(beorn7): Use putVarbitInt?
|
||||
a.nBucketsDelta[i] = delta
|
||||
}
|
||||
|
||||
default:
|
||||
tDelta = t - a.t
|
||||
cntDelta = int64(h.Count) - int64(a.cnt)
|
||||
zCntDelta = int64(h.ZeroCount) - int64(a.zCnt)
|
||||
|
||||
tDod := tDelta - a.tDelta
|
||||
cntDod := cntDelta - a.cntDelta
|
||||
zCntDod := zCntDelta - a.zCntDelta
|
||||
|
||||
if value.IsStaleNaN(h.Sum) {
|
||||
cntDod, zCntDod = 0, 0
|
||||
}
|
||||
|
||||
putVarbitInt(a.b, tDod)
|
||||
putVarbitInt(a.b, cntDod)
|
||||
putVarbitInt(a.b, zCntDod)
|
||||
|
||||
a.writeSumDelta(h.Sum)
|
||||
|
||||
for i, buck := range h.PositiveBuckets {
|
||||
delta := buck - a.pBuckets[i]
|
||||
dod := delta - a.pBucketsDelta[i]
|
||||
putVarbitInt(a.b, dod)
|
||||
a.pBucketsDelta[i] = delta
|
||||
}
|
||||
for i, buck := range h.NegativeBuckets {
|
||||
delta := buck - a.nBuckets[i]
|
||||
dod := delta - a.nBucketsDelta[i]
|
||||
putVarbitInt(a.b, dod)
|
||||
a.nBucketsDelta[i] = delta
|
||||
}
|
||||
}
|
||||
|
||||
binary.BigEndian.PutUint16(a.b.bytes(), num+1)
|
||||
|
||||
a.t = t
|
||||
a.cnt = h.Count
|
||||
a.zCnt = h.ZeroCount
|
||||
a.tDelta = tDelta
|
||||
a.cntDelta = cntDelta
|
||||
a.zCntDelta = zCntDelta
|
||||
|
||||
a.pBuckets, a.nBuckets = h.PositiveBuckets, h.NegativeBuckets
|
||||
// Note that the bucket deltas were already updated above.
|
||||
a.sum = h.Sum
|
||||
}
|
||||
|
||||
// Recode converts the current chunk to accommodate an expansion of the set of
|
||||
// (positive and/or negative) buckets used, according to the provided
|
||||
// interjections, resulting in the honoring of the provided new positive and
|
||||
// negative spans.
|
||||
func (a *HistogramAppender) Recode(
|
||||
positiveInterjections, negativeInterjections []Interjection,
|
||||
positiveSpans, negativeSpans []histogram.Span,
|
||||
) (Chunk, Appender) {
|
||||
// TODO(beorn7): This currently just decodes everything and then encodes
|
||||
// it again with the new span layout. This can probably be done in-place
|
||||
// by editing the chunk. But let's first see how expensive it is in the
|
||||
// big picture.
|
||||
byts := a.b.bytes()
|
||||
it := newHistogramIterator(byts)
|
||||
hc := NewHistogramChunk()
|
||||
app, err := hc.Appender()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
numPositiveBuckets, numNegativeBuckets := countSpans(positiveSpans), countSpans(negativeSpans)
|
||||
|
||||
for it.Next() {
|
||||
tOld, hOld := it.AtHistogram()
|
||||
|
||||
// We have to newly allocate slices for the modified buckets
|
||||
// here because they are kept by the appender until the next
|
||||
// append.
|
||||
// TODO(beorn7): We might be able to optimize this.
|
||||
positiveBuckets := make([]int64, numPositiveBuckets)
|
||||
negativeBuckets := make([]int64, numNegativeBuckets)
|
||||
|
||||
// Save the modified histogram to the new chunk.
|
||||
hOld.PositiveSpans, hOld.NegativeSpans = positiveSpans, negativeSpans
|
||||
if len(positiveInterjections) > 0 {
|
||||
hOld.PositiveBuckets = interject(hOld.PositiveBuckets, positiveBuckets, positiveInterjections)
|
||||
}
|
||||
if len(negativeInterjections) > 0 {
|
||||
hOld.NegativeBuckets = interject(hOld.NegativeBuckets, negativeBuckets, negativeInterjections)
|
||||
}
|
||||
app.AppendHistogram(tOld, hOld)
|
||||
}
|
||||
|
||||
hc.SetCounterResetHeader(CounterResetHeader(byts[2] & 0b11000000))
|
||||
return hc, app
|
||||
}
|
||||
|
||||
func (a *HistogramAppender) writeSumDelta(v float64) {
|
||||
vDelta := math.Float64bits(v) ^ math.Float64bits(a.sum)
|
||||
|
||||
if vDelta == 0 {
|
||||
a.b.writeBit(zero)
|
||||
return
|
||||
}
|
||||
a.b.writeBit(one)
|
||||
|
||||
leading := uint8(bits.LeadingZeros64(vDelta))
|
||||
trailing := uint8(bits.TrailingZeros64(vDelta))
|
||||
|
||||
// Clamp number of leading zeros to avoid overflow when encoding.
|
||||
if leading >= 32 {
|
||||
leading = 31
|
||||
}
|
||||
|
||||
if a.leading != 0xff && leading >= a.leading && trailing >= a.trailing {
|
||||
a.b.writeBit(zero)
|
||||
a.b.writeBits(vDelta>>a.trailing, 64-int(a.leading)-int(a.trailing))
|
||||
} else {
|
||||
a.leading, a.trailing = leading, trailing
|
||||
|
||||
a.b.writeBit(one)
|
||||
a.b.writeBits(uint64(leading), 5)
|
||||
|
||||
// Note that if leading == trailing == 0, then sigbits == 64.
|
||||
// But that value doesn't actually fit into the 6 bits we have.
|
||||
// Luckily, we never need to encode 0 significant bits, since
|
||||
// that would put us in the other case (vdelta == 0). So
|
||||
// instead we write out a 0 and adjust it back to 64 on
|
||||
// unpacking.
|
||||
sigbits := 64 - leading - trailing
|
||||
a.b.writeBits(uint64(sigbits), 6)
|
||||
a.b.writeBits(vDelta>>trailing, int(sigbits))
|
||||
}
|
||||
}
|
||||
|
||||
type histogramIterator struct {
|
||||
br bstreamReader
|
||||
numTotal uint16
|
||||
numRead uint16
|
||||
|
||||
// Metadata:
|
||||
schema int32
|
||||
zThreshold float64
|
||||
pSpans, nSpans []histogram.Span
|
||||
|
||||
// For the fields that are tracked as deltas and ultimately dod's.
|
||||
t int64
|
||||
cnt, zCnt uint64
|
||||
tDelta, cntDelta, zCntDelta int64
|
||||
pBuckets, nBuckets []int64
|
||||
pBucketsDelta, nBucketsDelta []int64
|
||||
|
||||
// The sum is Gorilla xor encoded.
|
||||
sum float64
|
||||
leading uint8
|
||||
trailing uint8
|
||||
|
||||
err error
|
||||
}
|
||||
|
||||
func (it *histogramIterator) Seek(t int64) bool {
|
||||
if it.err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
for t > it.t || it.numRead == 0 {
|
||||
if !it.Next() {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (it *histogramIterator) At() (int64, float64) {
|
||||
panic("cannot call histogramIterator.At")
|
||||
}
|
||||
|
||||
func (it *histogramIterator) ChunkEncoding() Encoding {
|
||||
return EncHistogram
|
||||
}
|
||||
|
||||
func (it *histogramIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
if value.IsStaleNaN(it.sum) {
|
||||
return it.t, histogram.Histogram{Sum: it.sum}
|
||||
}
|
||||
return it.t, histogram.Histogram{
|
||||
Count: it.cnt,
|
||||
ZeroCount: it.zCnt,
|
||||
Sum: it.sum,
|
||||
ZeroThreshold: it.zThreshold,
|
||||
Schema: it.schema,
|
||||
PositiveSpans: it.pSpans,
|
||||
NegativeSpans: it.nSpans,
|
||||
PositiveBuckets: it.pBuckets,
|
||||
NegativeBuckets: it.nBuckets,
|
||||
}
|
||||
}
|
||||
|
||||
func (it *histogramIterator) Err() error {
|
||||
return it.err
|
||||
}
|
||||
|
||||
func (it *histogramIterator) Reset(b []byte) {
|
||||
// The first 2 bytes contain chunk headers.
|
||||
// We skip that for actual samples.
|
||||
it.br = newBReader(b[2:])
|
||||
it.numTotal = binary.BigEndian.Uint16(b)
|
||||
it.numRead = 0
|
||||
|
||||
it.t, it.cnt, it.zCnt = 0, 0, 0
|
||||
it.tDelta, it.cntDelta, it.zCntDelta = 0, 0, 0
|
||||
|
||||
// TODO(beorn7): Those will be recreated anyway.
|
||||
// Either delete them here entirely or recycle them
|
||||
// below if big enough.
|
||||
for i := range it.pBuckets {
|
||||
it.pBuckets[i] = 0
|
||||
it.pBucketsDelta[i] = 0
|
||||
}
|
||||
for i := range it.nBuckets {
|
||||
it.nBuckets[i] = 0
|
||||
it.nBucketsDelta[i] = 0
|
||||
}
|
||||
|
||||
it.sum = 0
|
||||
it.leading = 0
|
||||
it.trailing = 0
|
||||
it.err = nil
|
||||
}
|
||||
|
||||
func (it *histogramIterator) Next() bool {
|
||||
if it.err != nil || it.numRead == it.numTotal {
|
||||
return false
|
||||
}
|
||||
|
||||
if it.numRead == 0 {
|
||||
|
||||
// The first read is responsible for reading the chunk metadata
|
||||
// and for initializing fields that depend on it. We give
|
||||
// counter reset info at chunk level, hence we discard it here.
|
||||
schema, zeroThreshold, posSpans, negSpans, err := readHistogramChunkMeta(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.schema = schema
|
||||
it.zThreshold = zeroThreshold
|
||||
it.pSpans, it.nSpans = posSpans, negSpans
|
||||
numPosBuckets, numNegBuckets := countSpans(posSpans), countSpans(negSpans)
|
||||
if numPosBuckets > 0 {
|
||||
it.pBuckets = make([]int64, numPosBuckets)
|
||||
it.pBucketsDelta = make([]int64, numPosBuckets)
|
||||
}
|
||||
if numNegBuckets > 0 {
|
||||
it.nBuckets = make([]int64, numNegBuckets)
|
||||
it.nBucketsDelta = make([]int64, numNegBuckets)
|
||||
}
|
||||
|
||||
// Now read the actual data.
|
||||
t, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.t = t
|
||||
|
||||
cnt, err := binary.ReadUvarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.cnt = cnt
|
||||
|
||||
zcnt, err := binary.ReadUvarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.zCnt = zcnt
|
||||
|
||||
sum, err := it.br.readBits(64)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.sum = math.Float64frombits(sum)
|
||||
|
||||
for i := range it.pBuckets {
|
||||
v, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.pBuckets[i] = v
|
||||
}
|
||||
for i := range it.nBuckets {
|
||||
v, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.nBuckets[i] = v
|
||||
}
|
||||
|
||||
it.numRead++
|
||||
return true
|
||||
}
|
||||
|
||||
if it.numRead == 1 {
|
||||
tDelta, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.tDelta = tDelta
|
||||
it.t += int64(it.tDelta)
|
||||
|
||||
cntDelta, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.cntDelta = cntDelta
|
||||
it.cnt = uint64(int64(it.cnt) + it.cntDelta)
|
||||
|
||||
zcntDelta, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.zCntDelta = zcntDelta
|
||||
it.zCnt = uint64(int64(it.zCnt) + it.zCntDelta)
|
||||
|
||||
ok := it.readSum()
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
if value.IsStaleNaN(it.sum) {
|
||||
it.numRead++
|
||||
return true
|
||||
}
|
||||
|
||||
for i := range it.pBuckets {
|
||||
delta, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.pBucketsDelta[i] = delta
|
||||
it.pBuckets[i] = it.pBuckets[i] + delta
|
||||
}
|
||||
|
||||
for i := range it.nBuckets {
|
||||
delta, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.nBucketsDelta[i] = delta
|
||||
it.nBuckets[i] = it.nBuckets[i] + delta
|
||||
}
|
||||
|
||||
it.numRead++
|
||||
return true
|
||||
}
|
||||
|
||||
tDod, err := readVarbitInt(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.tDelta = it.tDelta + tDod
|
||||
it.t += it.tDelta
|
||||
|
||||
cntDod, err := readVarbitInt(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.cntDelta = it.cntDelta + cntDod
|
||||
it.cnt = uint64(int64(it.cnt) + it.cntDelta)
|
||||
|
||||
zcntDod, err := readVarbitInt(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.zCntDelta = it.zCntDelta + zcntDod
|
||||
it.zCnt = uint64(int64(it.zCnt) + it.zCntDelta)
|
||||
|
||||
ok := it.readSum()
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
if value.IsStaleNaN(it.sum) {
|
||||
it.numRead++
|
||||
return true
|
||||
}
|
||||
|
||||
for i := range it.pBuckets {
|
||||
dod, err := readVarbitInt(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.pBucketsDelta[i] = it.pBucketsDelta[i] + dod
|
||||
it.pBuckets[i] = it.pBuckets[i] + it.pBucketsDelta[i]
|
||||
}
|
||||
|
||||
for i := range it.nBuckets {
|
||||
dod, err := readVarbitInt(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.nBucketsDelta[i] = it.nBucketsDelta[i] + dod
|
||||
it.nBuckets[i] = it.nBuckets[i] + it.nBucketsDelta[i]
|
||||
}
|
||||
|
||||
it.numRead++
|
||||
return true
|
||||
}
|
||||
|
||||
func (it *histogramIterator) readSum() bool {
|
||||
bit, err := it.br.readBitFast()
|
||||
if err != nil {
|
||||
bit, err = it.br.readBit()
|
||||
}
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
|
||||
if bit == zero {
|
||||
return true // it.sum = it.sum
|
||||
}
|
||||
|
||||
bit, err = it.br.readBitFast()
|
||||
if err != nil {
|
||||
bit, err = it.br.readBit()
|
||||
}
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
if bit == zero {
|
||||
// Reuse leading/trailing zero bits.
|
||||
// it.leading, it.trailing = it.leading, it.trailing
|
||||
} else {
|
||||
bits, err := it.br.readBitsFast(5)
|
||||
if err != nil {
|
||||
bits, err = it.br.readBits(5)
|
||||
}
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.leading = uint8(bits)
|
||||
|
||||
bits, err = it.br.readBitsFast(6)
|
||||
if err != nil {
|
||||
bits, err = it.br.readBits(6)
|
||||
}
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
mbits := uint8(bits)
|
||||
// 0 significant bits here means we overflowed and we actually
|
||||
// need 64; see comment in encoder.
|
||||
if mbits == 0 {
|
||||
mbits = 64
|
||||
}
|
||||
it.trailing = 64 - it.leading - mbits
|
||||
}
|
||||
|
||||
mbits := 64 - it.leading - it.trailing
|
||||
bits, err := it.br.readBitsFast(mbits)
|
||||
if err != nil {
|
||||
bits, err = it.br.readBits(mbits)
|
||||
}
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
vbits := math.Float64bits(it.sum)
|
||||
vbits ^= bits << it.trailing
|
||||
it.sum = math.Float64frombits(vbits)
|
||||
return true
|
||||
}
|
||||
286
tsdb/chunkenc/histogram_meta.go
Normal file
286
tsdb/chunkenc/histogram_meta.go
Normal file
|
|
@ -0,0 +1,286 @@
|
|||
// Copyright 2021 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package chunkenc
|
||||
|
||||
import (
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
)
|
||||
|
||||
func writeHistogramChunkMeta(b *bstream, schema int32, zeroThreshold float64, positiveSpans, negativeSpans []histogram.Span) {
|
||||
putVarbitInt(b, int64(schema))
|
||||
putVarbitFloat(b, zeroThreshold)
|
||||
putHistogramChunkMetaSpans(b, positiveSpans)
|
||||
putHistogramChunkMetaSpans(b, negativeSpans)
|
||||
}
|
||||
|
||||
func putHistogramChunkMetaSpans(b *bstream, spans []histogram.Span) {
|
||||
putVarbitInt(b, int64(len(spans)))
|
||||
for _, s := range spans {
|
||||
putVarbitInt(b, int64(s.Length))
|
||||
putVarbitInt(b, int64(s.Offset))
|
||||
}
|
||||
}
|
||||
|
||||
func readHistogramChunkMeta(b *bstreamReader) (
|
||||
schema int32, zeroThreshold float64,
|
||||
positiveSpans, negativeSpans []histogram.Span,
|
||||
err error,
|
||||
) {
|
||||
_, err = b.ReadByte() // The header.
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
v, err := readVarbitInt(b)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
schema = int32(v)
|
||||
|
||||
zeroThreshold, err = readVarbitFloat(b)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
positiveSpans, err = readHistogramChunkMetaSpans(b)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
negativeSpans, err = readHistogramChunkMetaSpans(b)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func readHistogramChunkMetaSpans(b *bstreamReader) ([]histogram.Span, error) {
|
||||
var spans []histogram.Span
|
||||
num, err := readVarbitInt(b)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := 0; i < int(num); i++ {
|
||||
|
||||
length, err := readVarbitInt(b)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
offset, err := readVarbitInt(b)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
spans = append(spans, histogram.Span{
|
||||
Length: uint32(length),
|
||||
Offset: int32(offset),
|
||||
})
|
||||
}
|
||||
return spans, nil
|
||||
}
|
||||
|
||||
type bucketIterator struct {
|
||||
spans []histogram.Span
|
||||
span int // Span position of last yielded bucket.
|
||||
bucket int // Bucket position within span of last yielded bucket.
|
||||
idx int // Bucket index (globally across all spans) of last yielded bucket.
|
||||
}
|
||||
|
||||
func newBucketIterator(spans []histogram.Span) *bucketIterator {
|
||||
b := bucketIterator{
|
||||
spans: spans,
|
||||
span: 0,
|
||||
bucket: -1,
|
||||
idx: -1,
|
||||
}
|
||||
if len(spans) > 0 {
|
||||
b.idx += int(spans[0].Offset)
|
||||
}
|
||||
return &b
|
||||
}
|
||||
|
||||
func (b *bucketIterator) Next() (int, bool) {
|
||||
// We're already out of bounds.
|
||||
if b.span >= len(b.spans) {
|
||||
return 0, false
|
||||
}
|
||||
try:
|
||||
if b.bucket < int(b.spans[b.span].Length-1) { // Try to move within same span.
|
||||
b.bucket++
|
||||
b.idx++
|
||||
return b.idx, true
|
||||
} else if b.span < len(b.spans)-1 { // Try to move from one span to the next.
|
||||
b.span++
|
||||
b.idx += int(b.spans[b.span].Offset + 1)
|
||||
b.bucket = 0
|
||||
if b.spans[b.span].Length == 0 {
|
||||
// Pathological case that should never happen. We can't use this span, let's try again.
|
||||
goto try
|
||||
}
|
||||
return b.idx, true
|
||||
}
|
||||
// We're out of options.
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// An Interjection describes how many new buckets have to be introduced before
|
||||
// processing the pos'th delta from the original slice.
|
||||
type Interjection struct {
|
||||
pos int
|
||||
num int
|
||||
}
|
||||
|
||||
// compareSpans returns the interjections to convert a slice of deltas to a new
|
||||
// slice representing an expanded set of buckets, or false if incompatible
|
||||
// (e.g. if buckets were removed).
|
||||
//
|
||||
// Example:
|
||||
//
|
||||
// Let's say the old buckets look like this:
|
||||
//
|
||||
// span syntax: [offset, length]
|
||||
// spans : [ 0 , 2 ] [2,1] [ 3 , 2 ] [3,1] [1,1]
|
||||
// bucket idx : [0] [1] 2 3 [4] 5 6 7 [8] [9] 10 11 12 [13] 14 [15]
|
||||
// raw values 6 3 3 2 4 5 1
|
||||
// deltas 6 -3 0 -1 2 1 -4
|
||||
//
|
||||
// But now we introduce a new bucket layout. (Carefully chosen example where we
|
||||
// have a span appended, one unchanged[*], one prepended, and two merge - in
|
||||
// that order.)
|
||||
//
|
||||
// [*] unchanged in terms of which bucket indices they represent. but to achieve
|
||||
// that, their offset needs to change if "disrupted" by spans changing ahead of
|
||||
// them
|
||||
//
|
||||
// \/ this one is "unchanged"
|
||||
// spans : [ 0 , 3 ] [1,1] [ 1 , 4 ] [ 3 , 3 ]
|
||||
// bucket idx : [0] [1] [2] 3 [4] 5 [6] [7] [8] [9] 10 11 12 [13] [14] [15]
|
||||
// raw values 6 3 0 3 0 0 2 4 5 0 1
|
||||
// deltas 6 -3 -3 3 -3 0 2 2 1 -5 1
|
||||
// delta mods: / \ / \ / \
|
||||
//
|
||||
// Note that whenever any new buckets are introduced, the subsequent "old"
|
||||
// bucket needs to readjust its delta to the new base of 0. Thus, for the caller
|
||||
// who wants to transform the set of original deltas to a new set of deltas to
|
||||
// match a new span layout that adds buckets, we simply need to generate a list
|
||||
// of interjections.
|
||||
//
|
||||
// Note: Within compareSpans we don't have to worry about the changes to the
|
||||
// spans themselves, thanks to the iterators we get to work with the more useful
|
||||
// bucket indices (which of course directly correspond to the buckets we have to
|
||||
// adjust).
|
||||
func compareSpans(a, b []histogram.Span) ([]Interjection, bool) {
|
||||
ai := newBucketIterator(a)
|
||||
bi := newBucketIterator(b)
|
||||
|
||||
var interjections []Interjection
|
||||
|
||||
// When inter.num becomes > 0, this becomes a valid interjection that
|
||||
// should be yielded when we finish a streak of new buckets.
|
||||
var inter Interjection
|
||||
|
||||
av, aOK := ai.Next()
|
||||
bv, bOK := bi.Next()
|
||||
loop:
|
||||
for {
|
||||
switch {
|
||||
case aOK && bOK:
|
||||
switch {
|
||||
case av == bv: // Both have an identical value. move on!
|
||||
// Finish WIP interjection and reset.
|
||||
if inter.num > 0 {
|
||||
interjections = append(interjections, inter)
|
||||
}
|
||||
inter.num = 0
|
||||
av, aOK = ai.Next()
|
||||
bv, bOK = bi.Next()
|
||||
inter.pos++
|
||||
case av < bv: // b misses a value that is in a.
|
||||
return interjections, false
|
||||
case av > bv: // a misses a value that is in b. Forward b and recompare.
|
||||
inter.num++
|
||||
bv, bOK = bi.Next()
|
||||
}
|
||||
case aOK && !bOK: // b misses a value that is in a.
|
||||
return interjections, false
|
||||
case !aOK && bOK: // a misses a value that is in b. Forward b and recompare.
|
||||
inter.num++
|
||||
bv, bOK = bi.Next()
|
||||
default: // Both iterators ran out. We're done.
|
||||
if inter.num > 0 {
|
||||
interjections = append(interjections, inter)
|
||||
}
|
||||
break loop
|
||||
}
|
||||
}
|
||||
|
||||
return interjections, true
|
||||
}
|
||||
|
||||
// interject merges 'in' with the provided interjections and writes them into
|
||||
// 'out', which must already have the appropriate length.
|
||||
func interject(in, out []int64, interjections []Interjection) []int64 {
|
||||
var (
|
||||
j int // Position in out.
|
||||
v int64 // The last value seen.
|
||||
interj int // The next interjection to process.
|
||||
)
|
||||
for i, d := range in {
|
||||
if interj < len(interjections) && i == interjections[interj].pos {
|
||||
|
||||
// We have an interjection!
|
||||
// Add interjection.num new delta values such that their
|
||||
// bucket values equate 0.
|
||||
out[j] = int64(-v)
|
||||
j++
|
||||
for x := 1; x < interjections[interj].num; x++ {
|
||||
out[j] = 0
|
||||
j++
|
||||
}
|
||||
interj++
|
||||
|
||||
// Now save the value from the input. The delta value we
|
||||
// should save is the original delta value + the last
|
||||
// value of the point before the interjection (to undo
|
||||
// the delta that was introduced by the interjection).
|
||||
out[j] = d + v
|
||||
j++
|
||||
v = d + v
|
||||
continue
|
||||
}
|
||||
|
||||
// If there was no interjection, the original delta is still
|
||||
// valid.
|
||||
out[j] = d
|
||||
j++
|
||||
v += d
|
||||
}
|
||||
switch interj {
|
||||
case len(interjections):
|
||||
// All interjections processed. Nothing more to do.
|
||||
case len(interjections) - 1:
|
||||
// One more interjection to process at the end.
|
||||
out[j] = int64(-v)
|
||||
j++
|
||||
for x := 1; x < interjections[interj].num; x++ {
|
||||
out[j] = 0
|
||||
j++
|
||||
}
|
||||
default:
|
||||
panic("unprocessed interjections left")
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
|
@ -21,13 +21,15 @@ package chunkenc
|
|||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// example of a span layout and resulting bucket indices (_idx_ is used in this histogram, others are shown just for context)
|
||||
// spans : [offset: 0, length: 2] [offset 1, length 1]
|
||||
// bucket idx : _0_ _1_ 2 [3] 4 ...
|
||||
// Example of a span layout and resulting bucket indices (_idx_ is used in this
|
||||
// histogram, others are shown just for context):
|
||||
//
|
||||
// spans : [offset: 0, length: 2] [offset 1, length 1]
|
||||
// bucket idx : _0_ _1_ 2 [3] 4 ...
|
||||
|
||||
func TestBucketIterator(t *testing.T) {
|
||||
type test struct {
|
||||
|
|
@ -74,7 +76,7 @@ func TestBucketIterator(t *testing.T) {
|
|||
},
|
||||
idxs: []int{100, 101, 102, 103, 112, 113, 114, 115, 116, 117, 118, 119},
|
||||
},
|
||||
// the below 2 sets ore the ones described in compareSpans's comments
|
||||
// The below 2 sets ore the ones described in compareSpans's comments.
|
||||
{
|
||||
spans: []histogram.Span{
|
||||
{Offset: 0, Length: 2},
|
||||
|
|
@ -16,21 +16,21 @@ package chunkenc
|
|||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestHistoChunkSameBuckets(t *testing.T) {
|
||||
c := NewHistoChunk()
|
||||
func TestHistogramChunkSameBuckets(t *testing.T) {
|
||||
c := NewHistogramChunk()
|
||||
var exp []res
|
||||
|
||||
// Create fresh appender and add the first histogram
|
||||
// Create fresh appender and add the first histogram.
|
||||
app, err := c.Appender()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, c.NumSamples())
|
||||
|
||||
ts := int64(1234567890)
|
||||
h := histogram.SparseHistogram{
|
||||
h := histogram.Histogram{
|
||||
Count: 5,
|
||||
ZeroCount: 2,
|
||||
Sum: 18.4,
|
||||
|
|
@ -56,7 +56,7 @@ func TestHistoChunkSameBuckets(t *testing.T) {
|
|||
exp = append(exp, res{t: ts, h: h})
|
||||
require.Equal(t, 2, c.NumSamples())
|
||||
|
||||
// Add update with new appender
|
||||
// Add update with new appender.
|
||||
app, err = c.Appender()
|
||||
require.NoError(t, err)
|
||||
|
||||
|
|
@ -113,12 +113,12 @@ func TestHistoChunkSameBuckets(t *testing.T) {
|
|||
|
||||
type res struct {
|
||||
t int64
|
||||
h histogram.SparseHistogram
|
||||
h histogram.Histogram
|
||||
}
|
||||
|
||||
// mimics the scenario described for compareSpans()
|
||||
func TestHistoChunkBucketChanges(t *testing.T) {
|
||||
c := Chunk(NewHistoChunk())
|
||||
// Mimics the scenario described for compareSpans().
|
||||
func TestHistogramChunkBucketChanges(t *testing.T) {
|
||||
c := Chunk(NewHistogramChunk())
|
||||
|
||||
// Create fresh appender and add the first histogram.
|
||||
app, err := c.Appender()
|
||||
|
|
@ -126,7 +126,7 @@ func TestHistoChunkBucketChanges(t *testing.T) {
|
|||
require.Equal(t, 0, c.NumSamples())
|
||||
|
||||
ts1 := int64(1234567890)
|
||||
h1 := histogram.SparseHistogram{
|
||||
h1 := histogram.Histogram{
|
||||
Count: 5,
|
||||
ZeroCount: 2,
|
||||
Sum: 18.4,
|
||||
|
|
@ -157,24 +157,26 @@ func TestHistoChunkBucketChanges(t *testing.T) {
|
|||
h2.Count += 9
|
||||
h2.ZeroCount++
|
||||
h2.Sum = 30
|
||||
// Existing histogram should get values converted from the above to: 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between)
|
||||
// Existing histogram should get values converted from the above to:
|
||||
// 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between)
|
||||
// so the new histogram should have new counts >= these per-bucket counts, e.g.:
|
||||
h2.PositiveBuckets = []int64{7, -2, -4, 2, -2, -1, 2, 3, 0, -5, 1} // 7 5 1 3 1 0 2 5 5 0 1 (total 30)
|
||||
|
||||
// This is how span changes will be handled.
|
||||
histoApp, _ := app.(*HistoAppender)
|
||||
posInterjections, negInterjections, ok, cr := histoApp.Appendable(h2)
|
||||
hApp, _ := app.(*HistogramAppender)
|
||||
posInterjections, negInterjections, ok, cr := hApp.Appendable(h2)
|
||||
require.Greater(t, len(posInterjections), 0)
|
||||
require.Equal(t, 0, len(negInterjections))
|
||||
require.True(t, ok) // Only new buckets came in.
|
||||
require.False(t, cr)
|
||||
c, app = histoApp.Recode(posInterjections, negInterjections, h2.PositiveSpans, h2.NegativeSpans)
|
||||
c, app = hApp.Recode(posInterjections, negInterjections, h2.PositiveSpans, h2.NegativeSpans)
|
||||
app.AppendHistogram(ts2, h2)
|
||||
|
||||
require.Equal(t, 2, c.NumSamples())
|
||||
|
||||
// Because the 2nd histogram has expanded buckets, we should expect all histograms (in particular the first)
|
||||
// to come back using the new spans metadata as well as the expanded buckets.
|
||||
// Because the 2nd histogram has expanded buckets, we should expect all
|
||||
// histograms (in particular the first) to come back using the new spans
|
||||
// metadata as well as the expanded buckets.
|
||||
h1.PositiveSpans = h2.PositiveSpans
|
||||
h1.PositiveBuckets = []int64{6, -3, -3, 3, -3, 0, 2, 2, 1, -5, 1}
|
||||
exp := []res{
|
||||
|
|
@ -192,7 +194,7 @@ func TestHistoChunkBucketChanges(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestHistoChunkAppendable(t *testing.T) {
|
||||
c := Chunk(NewHistoChunk())
|
||||
c := Chunk(NewHistogramChunk())
|
||||
|
||||
// Create fresh appender and add the first histogram.
|
||||
app, err := c.Appender()
|
||||
|
|
@ -200,7 +202,7 @@ func TestHistoChunkAppendable(t *testing.T) {
|
|||
require.Equal(t, 0, c.NumSamples())
|
||||
|
||||
ts := int64(1234567890)
|
||||
h1 := histogram.SparseHistogram{
|
||||
h1 := histogram.Histogram{
|
||||
Count: 5,
|
||||
ZeroCount: 2,
|
||||
Sum: 18.4,
|
||||
|
|
@ -230,12 +232,13 @@ func TestHistoChunkAppendable(t *testing.T) {
|
|||
h2.Count += 9
|
||||
h2.ZeroCount++
|
||||
h2.Sum = 30
|
||||
// Existing histogram should get values converted from the above to: 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between)
|
||||
// Existing histogram should get values converted from the above to:
|
||||
// 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between)
|
||||
// so the new histogram should have new counts >= these per-bucket counts, e.g.:
|
||||
h2.PositiveBuckets = []int64{7, -2, -4, 2, -2, -1, 2, 3, 0, -5, 1} // 7 5 1 3 1 0 2 5 5 0 1 (total 30)
|
||||
|
||||
histoApp, _ := app.(*HistoAppender)
|
||||
posInterjections, negInterjections, ok, cr := histoApp.Appendable(h2)
|
||||
hApp, _ := app.(*HistogramAppender)
|
||||
posInterjections, negInterjections, ok, cr := hApp.Appendable(h2)
|
||||
require.Greater(t, len(posInterjections), 0)
|
||||
require.Equal(t, 0, len(negInterjections))
|
||||
require.True(t, ok) // Only new buckets came in.
|
||||
|
|
@ -253,8 +256,8 @@ func TestHistoChunkAppendable(t *testing.T) {
|
|||
h2.Sum = 21
|
||||
h2.PositiveBuckets = []int64{6, -3, -1, 2, 1, -4} // counts: 6, 3, 2, 4, 5, 1 (total 21)
|
||||
|
||||
histoApp, _ := app.(*HistoAppender)
|
||||
posInterjections, negInterjections, ok, cr := histoApp.Appendable(h2)
|
||||
hApp, _ := app.(*HistogramAppender)
|
||||
posInterjections, negInterjections, ok, cr := hApp.Appendable(h2)
|
||||
require.Equal(t, 0, len(posInterjections))
|
||||
require.Equal(t, 0, len(negInterjections))
|
||||
require.False(t, ok) // Need to cut a new chunk.
|
||||
|
|
@ -266,8 +269,8 @@ func TestHistoChunkAppendable(t *testing.T) {
|
|||
h2.Sum = 23
|
||||
h2.PositiveBuckets = []int64{6, -4, 1, -1, 2, 1, -4} // counts: 6, 2, 3, 2, 4, 5, 1 (total 23)
|
||||
|
||||
histoApp, _ := app.(*HistoAppender)
|
||||
posInterjections, negInterjections, ok, cr := histoApp.Appendable(h2)
|
||||
hApp, _ := app.(*HistogramAppender)
|
||||
posInterjections, negInterjections, ok, cr := hApp.Appendable(h2)
|
||||
require.Equal(t, 0, len(posInterjections))
|
||||
require.Equal(t, 0, len(negInterjections))
|
||||
require.False(t, ok) // Need to cut a new chunk.
|
||||
|
|
@ -283,20 +286,24 @@ func TestHistoChunkAppendable(t *testing.T) {
|
|||
{Offset: 3, Length: 3},
|
||||
}
|
||||
h2.Sum = 29
|
||||
// Existing histogram should get values converted from the above to: 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between)
|
||||
// Existing histogram should get values converted from the above to:
|
||||
// 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between)
|
||||
// so the new histogram should have new counts >= these per-bucket counts, e.g.:
|
||||
h2.PositiveBuckets = []int64{7, -2, -4, 2, -2, -1, 2, 3, 0, -5, 0} // 7 5 1 3 1 0 2 5 5 0 0 (total 29)
|
||||
|
||||
histoApp, _ := app.(*HistoAppender)
|
||||
posInterjections, negInterjections, ok, cr := histoApp.Appendable(h2)
|
||||
hApp, _ := app.(*HistogramAppender)
|
||||
posInterjections, negInterjections, ok, cr := hApp.Appendable(h2)
|
||||
require.Equal(t, 0, len(posInterjections))
|
||||
require.Equal(t, 0, len(negInterjections))
|
||||
require.False(t, ok) // Need to cut a new chunk.
|
||||
require.True(t, cr)
|
||||
}
|
||||
|
||||
{ // New histogram that has a counter reset while new buckets were added before the first bucket and reset on first bucket.
|
||||
// (to catch the edge case where the new bucket should be forwarded ahead until first old bucket at start)
|
||||
{
|
||||
// New histogram that has a counter reset while new buckets were
|
||||
// added before the first bucket and reset on first bucket. (to
|
||||
// catch the edge case where the new bucket should be forwarded
|
||||
// ahead until first old bucket at start)
|
||||
h2 := h1
|
||||
h2.PositiveSpans = []histogram.Span{
|
||||
{Offset: -3, Length: 2},
|
||||
|
|
@ -307,12 +314,13 @@ func TestHistoChunkAppendable(t *testing.T) {
|
|||
{Offset: 1, Length: 1},
|
||||
}
|
||||
h2.Sum = 26
|
||||
// Existing histogram should get values converted from the above to: 0, 0, 6, 3, 3, 2, 4, 5, 1
|
||||
// Existing histogram should get values converted from the above to:
|
||||
// 0, 0, 6, 3, 3, 2, 4, 5, 1
|
||||
// so the new histogram should have new counts >= these per-bucket counts, e.g.:
|
||||
h2.PositiveBuckets = []int64{1, 1, 3, -2, 0, -1, 2, 1, -4} // counts: 1, 2, 5, 3, 3, 2, 4, 5, 1 (total 26)
|
||||
|
||||
histoApp, _ := app.(*HistoAppender)
|
||||
posInterjections, negInterjections, ok, cr := histoApp.Appendable(h2)
|
||||
hApp, _ := app.(*HistogramAppender)
|
||||
posInterjections, negInterjections, ok, cr := hApp.Appendable(h2)
|
||||
require.Equal(t, 0, len(posInterjections))
|
||||
require.Equal(t, 0, len(negInterjections))
|
||||
require.False(t, ok) // Need to cut a new chunk.
|
||||
143
tsdb/chunkenc/varbit.go
Normal file
143
tsdb/chunkenc/varbit.go
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
// Copyright 2021 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package chunkenc
|
||||
|
||||
import (
|
||||
"math"
|
||||
)
|
||||
|
||||
// putVarbitFloat writes a float64 using varbit encoding. It does so by
|
||||
// converting the underlying bits into an int64.
|
||||
func putVarbitFloat(b *bstream, val float64) {
|
||||
// TODO(beorn7): The resulting int64 here will almost never be a small
|
||||
// integer. Thus, the varbit encoding doesn't really make sense
|
||||
// here. This function is only used to encode the zero threshold in
|
||||
// histograms. Based on that, here is an idea to improve the encoding:
|
||||
//
|
||||
// It is recommended to use (usually negative) powers of two as
|
||||
// threshoulds. The default value for the zero threshald is in fact
|
||||
// 2^-128, or 0.5*2^-127, as it is represented by IEEE 754. It is
|
||||
// therefore worth a try to test if the threshold is a power of 2 and
|
||||
// then just store the exponent. 0 is also a commen threshold for those
|
||||
// use cases where only observations of precisely zero should go to the
|
||||
// zero bucket. This results in the following proposal:
|
||||
// - First we store 1 byte.
|
||||
// - Iff that byte is 255 (all bits set), it is followed by a direct
|
||||
// 8byte representation of the float.
|
||||
// - If the byte is 0, the threshold is 0.
|
||||
// - In all other cases, take the number represented by the byte,
|
||||
// subtract 246, and that's the exponent (i.e. between -245 and
|
||||
// +8, covering thresholds that are powers of 2 between 2^-246
|
||||
// to 128).
|
||||
putVarbitInt(b, int64(math.Float64bits(val)))
|
||||
}
|
||||
|
||||
// readVarbitFloat reads a float64 encoded with putVarbitFloat
|
||||
func readVarbitFloat(b *bstreamReader) (float64, error) {
|
||||
val, err := readVarbitInt(b)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return math.Float64frombits(uint64(val)), nil
|
||||
}
|
||||
|
||||
// putVarbitInt writes an int64 using varbit encoding with a bit bucketing
|
||||
// optimized for the dod's observed in histogram buckets.
|
||||
//
|
||||
// TODO(Dieterbe): We could improve this further: Each branch doesn't need to
|
||||
// support any values of any of the prior branches. So we can expand the range
|
||||
// of each branch. Do more with fewer bits. It comes at the price of more
|
||||
// expensive encoding and decoding (cutting out and later adding back that
|
||||
// center-piece we skip).
|
||||
func putVarbitInt(b *bstream, val int64) {
|
||||
switch {
|
||||
case val == 0:
|
||||
b.writeBit(zero)
|
||||
case bitRange(val, 3): // -3 <= val <= 4
|
||||
b.writeBits(0b10, 2)
|
||||
b.writeBits(uint64(val), 3)
|
||||
case bitRange(val, 6): // -31 <= val <= 32
|
||||
b.writeBits(0b110, 3)
|
||||
b.writeBits(uint64(val), 6)
|
||||
case bitRange(val, 9): // -255 <= val <= 256
|
||||
b.writeBits(0b1110, 4)
|
||||
b.writeBits(uint64(val), 9)
|
||||
case bitRange(val, 12): // -2047 <= val <= 2048
|
||||
b.writeBits(0b11110, 5)
|
||||
b.writeBits(uint64(val), 12)
|
||||
default:
|
||||
b.writeBits(0b11111, 5)
|
||||
b.writeBits(uint64(val), 64)
|
||||
}
|
||||
}
|
||||
|
||||
// readVarbitInt reads an int64 encoced with putVarbitInt.
|
||||
func readVarbitInt(b *bstreamReader) (int64, error) {
|
||||
var d byte
|
||||
for i := 0; i < 5; i++ {
|
||||
d <<= 1
|
||||
bit, err := b.readBitFast()
|
||||
if err != nil {
|
||||
bit, err = b.readBit()
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if bit == zero {
|
||||
break
|
||||
}
|
||||
d |= 1
|
||||
}
|
||||
|
||||
var val int64
|
||||
var sz uint8
|
||||
|
||||
switch d {
|
||||
case 0b0:
|
||||
// val == 0
|
||||
case 0b10:
|
||||
sz = 3
|
||||
case 0b110:
|
||||
sz = 6
|
||||
case 0b1110:
|
||||
sz = 9
|
||||
case 0b11110:
|
||||
sz = 12
|
||||
case 0b11111:
|
||||
// Do not use fast because it's very unlikely it will succeed.
|
||||
bits, err := b.readBits(64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
val = int64(bits)
|
||||
}
|
||||
|
||||
if sz != 0 {
|
||||
bits, err := b.readBitsFast(sz)
|
||||
if err != nil {
|
||||
bits, err = b.readBits(sz)
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if bits > (1 << (sz - 1)) {
|
||||
// Or something.
|
||||
bits = bits - (1 << sz)
|
||||
}
|
||||
val = int64(bits)
|
||||
}
|
||||
|
||||
return val, nil
|
||||
}
|
||||
|
|
@ -1,155 +0,0 @@
|
|||
// Copyright 2021 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// The code in this file was largely written by Damian Gryski as part of
|
||||
// https://github.com/dgryski/go-tsz and published under the license below.
|
||||
// It was modified to accommodate reading from byte slices without modifying
|
||||
// the underlying bytes, which would panic when reading from mmap'd
|
||||
// read-only byte slices.
|
||||
|
||||
// Copyright (c) 2015,2016 Damian Gryski <damian@gryski.com>
|
||||
// All rights reserved.
|
||||
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
|
||||
// * Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimer.
|
||||
//
|
||||
// * Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
package chunkenc
|
||||
|
||||
import (
|
||||
"math"
|
||||
)
|
||||
|
||||
// putFloat64VBBucket writes a float64 using varbit optimized for SHS buckets.
|
||||
// It does so by converting the underlying bits into an int64.
|
||||
func putFloat64VBBucket(b *bstream, val float64) {
|
||||
// TODO: Since this is used for the zero threshold, this almost always goes into the default
|
||||
// bit range (i.e. using 5+64 bits). So we can consider skipping `putInt64VBBucket` and directly
|
||||
// write the float and save 5 bits here.
|
||||
putInt64VBBucket(b, int64(math.Float64bits(val)))
|
||||
}
|
||||
|
||||
// readFloat64VBBucket reads a float64 using varbit optimized for SHS buckets
|
||||
func readFloat64VBBucket(b *bstreamReader) (float64, error) {
|
||||
val, err := readInt64VBBucket(b)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return math.Float64frombits(uint64(val)), nil
|
||||
}
|
||||
|
||||
// putInt64VBBucket writes an int64 using varbit optimized for SHS buckets.
|
||||
//
|
||||
// TODO(Dieterbe): We could improve this further: Each branch doesn't need to
|
||||
// support any values of any of the prior branches. So we can expand the range
|
||||
// of each branch. Do more with fewer bits. It comes at the price of more
|
||||
// expensive encoding and decoding (cutting out and later adding back that
|
||||
// center-piece we skip).
|
||||
func putInt64VBBucket(b *bstream, val int64) {
|
||||
switch {
|
||||
case val == 0:
|
||||
b.writeBit(zero)
|
||||
case bitRange(val, 3): // -3 <= val <= 4
|
||||
b.writeBits(0b10, 2)
|
||||
b.writeBits(uint64(val), 3)
|
||||
case bitRange(val, 6): // -31 <= val <= 32
|
||||
b.writeBits(0b110, 3)
|
||||
b.writeBits(uint64(val), 6)
|
||||
case bitRange(val, 9): // -255 <= val <= 256
|
||||
b.writeBits(0b1110, 4)
|
||||
b.writeBits(uint64(val), 9)
|
||||
case bitRange(val, 12): // -2047 <= val <= 2048
|
||||
b.writeBits(0b11110, 5)
|
||||
b.writeBits(uint64(val), 12)
|
||||
default:
|
||||
b.writeBits(0b11111, 5)
|
||||
b.writeBits(uint64(val), 64)
|
||||
}
|
||||
}
|
||||
|
||||
// readInt64VBBucket reads an int64 using varbit optimized for SHS buckets
|
||||
func readInt64VBBucket(b *bstreamReader) (int64, error) {
|
||||
var d byte
|
||||
for i := 0; i < 5; i++ {
|
||||
d <<= 1
|
||||
bit, err := b.readBitFast()
|
||||
if err != nil {
|
||||
bit, err = b.readBit()
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if bit == zero {
|
||||
break
|
||||
}
|
||||
d |= 1
|
||||
}
|
||||
|
||||
var val int64
|
||||
var sz uint8
|
||||
|
||||
switch d {
|
||||
case 0b0:
|
||||
// val == 0
|
||||
case 0b10:
|
||||
sz = 3
|
||||
case 0b110:
|
||||
sz = 6
|
||||
case 0b1110:
|
||||
sz = 9
|
||||
case 0b11110:
|
||||
sz = 12
|
||||
case 0b11111:
|
||||
// Do not use fast because it's very unlikely it will succeed.
|
||||
bits, err := b.readBits(64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
val = int64(bits)
|
||||
}
|
||||
|
||||
if sz != 0 {
|
||||
bits, err := b.readBitsFast(sz)
|
||||
if err != nil {
|
||||
bits, err = b.readBits(sz)
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if bits > (1 << (sz - 1)) {
|
||||
// or something
|
||||
bits = bits - (1 << sz)
|
||||
}
|
||||
val = int64(bits)
|
||||
}
|
||||
|
||||
return val, nil
|
||||
}
|
||||
|
|
@ -48,7 +48,7 @@ import (
|
|||
"math"
|
||||
"math/bits"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -150,8 +150,8 @@ type xorAppender struct {
|
|||
trailing uint8
|
||||
}
|
||||
|
||||
func (a *xorAppender) AppendHistogram(t int64, h histogram.SparseHistogram) {
|
||||
//panic("cannot call xorAppender.AppendHistogram().")
|
||||
func (a *xorAppender) AppendHistogram(t int64, h histogram.Histogram) {
|
||||
panic("appended a histogram to an xor chunk")
|
||||
}
|
||||
|
||||
func (a *xorAppender) Append(t int64, v float64) {
|
||||
|
|
@ -181,6 +181,12 @@ func (a *xorAppender) Append(t int64, v float64) {
|
|||
|
||||
// Gorilla has a max resolution of seconds, Prometheus milliseconds.
|
||||
// Thus we use higher value range steps with larger bit size.
|
||||
//
|
||||
// TODO(beorn7): This seems to needlessly jump to large bit
|
||||
// sizes even for very small deviations from zero. Timestamp
|
||||
// compression can probably benefit from some smaller bit
|
||||
// buckets. See also what was done for histogram encoding in
|
||||
// varbit.go.
|
||||
switch {
|
||||
case dod == 0:
|
||||
a.b.writeBit(zero)
|
||||
|
|
@ -278,7 +284,7 @@ func (it *xorIterator) At() (int64, float64) {
|
|||
return it.t, it.val
|
||||
}
|
||||
|
||||
func (it *xorIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
func (it *xorIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
panic("cannot call xorIterator.AtHistogram().")
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ import (
|
|||
prom_testutil "github.com/prometheus/client_golang/prometheus/testutil"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
|
|
@ -1328,7 +1328,7 @@ func TestHeadCompactionWithHistograms(t *testing.T) {
|
|||
|
||||
type timedHist struct {
|
||||
t int64
|
||||
h histogram.SparseHistogram
|
||||
h histogram.Histogram
|
||||
}
|
||||
|
||||
// Ingest samples.
|
||||
|
|
@ -1384,7 +1384,7 @@ func TestHeadCompactionWithHistograms(t *testing.T) {
|
|||
// Depending on numSeriesPerSchema, it can take few gigs of memory;
|
||||
// the test adds all samples to appender before committing instead of
|
||||
// buffering the writes to make it run faster.
|
||||
func TestSparseHistoSpaceSavings(t *testing.T) {
|
||||
func TestSparseHistogramSpaceSavings(t *testing.T) {
|
||||
t.Skip()
|
||||
|
||||
cases := []struct {
|
||||
|
|
@ -1455,7 +1455,7 @@ func TestSparseHistoSpaceSavings(t *testing.T) {
|
|||
|
||||
var allSparseSeries []struct {
|
||||
baseLabels labels.Labels
|
||||
hists []histogram.SparseHistogram
|
||||
hists []histogram.Histogram
|
||||
}
|
||||
|
||||
for sid, schema := range allSchemas {
|
||||
|
|
@ -1467,7 +1467,7 @@ func TestSparseHistoSpaceSavings(t *testing.T) {
|
|||
}
|
||||
allSparseSeries = append(allSparseSeries, struct {
|
||||
baseLabels labels.Labels
|
||||
hists []histogram.SparseHistogram
|
||||
hists []histogram.Histogram
|
||||
}{baseLabels: lbls, hists: generateCustomHistograms(numHistograms, c.numBuckets, c.numSpans, c.gapBetweenSpans, schema)})
|
||||
}
|
||||
}
|
||||
|
|
@ -1522,13 +1522,13 @@ func TestSparseHistoSpaceSavings(t *testing.T) {
|
|||
h := ah.hists[i]
|
||||
|
||||
numOldSeriesPerHistogram = 0
|
||||
it := histogram.CumulativeExpandSparseHistogram(h)
|
||||
it := h.CumulativeBucketIterator()
|
||||
itIdx := 0
|
||||
var err error
|
||||
for it.Next() {
|
||||
numOldSeriesPerHistogram++
|
||||
b := it.At()
|
||||
lbls := append(ah.baseLabels, labels.Label{Name: "le", Value: fmt.Sprintf("%.16f", b.Le)})
|
||||
lbls := append(ah.baseLabels, labels.Label{Name: "le", Value: fmt.Sprintf("%.16f", b.Upper)})
|
||||
refs[itIdx], err = oldApp.Append(refs[itIdx], lbls, ts, float64(b.Count))
|
||||
require.NoError(t, err)
|
||||
itIdx++
|
||||
|
|
@ -1614,9 +1614,9 @@ Savings: Index=%.2f%%, Chunks=%.2f%%, Total=%.2f%%
|
|||
}
|
||||
}
|
||||
|
||||
func generateCustomHistograms(numHists, numBuckets, numSpans, gapBetweenSpans, schema int) (r []histogram.SparseHistogram) {
|
||||
func generateCustomHistograms(numHists, numBuckets, numSpans, gapBetweenSpans, schema int) (r []histogram.Histogram) {
|
||||
// First histogram with all the settings.
|
||||
h := histogram.SparseHistogram{
|
||||
h := histogram.Histogram{
|
||||
Sum: 1000 * rand.Float64(),
|
||||
Schema: int32(schema),
|
||||
}
|
||||
|
|
@ -1711,7 +1711,7 @@ func TestSparseHistogramCompactionAndQuery(t *testing.T) {
|
|||
|
||||
type timedHist struct {
|
||||
t int64
|
||||
h histogram.SparseHistogram
|
||||
h histogram.Histogram
|
||||
}
|
||||
expHists := make(map[string][]timedHist)
|
||||
|
||||
|
|
|
|||
|
|
@ -34,8 +34,14 @@ in-file offset (lower 4 bytes) and segment sequence number (upper 4 bytes).
|
|||
└───────────────┴───────────────────┴──────────────┴────────────────┘
|
||||
```
|
||||
|
||||
## XOR chunk
|
||||
|
||||
TODO(beorn7): Add.
|
||||
|
||||
## Histogram chunk
|
||||
|
||||
TODO(beorn7): This is out of date. Update once settled on the (more or less) final format.
|
||||
|
||||
```
|
||||
┌──────────────┬─────────────────┬──────────────────────────┬──────────────────────────┬──────────────┐
|
||||
│ len <uint16> │ schema <varint> │ pos-spans <span-section> │ neg-spans <span-section> │ data <bytes> │
|
||||
|
|
|
|||
14
tsdb/head.go
14
tsdb/head.go
|
|
@ -29,8 +29,8 @@ import (
|
|||
"go.uber.org/atomic"
|
||||
|
||||
"github.com/prometheus/prometheus/config"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
|
|
@ -614,7 +614,7 @@ func (h *Head) Init(minValidTime int64) error {
|
|||
sparseHistogramSeries := 0
|
||||
for _, m := range h.series.series {
|
||||
for _, ms := range m {
|
||||
if ms.sparseHistogramSeries {
|
||||
if ms.histogramSeries {
|
||||
sparseHistogramSeries++
|
||||
}
|
||||
}
|
||||
|
|
@ -1397,7 +1397,7 @@ func (s *stripeSeries) gc(mint int64) (map[uint64]struct{}, int, int64, int) {
|
|||
s.locks[j].Lock()
|
||||
}
|
||||
|
||||
if series.sparseHistogramSeries {
|
||||
if series.histogramSeries {
|
||||
sparseHistogramSeriesDeleted++
|
||||
}
|
||||
deleted[series.ref] = struct{}{}
|
||||
|
|
@ -1488,9 +1488,9 @@ func (s *stripeSeries) getOrSet(hash uint64, lset labels.Labels, createSeries fu
|
|||
return series, true, nil
|
||||
}
|
||||
|
||||
type hist struct {
|
||||
type histogramSample struct {
|
||||
t int64
|
||||
h histogram.SparseHistogram
|
||||
h histogram.Histogram
|
||||
}
|
||||
|
||||
type sample struct {
|
||||
|
|
@ -1517,7 +1517,7 @@ type memSeries struct {
|
|||
|
||||
nextAt int64 // Timestamp at which to cut the next chunk.
|
||||
sampleBuf [4]sample
|
||||
histBuf [4]hist
|
||||
histogramBuf [4]histogramSample
|
||||
pendingCommit bool // Whether there are samples waiting to be committed to this series.
|
||||
|
||||
app chunkenc.Appender // Current appender for the chunk.
|
||||
|
|
@ -1527,7 +1527,7 @@ type memSeries struct {
|
|||
txs *txRing
|
||||
|
||||
// Temporary variable for sparsehistogram experiment.
|
||||
sparseHistogramSeries bool
|
||||
histogramSeries bool
|
||||
}
|
||||
|
||||
func newMemSeries(lset labels.Labels, id uint64, chunkRange int64, memChunkPool *sync.Pool) *memSeries {
|
||||
|
|
|
|||
|
|
@ -21,8 +21,8 @@ import (
|
|||
"github.com/go-kit/log/level"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/pkg/value"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
|
|
@ -67,14 +67,14 @@ func (a *initAppender) AppendExemplar(ref uint64, l labels.Labels, e exemplar.Ex
|
|||
return a.app.AppendExemplar(ref, l, e)
|
||||
}
|
||||
|
||||
func (a *initAppender) AppendHistogram(ref uint64, l labels.Labels, t int64, sh histogram.SparseHistogram) (uint64, error) {
|
||||
func (a *initAppender) AppendHistogram(ref uint64, l labels.Labels, t int64, h histogram.Histogram) (uint64, error) {
|
||||
if a.app != nil {
|
||||
return a.app.AppendHistogram(ref, l, t, sh)
|
||||
return a.app.AppendHistogram(ref, l, t, h)
|
||||
}
|
||||
a.head.initTime(t)
|
||||
a.app = a.head.appender()
|
||||
|
||||
return a.app.AppendHistogram(ref, l, t, sh)
|
||||
return a.app.AppendHistogram(ref, l, t, h)
|
||||
}
|
||||
|
||||
// initTime initializes a head with the first timestamp. This only needs to be called
|
||||
|
|
@ -269,8 +269,8 @@ func (a *headAppender) Append(ref uint64, lset labels.Labels, t int64, v float64
|
|||
}
|
||||
}
|
||||
|
||||
if value.IsStaleNaN(v) && s.sparseHistogramSeries {
|
||||
return a.AppendHistogram(ref, lset, t, histogram.SparseHistogram{Sum: v})
|
||||
if value.IsStaleNaN(v) && s.histogramSeries {
|
||||
return a.AppendHistogram(ref, lset, t, histogram.Histogram{Sum: v})
|
||||
}
|
||||
|
||||
s.Lock()
|
||||
|
|
@ -322,7 +322,7 @@ func (s *memSeries) appendable(t int64, v float64) error {
|
|||
}
|
||||
|
||||
// appendableHistogram checks whether the given sample is valid for appending to the series.
|
||||
func (s *memSeries) appendableHistogram(t int64, sh histogram.SparseHistogram) error {
|
||||
func (s *memSeries) appendableHistogram(t int64, sh histogram.Histogram) error {
|
||||
c := s.head()
|
||||
if c == nil {
|
||||
return nil
|
||||
|
|
@ -372,7 +372,7 @@ func (a *headAppender) AppendExemplar(ref uint64, _ labels.Labels, e exemplar.Ex
|
|||
return s.ref, nil
|
||||
}
|
||||
|
||||
func (a *headAppender) AppendHistogram(ref uint64, lset labels.Labels, t int64, sh histogram.SparseHistogram) (uint64, error) {
|
||||
func (a *headAppender) AppendHistogram(ref uint64, lset labels.Labels, t int64, h histogram.Histogram) (uint64, error) {
|
||||
if t < a.minValidTime {
|
||||
a.head.metrics.outOfBoundSamples.Inc()
|
||||
return 0, storage.ErrOutOfBounds
|
||||
|
|
@ -396,7 +396,7 @@ func (a *headAppender) AppendHistogram(ref uint64, lset labels.Labels, t int64,
|
|||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
s.sparseHistogramSeries = true
|
||||
s.histogramSeries = true
|
||||
if created {
|
||||
a.head.metrics.sparseHistogramSeries.Inc()
|
||||
a.series = append(a.series, record.RefSeries{
|
||||
|
|
@ -407,7 +407,7 @@ func (a *headAppender) AppendHistogram(ref uint64, lset labels.Labels, t int64,
|
|||
}
|
||||
|
||||
s.Lock()
|
||||
if err := s.appendableHistogram(t, sh); err != nil {
|
||||
if err := s.appendableHistogram(t, h); err != nil {
|
||||
s.Unlock()
|
||||
if err == storage.ErrOutOfOrderSample {
|
||||
a.head.metrics.outOfOrderSamples.Inc()
|
||||
|
|
@ -427,7 +427,7 @@ func (a *headAppender) AppendHistogram(ref uint64, lset labels.Labels, t int64,
|
|||
a.histograms = append(a.histograms, record.RefHistogram{
|
||||
Ref: s.ref,
|
||||
T: t,
|
||||
H: sh,
|
||||
H: h,
|
||||
})
|
||||
a.histogramSeries = append(a.histogramSeries, s)
|
||||
return s.ref, nil
|
||||
|
|
@ -604,37 +604,39 @@ func (s *memSeries) append(t int64, v float64, appendID uint64, chunkDiskMapper
|
|||
return true, chunkCreated
|
||||
}
|
||||
|
||||
// appendHistogram adds the sparse histogram.
|
||||
// appendHistogram adds the histogram.
|
||||
// It is unsafe to call this concurrently with s.iterator(...) without holding the series lock.
|
||||
func (s *memSeries) appendHistogram(t int64, sh histogram.SparseHistogram, appendID uint64, chunkDiskMapper *chunks.ChunkDiskMapper) (sampleInOrder, chunkCreated bool) {
|
||||
func (s *memSeries) appendHistogram(t int64, sh histogram.Histogram, appendID uint64, chunkDiskMapper *chunks.ChunkDiskMapper) (sampleInOrder, chunkCreated bool) {
|
||||
// Head controls the execution of recoding, so that we own the proper chunk reference afterwards.
|
||||
// We check for Appendable before appendPreprocessor because in case it ends up creating a new chunk,
|
||||
// we need to know if there was also a counter reset or not to set the meta properly.
|
||||
app, _ := s.app.(*chunkenc.HistoAppender)
|
||||
app, _ := s.app.(*chunkenc.HistogramAppender)
|
||||
var (
|
||||
posInterjections, negInterjections []chunkenc.Interjection
|
||||
okToAppend, counterReset bool
|
||||
positiveInterjections, negativeInterjections []chunkenc.Interjection
|
||||
okToAppend, counterReset bool
|
||||
)
|
||||
if app != nil {
|
||||
posInterjections, negInterjections, okToAppend, counterReset = app.Appendable(sh)
|
||||
positiveInterjections, negativeInterjections, okToAppend, counterReset = app.Appendable(sh)
|
||||
}
|
||||
|
||||
c, sampleInOrder, chunkCreated := s.appendPreprocessor(t, chunkenc.EncSHS, chunkDiskMapper)
|
||||
c, sampleInOrder, chunkCreated := s.appendPreprocessor(t, chunkenc.EncHistogram, chunkDiskMapper)
|
||||
if !sampleInOrder {
|
||||
return sampleInOrder, chunkCreated
|
||||
}
|
||||
|
||||
if !chunkCreated {
|
||||
// We have 3 cases here
|
||||
// !okToAppend -> we need to cut a new chunk
|
||||
// okToAppend but we have interjections -> existing chunk needs recoding before we can append our histogram
|
||||
// okToAppend and no interjections -> chunk is ready to support our histogram
|
||||
// - !okToAppend -> We need to cut a new chunk.
|
||||
// - okToAppend but we have interjections -> Existing chunk needs recoding before we can append our histogram.
|
||||
// - okToAppend and no interjections -> Chunk is ready to support our histogram.
|
||||
if !okToAppend || counterReset {
|
||||
c = s.cutNewHeadChunk(t, chunkenc.EncSHS, chunkDiskMapper)
|
||||
c = s.cutNewHeadChunk(t, chunkenc.EncHistogram, chunkDiskMapper)
|
||||
chunkCreated = true
|
||||
} else if len(posInterjections) > 0 || len(negInterjections) > 0 {
|
||||
// new buckets have appeared. we need to recode all prior histograms within the chunk before we can process this one.
|
||||
chunk, app := app.Recode(posInterjections, negInterjections, sh.PositiveSpans, sh.NegativeSpans)
|
||||
} else if len(positiveInterjections) > 0 || len(negativeInterjections) > 0 {
|
||||
// New buckets have appeared. We need to recode all
|
||||
// prior histogram samples within the chunk before we
|
||||
// can process this one.
|
||||
chunk, app := app.Recode(positiveInterjections, negativeInterjections, sh.PositiveSpans, sh.NegativeSpans)
|
||||
s.headChunk = &memChunk{
|
||||
minTime: s.headChunk.minTime,
|
||||
maxTime: s.headChunk.maxTime,
|
||||
|
|
@ -645,7 +647,7 @@ func (s *memSeries) appendHistogram(t int64, sh histogram.SparseHistogram, appen
|
|||
}
|
||||
|
||||
if chunkCreated {
|
||||
hc := s.headChunk.chunk.(*chunkenc.HistoChunk)
|
||||
hc := s.headChunk.chunk.(*chunkenc.HistogramChunk)
|
||||
header := chunkenc.UnknownCounterReset
|
||||
if counterReset {
|
||||
header = chunkenc.CounterReset
|
||||
|
|
@ -656,14 +658,14 @@ func (s *memSeries) appendHistogram(t int64, sh histogram.SparseHistogram, appen
|
|||
}
|
||||
|
||||
s.app.AppendHistogram(t, sh)
|
||||
s.sparseHistogramSeries = true
|
||||
s.histogramSeries = true
|
||||
|
||||
c.maxTime = t
|
||||
|
||||
s.histBuf[0] = s.histBuf[1]
|
||||
s.histBuf[1] = s.histBuf[2]
|
||||
s.histBuf[2] = s.histBuf[3]
|
||||
s.histBuf[3] = hist{t: t, h: sh}
|
||||
s.histogramBuf[0] = s.histogramBuf[1]
|
||||
s.histogramBuf[1] = s.histogramBuf[2]
|
||||
s.histogramBuf[2] = s.histogramBuf[3]
|
||||
s.histogramBuf[3] = histogramSample{t: t, h: sh}
|
||||
|
||||
if appendID > 0 {
|
||||
s.txs.add(appendID)
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ import (
|
|||
"github.com/go-kit/log/level"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
|
|
@ -443,7 +443,7 @@ func (s *memSeries) iterator(id int, isoState *isolationState, chunkDiskMapper *
|
|||
msIter.total = numSamples
|
||||
msIter.stopAfter = stopAfter
|
||||
msIter.buf = s.sampleBuf
|
||||
msIter.histBuf = s.histBuf
|
||||
msIter.histogramBuf = s.histogramBuf
|
||||
return msIter
|
||||
}
|
||||
return &memSafeIterator{
|
||||
|
|
@ -452,18 +452,18 @@ func (s *memSeries) iterator(id int, isoState *isolationState, chunkDiskMapper *
|
|||
i: -1,
|
||||
stopAfter: stopAfter,
|
||||
},
|
||||
total: numSamples,
|
||||
buf: s.sampleBuf,
|
||||
histBuf: s.histBuf,
|
||||
total: numSamples,
|
||||
buf: s.sampleBuf,
|
||||
histogramBuf: s.histogramBuf,
|
||||
}
|
||||
}
|
||||
|
||||
type memSafeIterator struct {
|
||||
stopIterator
|
||||
|
||||
total int
|
||||
buf [4]sample
|
||||
histBuf [4]hist
|
||||
total int
|
||||
buf [4]sample
|
||||
histogramBuf [4]histogramSample
|
||||
}
|
||||
|
||||
func (it *memSafeIterator) Seek(t int64) bool {
|
||||
|
|
@ -502,11 +502,11 @@ func (it *memSafeIterator) At() (int64, float64) {
|
|||
return s.t, s.v
|
||||
}
|
||||
|
||||
func (it *memSafeIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
func (it *memSafeIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
if it.total-it.i > 4 {
|
||||
return it.Iterator.AtHistogram()
|
||||
}
|
||||
s := it.histBuf[4-(it.total-it.i)]
|
||||
s := it.histogramBuf[4-(it.total-it.i)]
|
||||
return s.t, s.h
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -37,8 +37,8 @@ import (
|
|||
"go.uber.org/atomic"
|
||||
|
||||
"github.com/prometheus/prometheus/config"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/pkg/value"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
|
|
@ -2539,15 +2539,15 @@ func TestAppendHistogram(t *testing.T) {
|
|||
require.NoError(t, head.Init(0))
|
||||
app := head.Appender(context.Background())
|
||||
|
||||
type timedHist struct {
|
||||
type timedHistogram struct {
|
||||
t int64
|
||||
h histogram.SparseHistogram
|
||||
h histogram.Histogram
|
||||
}
|
||||
expHists := make([]timedHist, 0, numHistograms)
|
||||
expHistograms := make([]timedHistogram, 0, numHistograms)
|
||||
for i, h := range generateHistograms(numHistograms) {
|
||||
_, err := app.AppendHistogram(0, l, int64(i), h)
|
||||
require.NoError(t, err)
|
||||
expHists = append(expHists, timedHist{int64(i), h})
|
||||
expHistograms = append(expHistograms, timedHistogram{int64(i), h})
|
||||
}
|
||||
require.NoError(t, app.Commit())
|
||||
|
||||
|
|
@ -2564,13 +2564,13 @@ func TestAppendHistogram(t *testing.T) {
|
|||
require.False(t, ss.Next())
|
||||
|
||||
it := s.Iterator()
|
||||
actHists := make([]timedHist, 0, len(expHists))
|
||||
actHistograms := make([]timedHistogram, 0, len(expHistograms))
|
||||
for it.Next() {
|
||||
t, h := it.AtHistogram()
|
||||
actHists = append(actHists, timedHist{t, h.Copy()})
|
||||
actHistograms = append(actHistograms, timedHistogram{t, h.Copy()})
|
||||
}
|
||||
|
||||
require.Equal(t, expHists, actHists)
|
||||
require.Equal(t, expHistograms, actHistograms)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -2586,17 +2586,17 @@ func TestHistogramInWAL(t *testing.T) {
|
|||
require.NoError(t, head.Init(0))
|
||||
app := head.Appender(context.Background())
|
||||
|
||||
type timedHist struct {
|
||||
type timedHistogram struct {
|
||||
t int64
|
||||
h histogram.SparseHistogram
|
||||
h histogram.Histogram
|
||||
}
|
||||
expHists := make([]timedHist, 0, numHistograms)
|
||||
expHistograms := make([]timedHistogram, 0, numHistograms)
|
||||
for i, h := range generateHistograms(numHistograms) {
|
||||
h.NegativeSpans = h.PositiveSpans
|
||||
h.NegativeBuckets = h.PositiveBuckets
|
||||
_, err := app.AppendHistogram(0, l, int64(i), h)
|
||||
require.NoError(t, err)
|
||||
expHists = append(expHists, timedHist{int64(i), h})
|
||||
expHistograms = append(expHistograms, timedHistogram{int64(i), h})
|
||||
}
|
||||
require.NoError(t, app.Commit())
|
||||
|
||||
|
|
@ -2621,18 +2621,18 @@ func TestHistogramInWAL(t *testing.T) {
|
|||
require.False(t, ss.Next())
|
||||
|
||||
it := s.Iterator()
|
||||
actHists := make([]timedHist, 0, len(expHists))
|
||||
actHistograms := make([]timedHistogram, 0, len(expHistograms))
|
||||
for it.Next() {
|
||||
t, h := it.AtHistogram()
|
||||
actHists = append(actHists, timedHist{t, h.Copy()})
|
||||
actHistograms = append(actHistograms, timedHistogram{t, h.Copy()})
|
||||
}
|
||||
|
||||
require.Equal(t, expHists, actHists)
|
||||
require.Equal(t, expHistograms, actHistograms)
|
||||
}
|
||||
|
||||
func generateHistograms(n int) (r []histogram.SparseHistogram) {
|
||||
func generateHistograms(n int) (r []histogram.Histogram) {
|
||||
for i := 0; i < n; i++ {
|
||||
r = append(r, histogram.SparseHistogram{
|
||||
r = append(r, histogram.Histogram{
|
||||
Count: 5 + uint64(i*4),
|
||||
ZeroCount: 2 + uint64(i),
|
||||
ZeroThreshold: 0.001,
|
||||
|
|
@ -2957,22 +2957,22 @@ func TestSparseHistogramMetrics(t *testing.T) {
|
|||
})
|
||||
require.NoError(t, head.Init(0))
|
||||
|
||||
expHistSeries, expHistSamples := 0, 0
|
||||
expHSeries, expHSamples := 0, 0
|
||||
|
||||
for x := 0; x < 5; x++ {
|
||||
expHistSeries++
|
||||
expHSeries++
|
||||
l := labels.Labels{{Name: "a", Value: fmt.Sprintf("b%d", x)}}
|
||||
for i, h := range generateHistograms(10) {
|
||||
app := head.Appender(context.Background())
|
||||
_, err := app.AppendHistogram(0, l, int64(i), h)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, app.Commit())
|
||||
expHistSamples++
|
||||
expHSamples++
|
||||
}
|
||||
}
|
||||
|
||||
require.Equal(t, float64(expHistSeries), prom_testutil.ToFloat64(head.metrics.sparseHistogramSeries))
|
||||
require.Equal(t, float64(expHistSamples), prom_testutil.ToFloat64(head.metrics.sparseHistogramSamplesTotal))
|
||||
require.Equal(t, float64(expHSeries), prom_testutil.ToFloat64(head.metrics.sparseHistogramSeries))
|
||||
require.Equal(t, float64(expHSamples), prom_testutil.ToFloat64(head.metrics.sparseHistogramSamplesTotal))
|
||||
|
||||
require.NoError(t, head.Close())
|
||||
w, err := wal.NewSize(nil, nil, head.wal.Dir(), 32768, false)
|
||||
|
|
@ -2981,7 +2981,7 @@ func TestSparseHistogramMetrics(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
require.NoError(t, head.Init(0))
|
||||
|
||||
require.Equal(t, float64(expHistSeries), prom_testutil.ToFloat64(head.metrics.sparseHistogramSeries))
|
||||
require.Equal(t, float64(expHSeries), prom_testutil.ToFloat64(head.metrics.sparseHistogramSeries))
|
||||
require.Equal(t, float64(0), prom_testutil.ToFloat64(head.metrics.sparseHistogramSamplesTotal)) // Counter reset.
|
||||
}
|
||||
|
||||
|
|
@ -2994,11 +2994,11 @@ func TestSparseHistogramStaleSample(t *testing.T) {
|
|||
})
|
||||
require.NoError(t, head.Init(0))
|
||||
|
||||
type timedHist struct {
|
||||
type timedHistogram struct {
|
||||
t int64
|
||||
h histogram.SparseHistogram
|
||||
h histogram.Histogram
|
||||
}
|
||||
expHists := make([]timedHist, 0, numHistograms)
|
||||
expHistograms := make([]timedHistogram, 0, numHistograms)
|
||||
|
||||
testQuery := func(numStale int) {
|
||||
q, err := NewBlockQuerier(head, head.MinTime(), head.MaxTime())
|
||||
|
|
@ -3014,17 +3014,17 @@ func TestSparseHistogramStaleSample(t *testing.T) {
|
|||
require.False(t, ss.Next())
|
||||
|
||||
it := s.Iterator()
|
||||
actHists := make([]timedHist, 0, len(expHists))
|
||||
actHistograms := make([]timedHistogram, 0, len(expHistograms))
|
||||
for it.Next() {
|
||||
t, h := it.AtHistogram()
|
||||
actHists = append(actHists, timedHist{t, h.Copy()})
|
||||
actHistograms = append(actHistograms, timedHistogram{t, h.Copy()})
|
||||
}
|
||||
|
||||
// We cannot compare StaleNAN with require.Equal, hence checking each histogram manually.
|
||||
require.Equal(t, len(expHists), len(actHists))
|
||||
require.Equal(t, len(expHistograms), len(actHistograms))
|
||||
actNumStale := 0
|
||||
for i, eh := range expHists {
|
||||
ah := actHists[i]
|
||||
for i, eh := range expHistograms {
|
||||
ah := actHistograms[i]
|
||||
if value.IsStaleNaN(eh.h.Sum) {
|
||||
actNumStale++
|
||||
require.True(t, value.IsStaleNaN(ah.h.Sum))
|
||||
|
|
@ -3040,14 +3040,14 @@ func TestSparseHistogramStaleSample(t *testing.T) {
|
|||
// Adding stale in the same appender.
|
||||
app := head.Appender(context.Background())
|
||||
for _, h := range generateHistograms(numHistograms) {
|
||||
_, err := app.AppendHistogram(0, l, 100*int64(len(expHists)), h)
|
||||
_, err := app.AppendHistogram(0, l, 100*int64(len(expHistograms)), h)
|
||||
require.NoError(t, err)
|
||||
expHists = append(expHists, timedHist{100 * int64(len(expHists)), h})
|
||||
expHistograms = append(expHistograms, timedHistogram{100 * int64(len(expHistograms)), h})
|
||||
}
|
||||
// +1 so that delta-of-delta is not 0.
|
||||
_, err := app.Append(0, l, 100*int64(len(expHists))+1, math.Float64frombits(value.StaleNaN))
|
||||
_, err := app.Append(0, l, 100*int64(len(expHistograms))+1, math.Float64frombits(value.StaleNaN))
|
||||
require.NoError(t, err)
|
||||
expHists = append(expHists, timedHist{100*int64(len(expHists)) + 1, histogram.SparseHistogram{Sum: math.Float64frombits(value.StaleNaN)}})
|
||||
expHistograms = append(expHistograms, timedHistogram{100*int64(len(expHistograms)) + 1, histogram.Histogram{Sum: math.Float64frombits(value.StaleNaN)}})
|
||||
require.NoError(t, app.Commit())
|
||||
|
||||
// Only 1 chunk in the memory, no m-mapped chunk.
|
||||
|
|
@ -3059,17 +3059,17 @@ func TestSparseHistogramStaleSample(t *testing.T) {
|
|||
// Adding stale in different appender and continuing series after a stale sample.
|
||||
app = head.Appender(context.Background())
|
||||
for _, h := range generateHistograms(2 * numHistograms)[numHistograms:] {
|
||||
_, err := app.AppendHistogram(0, l, 100*int64(len(expHists)), h)
|
||||
_, err := app.AppendHistogram(0, l, 100*int64(len(expHistograms)), h)
|
||||
require.NoError(t, err)
|
||||
expHists = append(expHists, timedHist{100 * int64(len(expHists)), h})
|
||||
expHistograms = append(expHistograms, timedHistogram{100 * int64(len(expHistograms)), h})
|
||||
}
|
||||
require.NoError(t, app.Commit())
|
||||
|
||||
app = head.Appender(context.Background())
|
||||
// +1 so that delta-of-delta is not 0.
|
||||
_, err = app.Append(0, l, 100*int64(len(expHists))+1, math.Float64frombits(value.StaleNaN))
|
||||
_, err = app.Append(0, l, 100*int64(len(expHistograms))+1, math.Float64frombits(value.StaleNaN))
|
||||
require.NoError(t, err)
|
||||
expHists = append(expHists, timedHist{100*int64(len(expHists)) + 1, histogram.SparseHistogram{Sum: math.Float64frombits(value.StaleNaN)}})
|
||||
expHistograms = append(expHistograms, timedHistogram{100*int64(len(expHistograms)) + 1, histogram.Histogram{Sum: math.Float64frombits(value.StaleNaN)}})
|
||||
require.NoError(t, app.Commit())
|
||||
|
||||
// Total 2 chunks, 1 m-mapped.
|
||||
|
|
|
|||
|
|
@ -15,11 +15,6 @@ package tsdb
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
"github.com/prometheus/prometheus/tsdb/encoding"
|
||||
tsdb_errors "github.com/prometheus/prometheus/tsdb/errors"
|
||||
"github.com/prometheus/prometheus/tsdb/fileutil"
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"os"
|
||||
|
|
@ -30,6 +25,12 @@ import (
|
|||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
"github.com/prometheus/prometheus/tsdb/encoding"
|
||||
tsdb_errors "github.com/prometheus/prometheus/tsdb/errors"
|
||||
"github.com/prometheus/prometheus/tsdb/fileutil"
|
||||
|
||||
"github.com/go-kit/log/level"
|
||||
"github.com/pkg/errors"
|
||||
"go.uber.org/atomic"
|
||||
|
|
@ -159,7 +160,7 @@ func (h *Head) loadWAL(r *wal.Reader, multiRef map[uint64]uint64, mmappedChunks
|
|||
|
||||
if ms.head() == nil {
|
||||
// First histogram for the series. Count this in metrics.
|
||||
ms.sparseHistogramSeries = true
|
||||
ms.histogramSeries = true
|
||||
}
|
||||
|
||||
if rh.T < h.minValidTime.Load() {
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ import (
|
|||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
|
|
@ -649,7 +649,7 @@ func (p *populateWithDelSeriesIterator) Seek(t int64) bool {
|
|||
}
|
||||
|
||||
func (p *populateWithDelSeriesIterator) At() (int64, float64) { return p.curr.At() }
|
||||
func (p *populateWithDelSeriesIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
func (p *populateWithDelSeriesIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
return p.curr.AtHistogram()
|
||||
}
|
||||
func (p *populateWithDelSeriesIterator) ChunkEncoding() chunkenc.Encoding {
|
||||
|
|
@ -688,8 +688,8 @@ func (p *populateWithDelChunkSeriesIterator) Next() bool {
|
|||
app chunkenc.Appender
|
||||
err error
|
||||
)
|
||||
if p.currDelIter.ChunkEncoding() == chunkenc.EncSHS {
|
||||
newChunk = chunkenc.NewHistoChunk()
|
||||
if p.currDelIter.ChunkEncoding() == chunkenc.EncHistogram {
|
||||
newChunk = chunkenc.NewHistogramChunk()
|
||||
app, err = newChunk.Appender()
|
||||
} else {
|
||||
newChunk = chunkenc.NewXORChunk()
|
||||
|
|
@ -714,11 +714,11 @@ func (p *populateWithDelChunkSeriesIterator) Next() bool {
|
|||
var (
|
||||
t int64
|
||||
v float64
|
||||
h histogram.SparseHistogram
|
||||
h histogram.Histogram
|
||||
)
|
||||
if p.currDelIter.ChunkEncoding() == chunkenc.EncSHS {
|
||||
if hc, ok := p.currChkMeta.Chunk.(*chunkenc.HistoChunk); ok {
|
||||
newChunk.(*chunkenc.HistoChunk).SetCounterResetHeader(hc.GetCounterResetHeader())
|
||||
if p.currDelIter.ChunkEncoding() == chunkenc.EncHistogram {
|
||||
if hc, ok := p.currChkMeta.Chunk.(*chunkenc.HistogramChunk); ok {
|
||||
newChunk.(*chunkenc.HistogramChunk).SetCounterResetHeader(hc.GetCounterResetHeader())
|
||||
}
|
||||
t, h = p.currDelIter.AtHistogram()
|
||||
p.curr.MinTime = t
|
||||
|
|
@ -870,7 +870,7 @@ func (it *DeletedIterator) At() (int64, float64) {
|
|||
return it.Iter.At()
|
||||
}
|
||||
|
||||
func (it *DeletedIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
func (it *DeletedIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
t, h := it.Iter.AtHistogram()
|
||||
return t, h
|
||||
}
|
||||
|
|
@ -889,7 +889,7 @@ func (it *DeletedIterator) Seek(t int64) bool {
|
|||
|
||||
// Now double check if the entry falls into a deleted interval.
|
||||
var ts int64
|
||||
if it.ChunkEncoding() == chunkenc.EncSHS {
|
||||
if it.ChunkEncoding() == chunkenc.EncHistogram {
|
||||
ts, _ = it.AtHistogram()
|
||||
} else {
|
||||
ts, _ = it.At()
|
||||
|
|
@ -916,7 +916,7 @@ func (it *DeletedIterator) Next() bool {
|
|||
Outer:
|
||||
for it.Iter.Next() {
|
||||
var ts int64
|
||||
if it.ChunkEncoding() == chunkenc.EncSHS {
|
||||
if it.ChunkEncoding() == chunkenc.EncHistogram {
|
||||
ts, _ = it.AtHistogram()
|
||||
} else {
|
||||
ts, _ = it.At()
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ import (
|
|||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/tsdb/encoding"
|
||||
"github.com/prometheus/prometheus/tsdb/tombstones"
|
||||
|
|
@ -74,7 +74,7 @@ type RefExemplar struct {
|
|||
type RefHistogram struct {
|
||||
Ref uint64
|
||||
T int64
|
||||
H histogram.SparseHistogram
|
||||
H histogram.Histogram
|
||||
}
|
||||
|
||||
// Decoder decodes series, sample, and tombstone records.
|
||||
|
|
@ -233,14 +233,14 @@ func (d *Decoder) ExemplarsFromBuffer(dec *encoding.Decbuf, exemplars []RefExemp
|
|||
return exemplars, nil
|
||||
}
|
||||
|
||||
func (d *Decoder) Histograms(rec []byte, hists []RefHistogram) ([]RefHistogram, error) {
|
||||
func (d *Decoder) Histograms(rec []byte, histograms []RefHistogram) ([]RefHistogram, error) {
|
||||
dec := encoding.Decbuf{B: rec}
|
||||
t := Type(dec.Byte())
|
||||
if t != Histograms {
|
||||
return nil, errors.New("invalid record type")
|
||||
}
|
||||
if dec.Len() == 0 {
|
||||
return hists, nil
|
||||
return histograms, nil
|
||||
}
|
||||
var (
|
||||
baseRef = dec.Be64()
|
||||
|
|
@ -253,7 +253,7 @@ func (d *Decoder) Histograms(rec []byte, hists []RefHistogram) ([]RefHistogram,
|
|||
rh := RefHistogram{
|
||||
Ref: baseRef + uint64(dref),
|
||||
T: baseTime + dtime,
|
||||
H: histogram.SparseHistogram{
|
||||
H: histogram.Histogram{
|
||||
Schema: 0,
|
||||
ZeroThreshold: 0,
|
||||
ZeroCount: 0,
|
||||
|
|
@ -303,16 +303,16 @@ func (d *Decoder) Histograms(rec []byte, hists []RefHistogram) ([]RefHistogram,
|
|||
rh.H.NegativeBuckets[i] = dec.Varint64()
|
||||
}
|
||||
|
||||
hists = append(hists, rh)
|
||||
histograms = append(histograms, rh)
|
||||
}
|
||||
|
||||
if dec.Err() != nil {
|
||||
return nil, errors.Wrapf(dec.Err(), "decode error after %d histograms", len(hists))
|
||||
return nil, errors.Wrapf(dec.Err(), "decode error after %d histograms", len(histograms))
|
||||
}
|
||||
if len(dec.B) > 0 {
|
||||
return nil, errors.Errorf("unexpected %d bytes left in entry", len(dec.B))
|
||||
}
|
||||
return hists, nil
|
||||
return histograms, nil
|
||||
}
|
||||
|
||||
// Encoder encodes series, sample, and tombstones records.
|
||||
|
|
@ -410,21 +410,21 @@ func (e *Encoder) EncodeExemplarsIntoBuffer(exemplars []RefExemplar, buf *encodi
|
|||
}
|
||||
}
|
||||
|
||||
func (e *Encoder) Histograms(hists []RefHistogram, b []byte) []byte {
|
||||
func (e *Encoder) Histograms(histograms []RefHistogram, b []byte) []byte {
|
||||
buf := encoding.Encbuf{B: b}
|
||||
buf.PutByte(byte(Histograms))
|
||||
|
||||
if len(hists) == 0 {
|
||||
if len(histograms) == 0 {
|
||||
return buf.Get()
|
||||
}
|
||||
|
||||
// Store base timestamp and base reference number of first histogram.
|
||||
// All histograms encode their timestamp and ref as delta to those.
|
||||
first := hists[0]
|
||||
first := histograms[0]
|
||||
buf.PutBE64(first.Ref)
|
||||
buf.PutBE64int64(first.T)
|
||||
|
||||
for _, h := range hists {
|
||||
for _, h := range histograms {
|
||||
buf.PutVarint64(int64(h.Ref) - int64(first.Ref))
|
||||
buf.PutVarint64(h.T - first.T)
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ package tsdbutil
|
|||
import (
|
||||
"math"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
)
|
||||
|
||||
|
|
@ -160,8 +160,8 @@ func (it *sampleRingIterator) At() (int64, float64) {
|
|||
return it.r.at(it.i)
|
||||
}
|
||||
|
||||
func (it *sampleRingIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
return 0, histogram.SparseHistogram{}
|
||||
func (it *sampleRingIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
return 0, histogram.Histogram{}
|
||||
}
|
||||
|
||||
func (it *sampleRingIterator) ChunkEncoding() chunkenc.Encoding {
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import (
|
|||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
|
@ -152,8 +152,8 @@ func (it *listSeriesIterator) At() (int64, float64) {
|
|||
return s.t, s.v
|
||||
}
|
||||
|
||||
func (it *listSeriesIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
return 0, histogram.SparseHistogram{}
|
||||
func (it *listSeriesIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
return 0, histogram.Histogram{}
|
||||
}
|
||||
|
||||
func (it *listSeriesIterator) ChunkEncoding() chunkenc.Encoding {
|
||||
|
|
|
|||
|
|
@ -296,7 +296,6 @@ var sampleFlagMap = map[string]string{
|
|||
}
|
||||
|
||||
func TestEndpoints(t *testing.T) {
|
||||
t.Skip()
|
||||
suite, err := promql.NewTest(t, `
|
||||
load 1m
|
||||
test_metric1{foo="bar"} 0+100x100
|
||||
|
|
|
|||
Loading…
Reference in a new issue