diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index c124d23379..ce49a9dd93 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -58,9 +58,9 @@ import ( "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/discovery" _ "github.com/prometheus/prometheus/discovery/install" // Register service discovery implementations. + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/notifier" "github.com/prometheus/prometheus/pkg/exemplar" - "github.com/prometheus/prometheus/pkg/histogram" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/logging" "github.com/prometheus/prometheus/pkg/relabel" @@ -1190,7 +1190,7 @@ func (n notReadyAppender) AppendExemplar(ref uint64, l labels.Labels, e exemplar return 0, tsdb.ErrNotReady } -func (n notReadyAppender) AppendHistogram(ref uint64, l labels.Labels, t int64, sh histogram.SparseHistogram) (uint64, error) { +func (n notReadyAppender) AppendHistogram(ref uint64, l labels.Labels, t int64, sh histogram.Histogram) (uint64, error) { return 0, tsdb.ErrNotReady } diff --git a/cmd/prometheus/query_log_test.go b/cmd/prometheus/query_log_test.go index c786ccd288..4e75ecae9c 100644 --- a/cmd/prometheus/query_log_test.go +++ b/cmd/prometheus/query_log_test.go @@ -405,7 +405,6 @@ func readQueryLog(t *testing.T, path string) []queryLogLine { } func TestQueryLog(t *testing.T) { - t.Skip() if testing.Short() { t.Skip("skipping test in short mode.") } diff --git a/go.mod b/go.mod index f03bcb8a5b..800294a868 100644 --- a/go.mod +++ b/go.mod @@ -62,7 +62,7 @@ require ( golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34 golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac - golang.org/x/tools v0.1.5 + golang.org/x/tools v0.1.7 google.golang.org/api v0.56.0 google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83 google.golang.org/protobuf v1.27.1 diff --git a/go.sum b/go.sum index b1fc5326cd..01e538b9c5 100644 --- a/go.sum +++ b/go.sum @@ -1308,6 +1308,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= @@ -1497,6 +1498,7 @@ golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210903162142-ad29c8ab022f h1:w6wWR0H+nyVpbSAQbzVEIACVyr/h8l/BEkY6Sokc7Eg= golang.org/x/net v0.0.0-20210903162142-ad29c8ab022f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1634,6 +1636,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34 h1:GkvMjFtXUmahfDtashnc1mnrCtuBVcwse5QV2lUk/tI= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1739,8 +1742,9 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ= +golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/histogram/sparse_histogram.go b/model/histogram/histogram.go similarity index 78% rename from pkg/histogram/sparse_histogram.go rename to model/histogram/histogram.go index 0e94e9adc5..5a2ddea340 100644 --- a/pkg/histogram/sparse_histogram.go +++ b/model/histogram/histogram.go @@ -17,55 +17,90 @@ import ( "math" ) -// SparseHistogram encodes a sparse histogram -// full details: https://docs.google.com/document/d/1cLNv3aufPZb3fNfaJgdaRBZsInZKKIHo9E6HinJVbpM/edit# -// the most tricky bit is how bucket indices represent real bucket boundaries +// Histogram encodes a sparse, high-resolution histogram. See the design +// document for full details: +// https://docs.google.com/document/d/1cLNv3aufPZb3fNfaJgdaRBZsInZKKIHo9E6HinJVbpM/edit# // -// an example for schema 0 (which doubles the size of consecutive buckets): +// The most tricky bit is how bucket indices represent real bucket boundaries. +// An example for schema 0 (by which each bucket is twice as wide as the +// previous bucket): // -// buckets syntax (LE,GE) (-2,-1) (-1,-0.5) (-0.5,-0.25) ... (-0.001-0.001) ... (0.25-0.5)(0.5-1) (1-2) .... -// ^ -// zero bucket (here width a width of 0.001) ZB -// pos bucket idx ... -1 0 1 2 3 -// neg bucket idx 3 2 1 0 -1 ... -// actively used bucket indices themselves are represented by the spans -type SparseHistogram struct { - Schema int32 - ZeroThreshold float64 - ZeroCount, Count uint64 - Sum float64 - PositiveSpans, NegativeSpans []Span +// Bucket boundaries → [-2,-1) [-1,-0.5) [-0.5,-0.25) ... [-0.001,0.001] ... (0.25,0.5] (0.5,1] (1,2] .... +// ↑ ↑ ↑ ↑ ↑ ↑ ↑ +// Zero bucket (width e.g. 0.001) → | | | ZB | | | +// Positive bucket indices → | | | ... -1 0 1 2 3 +// Negative bucket indices → 3 2 1 0 -1 ... +// +// Wich bucket indices are actually used is determined by the spans. +type Histogram struct { + // Currently valid schema numbers are -4 <= n <= 8. They are all for + // base-2 bucket schemas, where 1 is a bucket boundary in each case, and + // then each power of two is divided into 2^n logarithmic buckets. Or + // in other words, each bucket boundary is the previous boundary times + // 2^(2^-n). + Schema int32 + // Width of the zero bucket. + ZeroThreshold float64 + // Observations falling into the zero bucket. + ZeroCount uint64 + // Total number of observations. + Count uint64 + // Sum of observations. + Sum float64 + // Spans for positive and negative buckets (see Span below). + PositiveSpans, NegativeSpans []Span + // Observation counts in buckets. The first element is an absolute + // count. All following ones are deltas relative to the previous + // element. PositiveBuckets, NegativeBuckets []int64 } +// A Span defines a continuous sequence of buckets. type Span struct { + // Gap to previous span (always positive), or starting index for the 1st + // span (which can be negative). Offset int32 + // Length of the span. Length uint32 } -func (s SparseHistogram) Copy() SparseHistogram { - c := s +// Copy returns a deep copy of the Histogram. +func (h Histogram) Copy() Histogram { + c := h - if s.PositiveSpans != nil { - c.PositiveSpans = make([]Span, len(s.PositiveSpans)) - copy(c.PositiveSpans, s.PositiveSpans) + if h.PositiveSpans != nil { + c.PositiveSpans = make([]Span, len(h.PositiveSpans)) + copy(c.PositiveSpans, h.PositiveSpans) } - if s.NegativeSpans != nil { - c.NegativeSpans = make([]Span, len(s.NegativeSpans)) - copy(c.NegativeSpans, s.NegativeSpans) + if h.NegativeSpans != nil { + c.NegativeSpans = make([]Span, len(h.NegativeSpans)) + copy(c.NegativeSpans, h.NegativeSpans) } - if s.PositiveBuckets != nil { - c.PositiveBuckets = make([]int64, len(s.PositiveBuckets)) - copy(c.PositiveBuckets, s.PositiveBuckets) + if h.PositiveBuckets != nil { + c.PositiveBuckets = make([]int64, len(h.PositiveBuckets)) + copy(c.PositiveBuckets, h.PositiveBuckets) } - if s.NegativeBuckets != nil { - c.NegativeBuckets = make([]int64, len(s.NegativeBuckets)) - copy(c.NegativeBuckets, s.NegativeBuckets) + if h.NegativeBuckets != nil { + c.NegativeBuckets = make([]int64, len(h.NegativeBuckets)) + copy(c.NegativeBuckets, h.NegativeBuckets) } return c } +// CumulativeBucketIterator returns a BucketIterator to iterate over a +// cumulative view of the buckets. This method currently only supports +// Histograms without negative buckets and panics if the Histogram has negative +// buckets. It is currently only used for testing. +func (h Histogram) CumulativeBucketIterator() BucketIterator { + if len(h.NegativeBuckets) > 0 { + panic("CumulativeIterator called on Histogram with negative buckets") + } + return &cumulativeBucketIterator{h: h, posSpansIdx: -1} +} + +// BucketIterator iterates over the buckets of a Histogram, returning decoded +// buckets. type BucketIterator interface { // Next advances the iterator by one. Next() bool @@ -76,28 +111,23 @@ type BucketIterator interface { Err() error } +// Bucket represents a bucket (currently only a cumulative one with an upper +// inclusive bound and a cumulative count). type Bucket struct { - Le float64 + Upper float64 Count uint64 } -// CumulativeExpandSparseHistogram expands the given histogram to produce cumulative buckets. -// It assumes that the total length of spans matches the number of buckets for pos and neg respectively. -// TODO: supports only positive buckets, also do for negative. -func CumulativeExpandSparseHistogram(h SparseHistogram) BucketIterator { - return &cumulativeBucketIterator{h: h, posSpansIdx: -1} -} - type cumulativeBucketIterator struct { - h SparseHistogram + h Histogram posSpansIdx int // Index in h.PositiveSpans we are in. -1 means 0 bucket. - posBucketsIdx int // Index in h.PositiveBuckets + posBucketsIdx int // Index in h.PositiveBuckets. idxInSpan uint32 // Index in the current span. 0 <= idxInSpan < span.Length. initialised bool currIdx int32 // The actual bucket index after decoding from spans. - currLe float64 // The upper boundary of the current bucket. + currUpper float64 // The upper boundary of the current bucket. currCount int64 // Current non-cumulative count for the current bucket. Does not apply for empty bucket. currCumulativeCount uint64 // Current "cumulative" count for the current bucket. @@ -116,7 +146,7 @@ func (c *cumulativeBucketIterator) Next() bool { return c.Next() } - c.currLe = c.h.ZeroThreshold + c.currUpper = c.h.ZeroThreshold c.currCount = int64(c.h.ZeroCount) c.currCumulativeCount = uint64(c.currCount) return true @@ -128,7 +158,7 @@ func (c *cumulativeBucketIterator) Next() bool { if c.emptyBucketCount > 0 { // We are traversing through empty buckets at the moment. - c.currLe = getLe(c.currIdx, c.h.Schema) + c.currUpper = getUpper(c.currIdx, c.h.Schema) c.currIdx++ c.emptyBucketCount-- return true @@ -145,7 +175,7 @@ func (c *cumulativeBucketIterator) Next() bool { c.currCount += c.h.PositiveBuckets[c.posBucketsIdx] c.currCumulativeCount += uint64(c.currCount) - c.currLe = getLe(c.currIdx, c.h.Schema) + c.currUpper = getUpper(c.currIdx, c.h.Schema) c.posBucketsIdx++ c.idxInSpan++ @@ -163,24 +193,26 @@ func (c *cumulativeBucketIterator) Next() bool { } func (c *cumulativeBucketIterator) At() Bucket { return Bucket{ - Le: c.currLe, + Upper: c.currUpper, Count: c.currCumulativeCount, } } func (c *cumulativeBucketIterator) Err() error { return nil } -func getLe(idx, schema int32) float64 { +func getUpper(idx, schema int32) float64 { if schema < 0 { return math.Ldexp(1, int(idx)<<(-schema)) } fracIdx := idx & ((1 << schema) - 1) - frac := sparseBounds[schema][fracIdx] + frac := exponentialBounds[schema][fracIdx] exp := (int(idx) >> schema) + 1 return math.Ldexp(frac, exp) } -var sparseBounds = [][]float64{ +// exponentialBounds is a precalculated table of bucket bounds in the interval +// [0.5,1) in schema 0 to 8. +var exponentialBounds = [][]float64{ // Schema "0": {0.5}, // Schema 1: diff --git a/model/histogram/histogram_test.go b/model/histogram/histogram_test.go new file mode 100644 index 0000000000..2a33fdb85b --- /dev/null +++ b/model/histogram/histogram_test.go @@ -0,0 +1,165 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package histogram + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCumulativeBucketIterator(t *testing.T) { + cases := []struct { + histogram Histogram + expectedCumulativeBuckets []Bucket + }{ + { + histogram: Histogram{ + Schema: 0, + PositiveSpans: []Span{ + {Offset: 0, Length: 2}, + {Offset: 1, Length: 2}, + }, + PositiveBuckets: []int64{1, 1, -1, 0}, + }, + expectedCumulativeBuckets: []Bucket{ + {Upper: 1, Count: 1}, + {Upper: 2, Count: 3}, + + {Upper: 4, Count: 3}, + + {Upper: 8, Count: 4}, + {Upper: 16, Count: 5}, + }, + }, + { + histogram: Histogram{ + Schema: 0, + PositiveSpans: []Span{ + {Offset: 0, Length: 5}, + {Offset: 1, Length: 1}, + }, + PositiveBuckets: []int64{1, 2, -2, 1, -1, 0}, + }, + expectedCumulativeBuckets: []Bucket{ + {Upper: 1, Count: 1}, + {Upper: 2, Count: 4}, + {Upper: 4, Count: 5}, + {Upper: 8, Count: 7}, + + {Upper: 16, Count: 8}, + + {Upper: 32, Count: 8}, + {Upper: 64, Count: 9}, + }, + }, + { + histogram: Histogram{ + Schema: 0, + PositiveSpans: []Span{ + {Offset: 0, Length: 7}, + }, + PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 0}, + }, + expectedCumulativeBuckets: []Bucket{ + {Upper: 1, Count: 1}, + {Upper: 2, Count: 4}, + {Upper: 4, Count: 5}, + {Upper: 8, Count: 7}, + {Upper: 16, Count: 8}, + {Upper: 32, Count: 9}, + {Upper: 64, Count: 10}, + }, + }, + { + histogram: Histogram{ + Schema: 3, + PositiveSpans: []Span{ + {Offset: -5, Length: 2}, // -5 -4 + {Offset: 2, Length: 3}, // -1 0 1 + {Offset: 2, Length: 2}, // 4 5 + }, + PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 3}, + }, + expectedCumulativeBuckets: []Bucket{ + {Upper: 0.6484197773255048, Count: 1}, // -5 + {Upper: 0.7071067811865475, Count: 4}, // -4 + + {Upper: 0.7711054127039704, Count: 4}, // -3 + {Upper: 0.8408964152537144, Count: 4}, // -2 + + {Upper: 0.9170040432046711, Count: 5}, // -1 + {Upper: 1, Count: 7}, // 1 + {Upper: 1.0905077326652577, Count: 8}, // 0 + + {Upper: 1.189207115002721, Count: 8}, // 1 + {Upper: 1.2968395546510096, Count: 8}, // 2 + + {Upper: 1.414213562373095, Count: 9}, // 3 + {Upper: 1.5422108254079407, Count: 13}, // 4 + }, + }, + { + histogram: Histogram{ + Schema: -2, + PositiveSpans: []Span{ + {Offset: -2, Length: 4}, // -2 -1 0 1 + {Offset: 2, Length: 2}, // 4 5 + }, + PositiveBuckets: []int64{1, 2, -2, 1, -1, 0}, + }, + expectedCumulativeBuckets: []Bucket{ + {Upper: 0.00390625, Count: 1}, // -2 + {Upper: 0.0625, Count: 4}, // -1 + {Upper: 1, Count: 5}, // 0 + {Upper: 16, Count: 7}, // 1 + + {Upper: 256, Count: 7}, // 2 + {Upper: 4096, Count: 7}, // 3 + + {Upper: 65536, Count: 8}, // 4 + {Upper: 1048576, Count: 9}, // 5 + }, + }, + { + histogram: Histogram{ + Schema: -1, + PositiveSpans: []Span{ + {Offset: -2, Length: 5}, // -2 -1 0 1 2 + }, + PositiveBuckets: []int64{1, 2, -2, 1, -1}, + }, + expectedCumulativeBuckets: []Bucket{ + {Upper: 0.0625, Count: 1}, // -2 + {Upper: 0.25, Count: 4}, // -1 + {Upper: 1, Count: 5}, // 0 + {Upper: 4, Count: 7}, // 1 + {Upper: 16, Count: 8}, // 2 + }, + }, + } + + for i, c := range cases { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + it := c.histogram.CumulativeBucketIterator() + actualBuckets := make([]Bucket, 0, len(c.expectedCumulativeBuckets)) + for it.Next() { + actualBuckets = append(actualBuckets, it.At()) + } + require.NoError(t, it.Err()) + require.Equal(t, c.expectedCumulativeBuckets, actualBuckets) + }) + } +} diff --git a/pkg/histogram/sparse_histogram_test.go b/pkg/histogram/sparse_histogram_test.go deleted file mode 100644 index 1a59e5895a..0000000000 --- a/pkg/histogram/sparse_histogram_test.go +++ /dev/null @@ -1,165 +0,0 @@ -// Copyright 2021 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package histogram - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestCumulativeExpandSparseHistogram(t *testing.T) { - cases := []struct { - hist SparseHistogram - expBuckets []Bucket - }{ - { - hist: SparseHistogram{ - Schema: 0, - PositiveSpans: []Span{ - {Offset: 0, Length: 2}, - {Offset: 1, Length: 2}, - }, - PositiveBuckets: []int64{1, 1, -1, 0}, - }, - expBuckets: []Bucket{ - {Le: 1, Count: 1}, - {Le: 2, Count: 3}, - - {Le: 4, Count: 3}, - - {Le: 8, Count: 4}, - {Le: 16, Count: 5}, - }, - }, - { - hist: SparseHistogram{ - Schema: 0, - PositiveSpans: []Span{ - {Offset: 0, Length: 5}, - {Offset: 1, Length: 1}, - }, - PositiveBuckets: []int64{1, 2, -2, 1, -1, 0}, - }, - expBuckets: []Bucket{ - {Le: 1, Count: 1}, - {Le: 2, Count: 4}, - {Le: 4, Count: 5}, - {Le: 8, Count: 7}, - - {Le: 16, Count: 8}, - - {Le: 32, Count: 8}, - {Le: 64, Count: 9}, - }, - }, - { - hist: SparseHistogram{ - Schema: 0, - PositiveSpans: []Span{ - {Offset: 0, Length: 7}, - }, - PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 0}, - }, - expBuckets: []Bucket{ - {Le: 1, Count: 1}, - {Le: 2, Count: 4}, - {Le: 4, Count: 5}, - {Le: 8, Count: 7}, - {Le: 16, Count: 8}, - {Le: 32, Count: 9}, - {Le: 64, Count: 10}, - }, - }, - { - hist: SparseHistogram{ - Schema: 3, - PositiveSpans: []Span{ - {Offset: -5, Length: 2}, // -5 -4 - {Offset: 2, Length: 3}, // -1 0 1 - {Offset: 2, Length: 2}, // 4 5 - }, - PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 3}, - }, - expBuckets: []Bucket{ - {Le: 0.6484197773255048, Count: 1}, // -5 - {Le: 0.7071067811865475, Count: 4}, // -4 - - {Le: 0.7711054127039704, Count: 4}, // -3 - {Le: 0.8408964152537144, Count: 4}, // -2 - - {Le: 0.9170040432046711, Count: 5}, // -1 - {Le: 1, Count: 7}, // 1 - {Le: 1.0905077326652577, Count: 8}, // 0 - - {Le: 1.189207115002721, Count: 8}, // 1 - {Le: 1.2968395546510096, Count: 8}, // 2 - - {Le: 1.414213562373095, Count: 9}, // 3 - {Le: 1.5422108254079407, Count: 13}, // 4 - }, - }, - { - hist: SparseHistogram{ - Schema: -2, - PositiveSpans: []Span{ - {Offset: -2, Length: 4}, // -2 -1 0 1 - {Offset: 2, Length: 2}, // 4 5 - }, - PositiveBuckets: []int64{1, 2, -2, 1, -1, 0}, - }, - expBuckets: []Bucket{ - {Le: 0.00390625, Count: 1}, // -2 - {Le: 0.0625, Count: 4}, // -1 - {Le: 1, Count: 5}, // 0 - {Le: 16, Count: 7}, // 1 - - {Le: 256, Count: 7}, // 2 - {Le: 4096, Count: 7}, // 3 - - {Le: 65536, Count: 8}, // 4 - {Le: 1048576, Count: 9}, // 5 - }, - }, - { - hist: SparseHistogram{ - Schema: -1, - PositiveSpans: []Span{ - {Offset: -2, Length: 5}, // -2 -1 0 1 2 - }, - PositiveBuckets: []int64{1, 2, -2, 1, -1}, - }, - expBuckets: []Bucket{ - {Le: 0.0625, Count: 1}, // -2 - {Le: 0.25, Count: 4}, // -1 - {Le: 1, Count: 5}, // 0 - {Le: 4, Count: 7}, // 1 - {Le: 16, Count: 8}, // 2 - }, - }, - } - - for i, c := range cases { - t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { - it := CumulativeExpandSparseHistogram(c.hist) - actBuckets := make([]Bucket, 0, len(c.expBuckets)) - for it.Next() { - actBuckets = append(actBuckets, it.At()) - } - require.NoError(t, it.Err()) - require.Equal(t, c.expBuckets, actBuckets) - }) - } -} diff --git a/pkg/textparse/interface.go b/pkg/textparse/interface.go index 1dbcc51d8a..e55e628b6c 100644 --- a/pkg/textparse/interface.go +++ b/pkg/textparse/interface.go @@ -16,8 +16,8 @@ package textparse import ( "mime" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/pkg/exemplar" - "github.com/prometheus/prometheus/pkg/histogram" "github.com/prometheus/prometheus/pkg/labels" ) @@ -29,9 +29,8 @@ type Parser interface { Series() ([]byte, *int64, float64) // Histogram returns the bytes of a series with a sparse histogram as a - // value, the timestamp if set, and the sparse histogram in the current - // sample. - Histogram() ([]byte, *int64, histogram.SparseHistogram) + // value, the timestamp if set, and the histogram in the current sample. + Histogram() ([]byte, *int64, histogram.Histogram) // Help returns the metric name and help text in the current entry. // Must only be called after Next returned a help entry. diff --git a/pkg/textparse/openmetricsparse.go b/pkg/textparse/openmetricsparse.go index ef1719321d..265ca22b18 100644 --- a/pkg/textparse/openmetricsparse.go +++ b/pkg/textparse/openmetricsparse.go @@ -27,8 +27,8 @@ import ( "github.com/pkg/errors" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/pkg/exemplar" - "github.com/prometheus/prometheus/pkg/histogram" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/value" ) @@ -114,10 +114,10 @@ func (p *OpenMetricsParser) Series() ([]byte, *int64, float64) { return p.series, nil, p.val } -// Histogram always returns (nil, nil, SparseHistogram{}) because OpenMetrics -// does not support sparse histograms. -func (p *OpenMetricsParser) Histogram() ([]byte, *int64, histogram.SparseHistogram) { - return nil, nil, histogram.SparseHistogram{} +// Histogram always returns (nil, nil, histogram.Histogram{}) because +// OpenMetrics does not support sparse histograms. +func (p *OpenMetricsParser) Histogram() ([]byte, *int64, histogram.Histogram) { + return nil, nil, histogram.Histogram{} } // Help returns the metric name and help text in the current entry. diff --git a/pkg/textparse/promparse.go b/pkg/textparse/promparse.go index 407dcea6d2..c7059a2b13 100644 --- a/pkg/textparse/promparse.go +++ b/pkg/textparse/promparse.go @@ -28,8 +28,8 @@ import ( "github.com/pkg/errors" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/pkg/exemplar" - "github.com/prometheus/prometheus/pkg/histogram" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/value" ) @@ -169,10 +169,10 @@ func (p *PromParser) Series() ([]byte, *int64, float64) { return p.series, nil, p.val } -// Histogram always returns (nil, nil, SparseHistogram{}) because the Prometheus -// text format does not support sparse histograms. -func (p *PromParser) Histogram() ([]byte, *int64, histogram.SparseHistogram) { - return nil, nil, histogram.SparseHistogram{} +// Histogram always returns (nil, nil, histogram.Histogram{}) because the +// Prometheus text format does not support sparse histograms. +func (p *PromParser) Histogram() ([]byte, *int64, histogram.Histogram) { + return nil, nil, histogram.Histogram{} } // Help returns the metric name and help text in the current entry. diff --git a/pkg/textparse/protobufparse.go b/pkg/textparse/protobufparse.go index 575c806a87..5a43108f13 100644 --- a/pkg/textparse/protobufparse.go +++ b/pkg/textparse/protobufparse.go @@ -27,8 +27,8 @@ import ( "github.com/pkg/errors" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/pkg/exemplar" - "github.com/prometheus/prometheus/pkg/histogram" "github.com/prometheus/prometheus/pkg/labels" dto "github.com/prometheus/prometheus/prompb/io/prometheus/client" @@ -65,6 +65,7 @@ type ProtobufParser struct { metricBytes *bytes.Buffer // A somewhat fluid representation of the current metric. } +// NewProtobufParser returns a parser for the payload in the byte slice. func NewProtobufParser(b []byte) Parser { return &ProtobufParser{ in: b, @@ -134,13 +135,13 @@ func (p *ProtobufParser) Series() ([]byte, *int64, float64) { // Histogram returns the bytes of a series with a sparse histogram as a // value, the timestamp if set, and the sparse histogram in the current // sample. -func (p *ProtobufParser) Histogram() ([]byte, *int64, histogram.SparseHistogram) { +func (p *ProtobufParser) Histogram() ([]byte, *int64, histogram.Histogram) { var ( m = p.mf.GetMetric()[p.metricPos] ts = m.GetTimestampMs() h = m.GetHistogram() ) - sh := histogram.SparseHistogram{ + sh := histogram.Histogram{ Count: h.GetSampleCount(), Sum: h.GetSampleSum(), ZeroThreshold: h.GetSbZeroThreshold(), diff --git a/pkg/textparse/protobufparse_test.go b/pkg/textparse/protobufparse_test.go index 06e06b83d5..9df8c4e632 100644 --- a/pkg/textparse/protobufparse_test.go +++ b/pkg/textparse/protobufparse_test.go @@ -22,8 +22,8 @@ import ( "github.com/gogo/protobuf/proto" "github.com/stretchr/testify/require" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/pkg/exemplar" - "github.com/prometheus/prometheus/pkg/histogram" "github.com/prometheus/prometheus/pkg/labels" dto "github.com/prometheus/prometheus/prompb/io/prometheus/client" @@ -266,7 +266,7 @@ metric: < help string unit string comment string - shs histogram.SparseHistogram + shs histogram.Histogram e []exemplar.Exemplar }{ { @@ -332,7 +332,7 @@ metric: < { m: "test_histogram", t: 1234568, - shs: histogram.SparseHistogram{ + shs: histogram.Histogram{ Count: 175, ZeroCount: 2, Sum: 0.0008280461746287094, diff --git a/promql/value.go b/promql/value.go index 0936bb6c73..3a724d0533 100644 --- a/promql/value.go +++ b/promql/value.go @@ -21,7 +21,7 @@ import ( "github.com/pkg/errors" - "github.com/prometheus/prometheus/pkg/histogram" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/promql/parser" "github.com/prometheus/prometheus/storage" @@ -296,8 +296,11 @@ func (ssi *storageSeriesIterator) At() (t int64, v float64) { return p.T, p.V } -func (ssi *storageSeriesIterator) AtHistogram() (int64, histogram.SparseHistogram) { - return 0, histogram.SparseHistogram{} +// AtHistogram always returns (0, histogram.Histogram{}) because there is no +// support for histogram values yet. +// TODO(beorn7): Fix that for histogram support in PromQL. +func (ssi *storageSeriesIterator) AtHistogram() (int64, histogram.Histogram) { + return 0, histogram.Histogram{} } func (ssi *storageSeriesIterator) ChunkEncoding() chunkenc.Encoding { diff --git a/scrape/helpers_test.go b/scrape/helpers_test.go index 1f2e065f18..16eb8e272b 100644 --- a/scrape/helpers_test.go +++ b/scrape/helpers_test.go @@ -17,8 +17,8 @@ import ( "context" "math/rand" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/pkg/exemplar" - "github.com/prometheus/prometheus/pkg/histogram" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/storage" ) @@ -35,7 +35,7 @@ func (a nopAppender) Append(uint64, labels.Labels, int64, float64) (uint64, erro func (a nopAppender) AppendExemplar(uint64, labels.Labels, exemplar.Exemplar) (uint64, error) { return 0, nil } -func (a nopAppender) AppendHistogram(uint64, labels.Labels, int64, histogram.SparseHistogram) (uint64, error) { +func (a nopAppender) AppendHistogram(uint64, labels.Labels, int64, histogram.Histogram) (uint64, error) { return 0, nil } func (a nopAppender) Commit() error { return nil } @@ -47,9 +47,9 @@ type sample struct { v float64 } -type hist struct { - h histogram.SparseHistogram +type histogramSample struct { t int64 + h histogram.Histogram } // collectResultAppender records all samples that were added through the appender. @@ -61,9 +61,9 @@ type collectResultAppender struct { rolledbackResult []sample pendingExemplars []exemplar.Exemplar resultExemplars []exemplar.Exemplar - resultHistograms []hist - pendingHistograms []hist - rolledbackHistograms []hist + resultHistograms []histogramSample + pendingHistograms []histogramSample + rolledbackHistograms []histogramSample } func (a *collectResultAppender) Append(ref uint64, lset labels.Labels, t int64, v float64) (uint64, error) { @@ -96,13 +96,13 @@ func (a *collectResultAppender) AppendExemplar(ref uint64, l labels.Labels, e ex return a.next.AppendExemplar(ref, l, e) } -func (a *collectResultAppender) AppendHistogram(ref uint64, l labels.Labels, t int64, sh histogram.SparseHistogram) (uint64, error) { - a.pendingHistograms = append(a.pendingHistograms, hist{h: sh, t: t}) +func (a *collectResultAppender) AppendHistogram(ref uint64, l labels.Labels, t int64, h histogram.Histogram) (uint64, error) { + a.pendingHistograms = append(a.pendingHistograms, histogramSample{h: h, t: t}) if a.next == nil { return 0, nil } - return a.next.AppendHistogram(ref, l, t, sh) + return a.next.AppendHistogram(ref, l, t, h) } func (a *collectResultAppender) Commit() error { diff --git a/scrape/scrape.go b/scrape/scrape.go index a6a799658f..40c4c6bbb3 100644 --- a/scrape/scrape.go +++ b/scrape/scrape.go @@ -39,8 +39,8 @@ import ( "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/discovery/targetgroup" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/pkg/exemplar" - "github.com/prometheus/prometheus/pkg/histogram" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/pool" "github.com/prometheus/prometheus/pkg/relabel" @@ -1411,7 +1411,7 @@ loop: met []byte parsedTimestamp *int64 val float64 - his histogram.SparseHistogram + h histogram.Histogram ) if et, err = p.Next(); err != nil { if err == io.EOF { @@ -1439,7 +1439,7 @@ loop: t := defTime if isHistogram { - met, parsedTimestamp, his = p.Histogram() + met, parsedTimestamp, h = p.Histogram() } else { met, parsedTimestamp, val = p.Series() } @@ -1491,7 +1491,7 @@ loop: } if isHistogram { - ref, err = app.AppendHistogram(ref, lset, t, his) + ref, err = app.AppendHistogram(ref, lset, t, h) } else { ref, err = app.Append(ref, lset, t, val) } @@ -1554,7 +1554,6 @@ loop: if err == nil { sl.cache.forEachStale(func(lset labels.Labels) bool { // Series no longer exposed, mark it stale. - // TODO(beorn7): Appending staleness markers breaks horribly for histograms. _, err = app.Append(0, lset, defTime, math.Float64frombits(value.StaleNaN)) switch errors.Cause(err) { case storage.ErrOutOfOrderSample, storage.ErrDuplicateSampleForTimestamp: diff --git a/storage/buffer.go b/storage/buffer.go index 82e5a28324..2d06d3fb26 100644 --- a/storage/buffer.go +++ b/storage/buffer.go @@ -16,7 +16,7 @@ package storage import ( "math" - "github.com/prometheus/prometheus/pkg/histogram" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/tsdb/chunkenc" ) @@ -198,8 +198,11 @@ func (it *sampleRingIterator) At() (int64, float64) { return it.r.at(it.i) } -func (it *sampleRingIterator) AtHistogram() (int64, histogram.SparseHistogram) { - return 0, histogram.SparseHistogram{} +// AtHistogram always returns (0, histogram.Histogram{}) because there is no +// support for histogram values yet. +// TODO(beorn7): Fix that for histogram support in PromQL. +func (it *sampleRingIterator) AtHistogram() (int64, histogram.Histogram) { + return 0, histogram.Histogram{} } func (it *sampleRingIterator) ChunkEncoding() chunkenc.Encoding { diff --git a/storage/buffer_test.go b/storage/buffer_test.go index c242bfe10f..032c61f078 100644 --- a/storage/buffer_test.go +++ b/storage/buffer_test.go @@ -17,7 +17,7 @@ import ( "math/rand" "testing" - "github.com/prometheus/prometheus/pkg/histogram" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/stretchr/testify/require" ) @@ -196,8 +196,8 @@ type mockSeriesIterator struct { func (m *mockSeriesIterator) Seek(t int64) bool { return m.seek(t) } func (m *mockSeriesIterator) At() (int64, float64) { return m.at() } -func (m *mockSeriesIterator) AtHistogram() (int64, histogram.SparseHistogram) { - return 0, histogram.SparseHistogram{} +func (m *mockSeriesIterator) AtHistogram() (int64, histogram.Histogram) { + return 0, histogram.Histogram{} } func (m *mockSeriesIterator) ChunkEncoding() chunkenc.Encoding { return chunkenc.EncXOR @@ -219,8 +219,8 @@ func (it *fakeSeriesIterator) At() (int64, float64) { return it.idx * it.step, 123 // value doesn't matter } -func (it *fakeSeriesIterator) AtHistogram() (int64, histogram.SparseHistogram) { - return it.idx * it.step, histogram.SparseHistogram{} // value doesn't matter +func (it *fakeSeriesIterator) AtHistogram() (int64, histogram.Histogram) { + return it.idx * it.step, histogram.Histogram{} // value doesn't matter } func (it *fakeSeriesIterator) ChunkEncoding() chunkenc.Encoding { diff --git a/storage/fanout.go b/storage/fanout.go index 754544d35f..ccb123895e 100644 --- a/storage/fanout.go +++ b/storage/fanout.go @@ -20,8 +20,8 @@ import ( "github.com/go-kit/log/level" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/pkg/exemplar" - "github.com/prometheus/prometheus/pkg/histogram" "github.com/prometheus/prometheus/pkg/labels" tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" ) @@ -173,14 +173,14 @@ func (f *fanoutAppender) AppendExemplar(ref uint64, l labels.Labels, e exemplar. return ref, nil } -func (f *fanoutAppender) AppendHistogram(ref uint64, l labels.Labels, t int64, sh histogram.SparseHistogram) (uint64, error) { - ref, err := f.primary.AppendHistogram(ref, l, t, sh) +func (f *fanoutAppender) AppendHistogram(ref uint64, l labels.Labels, t int64, h histogram.Histogram) (uint64, error) { + ref, err := f.primary.AppendHistogram(ref, l, t, h) if err != nil { return ref, err } for _, appender := range f.secondaries { - if _, err := appender.AppendHistogram(ref, l, t, sh); err != nil { + if _, err := appender.AppendHistogram(ref, l, t, h); err != nil { return 0, err } } diff --git a/storage/interface.go b/storage/interface.go index bd4e0c6c79..be55a5bb2c 100644 --- a/storage/interface.go +++ b/storage/interface.go @@ -18,8 +18,8 @@ import ( "errors" "fmt" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/pkg/exemplar" - "github.com/prometheus/prometheus/pkg/histogram" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/chunks" @@ -213,17 +213,16 @@ type ExemplarAppender interface { AppendExemplar(ref uint64, l labels.Labels, e exemplar.Exemplar) (uint64, error) } -// HistogramAppender provides an interface for adding sparse histogram to the Prometheus. +// HistogramAppender provides an interface for appending histograms to the storage. type HistogramAppender interface { - // AppendHistogram adds a sparse histogram for the given series labels. - // An optional reference number can be provided to accelerate calls. - // A reference number is returned which can be used to add further - // histograms in the same or later transactions. - // Returned reference numbers are ephemeral and may be rejected in calls - // to Append() at any point. Adding the sample via Append() returns a new - // reference number. + // AppendHistogram adds a histogram for the given series labels. An + // optional reference number can be provided to accelerate calls. A + // reference number is returned which can be used to add further + // histograms in the same or later transactions. Returned reference + // numbers are ephemeral and may be rejected in calls to Append() at any + // point. Adding the sample via Append() returns a new reference number. // If the reference is 0 it must not be used for caching. - AppendHistogram(ref uint64, l labels.Labels, t int64, sh histogram.SparseHistogram) (uint64, error) + AppendHistogram(ref uint64, l labels.Labels, t int64, h histogram.Histogram) (uint64, error) } // SeriesSet contains a set of series. diff --git a/storage/merge.go b/storage/merge.go index 3021a7ee8a..6327feffc4 100644 --- a/storage/merge.go +++ b/storage/merge.go @@ -22,7 +22,7 @@ import ( "github.com/pkg/errors" - "github.com/prometheus/prometheus/pkg/histogram" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/chunks" @@ -465,7 +465,7 @@ func (c *chainSampleIterator) Seek(t int64) bool { } if len(c.h) > 0 { c.curr = heap.Pop(&c.h).(chunkenc.Iterator) - if c.curr.ChunkEncoding() == chunkenc.EncSHS { + if c.curr.ChunkEncoding() == chunkenc.EncHistogram { c.lastt, _ = c.curr.AtHistogram() } else { c.lastt, _ = c.curr.At() @@ -483,7 +483,7 @@ func (c *chainSampleIterator) At() (t int64, v float64) { return c.curr.At() } -func (c *chainSampleIterator) AtHistogram() (int64, histogram.SparseHistogram) { +func (c *chainSampleIterator) AtHistogram() (int64, histogram.Histogram) { if c.curr == nil { panic("chainSampleIterator.AtHistogram() called before first .Next() or after .Next() returned false.") } @@ -517,7 +517,7 @@ func (c *chainSampleIterator) Next() bool { var currt int64 for { if c.curr.Next() { - if c.curr.ChunkEncoding() == chunkenc.EncSHS { + if c.curr.ChunkEncoding() == chunkenc.EncHistogram { currt, _ = c.curr.AtHistogram() } else { currt, _ = c.curr.At() @@ -534,7 +534,7 @@ func (c *chainSampleIterator) Next() bool { // Check current iterator with the top of the heap. var nextt int64 - if c.h[0].ChunkEncoding() == chunkenc.EncSHS { + if c.h[0].ChunkEncoding() == chunkenc.EncHistogram { nextt, _ = c.h[0].AtHistogram() } else { nextt, _ = c.h[0].At() @@ -552,7 +552,7 @@ func (c *chainSampleIterator) Next() bool { } c.curr = heap.Pop(&c.h).(chunkenc.Iterator) - if c.curr.ChunkEncoding() == chunkenc.EncSHS { + if c.curr.ChunkEncoding() == chunkenc.EncHistogram { currt, _ = c.curr.AtHistogram() } else { currt, _ = c.curr.At() @@ -581,12 +581,12 @@ func (h samplesIteratorHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } func (h samplesIteratorHeap) Less(i, j int) bool { var at, bt int64 - if h[i].ChunkEncoding() == chunkenc.EncSHS { + if h[i].ChunkEncoding() == chunkenc.EncHistogram { at, _ = h[i].AtHistogram() } else { at, _ = h[i].At() } - if h[j].ChunkEncoding() == chunkenc.EncSHS { + if h[j].ChunkEncoding() == chunkenc.EncHistogram { bt, _ = h[j].AtHistogram() } else { bt, _ = h[j].At() diff --git a/storage/remote/codec.go b/storage/remote/codec.go index 1fffc0a056..9e414ad833 100644 --- a/storage/remote/codec.go +++ b/storage/remote/codec.go @@ -26,8 +26,8 @@ import ( "github.com/pkg/errors" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/pkg/exemplar" - "github.com/prometheus/prometheus/pkg/histogram" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/textparse" "github.com/prometheus/prometheus/prompb" @@ -370,8 +370,11 @@ func (c *concreteSeriesIterator) At() (t int64, v float64) { return s.Timestamp, s.Value } -func (c *concreteSeriesIterator) AtHistogram() (int64, histogram.SparseHistogram) { - return 0, histogram.SparseHistogram{} +// AtHistogram always returns (0, histogram.Histogram{}) because there is no +// support for histogram values yet. +// TODO(beorn7): Fix that for histogram support in remote storage. +func (c *concreteSeriesIterator) AtHistogram() (int64, histogram.Histogram) { + return 0, histogram.Histogram{} } func (c *concreteSeriesIterator) ChunkEncoding() chunkenc.Encoding { diff --git a/storage/remote/write.go b/storage/remote/write.go index 3f7dd8a06d..69fe363611 100644 --- a/storage/remote/write.go +++ b/storage/remote/write.go @@ -24,8 +24,8 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/pkg/exemplar" - "github.com/prometheus/prometheus/pkg/histogram" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/wal" @@ -241,7 +241,7 @@ func (t *timestampTracker) AppendExemplar(_ uint64, _ labels.Labels, _ exemplar. return 0, nil } -func (t *timestampTracker) AppendHistogram(_ uint64, _ labels.Labels, ts int64, _ histogram.SparseHistogram) (uint64, error) { +func (t *timestampTracker) AppendHistogram(_ uint64, _ labels.Labels, ts int64, _ histogram.Histogram) (uint64, error) { t.histograms++ if ts > t.highestTimestamp { t.highestTimestamp = ts diff --git a/storage/remote/write_handler_test.go b/storage/remote/write_handler_test.go index 7fb61df0fa..72603f2970 100644 --- a/storage/remote/write_handler_test.go +++ b/storage/remote/write_handler_test.go @@ -25,8 +25,8 @@ import ( "github.com/go-kit/log" "github.com/stretchr/testify/require" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/pkg/exemplar" - "github.com/prometheus/prometheus/pkg/histogram" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/prompb" "github.com/prometheus/prometheus/storage" @@ -188,7 +188,7 @@ func (m *mockAppendable) AppendExemplar(_ uint64, l labels.Labels, e exemplar.Ex return 0, nil } -func (*mockAppendable) AppendHistogram(ref uint64, l labels.Labels, t int64, sh histogram.SparseHistogram) (uint64, error) { - // noop until we implement sparse histograms over remote write +func (*mockAppendable) AppendHistogram(ref uint64, l labels.Labels, t int64, h histogram.Histogram) (uint64, error) { + // TODO(beorn7): Noop until we implement sparse histograms over remote write. return 0, nil } diff --git a/storage/series.go b/storage/series.go index a7c559f2b7..60541164b4 100644 --- a/storage/series.go +++ b/storage/series.go @@ -17,7 +17,7 @@ import ( "math" "sort" - "github.com/prometheus/prometheus/pkg/histogram" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/chunks" @@ -91,8 +91,10 @@ func (it *listSeriesIterator) At() (int64, float64) { return s.T(), s.V() } -func (it *listSeriesIterator) AtHistogram() (int64, histogram.SparseHistogram) { - return 0, histogram.SparseHistogram{} +// AtHistogram always returns (0, histogram.Histogram{}) because there is no +// support for histogram values yet. +func (it *listSeriesIterator) AtHistogram() (int64, histogram.Histogram) { + return 0, histogram.Histogram{} } func (it *listSeriesIterator) ChunkEncoding() chunkenc.Encoding { diff --git a/tsdb/chunkenc/chunk.go b/tsdb/chunkenc/chunk.go index a356ea019e..546937dc0f 100644 --- a/tsdb/chunkenc/chunk.go +++ b/tsdb/chunkenc/chunk.go @@ -18,7 +18,7 @@ import ( "sync" "github.com/pkg/errors" - "github.com/prometheus/prometheus/pkg/histogram" + "github.com/prometheus/prometheus/model/histogram" ) // Encoding is the identifier for a chunk encoding. @@ -30,8 +30,8 @@ func (e Encoding) String() string { return "none" case EncXOR: return "XOR" - case EncSHS: - return "SHS" + case EncHistogram: + return "histogram" } return "" } @@ -39,7 +39,7 @@ func (e Encoding) String() string { // IsValidEncoding returns true for supported encodings. func IsValidEncoding(e Encoding) bool { switch e { - case EncXOR, EncSHS: + case EncXOR, EncHistogram: return true } return false @@ -49,7 +49,7 @@ func IsValidEncoding(e Encoding) bool { const ( EncNone Encoding = iota EncXOR - EncSHS + EncHistogram ) // Chunk holds a sequence of sample pairs that can be iterated over and appended to. @@ -82,7 +82,7 @@ type Chunk interface { // Appender adds sample pairs to a chunk. type Appender interface { Append(int64, float64) - AppendHistogram(t int64, h histogram.SparseHistogram) + AppendHistogram(t int64, h histogram.Histogram) } // Iterator is a simple iterator that can only get the next value. @@ -100,7 +100,7 @@ type Iterator interface { At() (int64, float64) // AtHistogram returns the current timestamp/histogram pair. // Before the iterator has advanced AtHistogram behaviour is unspecified. - AtHistogram() (int64, histogram.SparseHistogram) + AtHistogram() (int64, histogram.Histogram) // Err returns the current error. It should be used only after iterator is // exhausted, that is `Next` or `Seek` returns false. Err() error @@ -117,8 +117,8 @@ type nopIterator struct{} func (nopIterator) Seek(int64) bool { return false } func (nopIterator) At() (int64, float64) { return math.MinInt64, 0 } -func (nopIterator) AtHistogram() (int64, histogram.SparseHistogram) { - return math.MinInt64, histogram.SparseHistogram{} +func (nopIterator) AtHistogram() (int64, histogram.Histogram) { + return math.MinInt64, histogram.Histogram{} } func (nopIterator) Next() bool { return false } func (nopIterator) Err() error { return nil } @@ -132,8 +132,8 @@ type Pool interface { // pool is a memory pool of chunk objects. type pool struct { - xor sync.Pool - shs sync.Pool + xor sync.Pool + histogram sync.Pool } // NewPool returns a new pool. @@ -144,9 +144,9 @@ func NewPool() Pool { return &XORChunk{b: bstream{}} }, }, - shs: sync.Pool{ + histogram: sync.Pool{ New: func() interface{} { - return &HistoChunk{b: bstream{}} + return &HistogramChunk{b: bstream{}} }, }, } @@ -159,9 +159,9 @@ func (p *pool) Get(e Encoding, b []byte) (Chunk, error) { c.b.stream = b c.b.count = 0 return c, nil - case EncSHS: + case EncHistogram: // TODO: update metadata - c := p.shs.Get().(*HistoChunk) + c := p.histogram.Get().(*HistogramChunk) c.b.stream = b c.b.count = 0 return c, nil @@ -182,9 +182,9 @@ func (p *pool) Put(c Chunk) error { xc.b.stream = nil xc.b.count = 0 p.xor.Put(c) - case EncSHS: + case EncHistogram: // TODO: update metadata - sh, ok := c.(*HistoChunk) + sh, ok := c.(*HistogramChunk) // This may happen often with wrapped chunks. Nothing we can really do about // it but returning an error would cause a lot of allocations again. Thus, // we just skip it. @@ -193,7 +193,7 @@ func (p *pool) Put(c Chunk) error { } sh.b.stream = nil sh.b.count = 0 - p.shs.Put(c) + p.histogram.Put(c) default: return errors.Errorf("invalid chunk encoding %q", c.Encoding()) } @@ -207,9 +207,9 @@ func FromData(e Encoding, d []byte) (Chunk, error) { switch e { case EncXOR: return &XORChunk{b: bstream{count: 0, stream: d}}, nil - case EncSHS: + case EncHistogram: // TODO: update metadata - return &HistoChunk{b: bstream{count: 0, stream: d}}, nil + return &HistogramChunk{b: bstream{count: 0, stream: d}}, nil } return nil, errors.Errorf("invalid chunk encoding %q", e) } @@ -219,8 +219,8 @@ func NewEmptyChunk(e Encoding) (Chunk, error) { switch e { case EncXOR: return NewXORChunk(), nil - case EncSHS: - return NewHistoChunk(), nil + case EncHistogram: + return NewHistogramChunk(), nil } return nil, errors.Errorf("invalid chunk encoding %q", e) } diff --git a/tsdb/chunkenc/histo.go b/tsdb/chunkenc/histo.go deleted file mode 100644 index db83c53e3a..0000000000 --- a/tsdb/chunkenc/histo.go +++ /dev/null @@ -1,943 +0,0 @@ -// Copyright 2021 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The code in this file was largely written by Damian Gryski as part of -// https://github.com/dgryski/go-tsz and published under the license below. -// It was modified to accommodate reading from byte slices without modifying -// the underlying bytes, which would panic when reading from mmap'd -// read-only byte slices. - -// Copyright (c) 2015,2016 Damian Gryski -// All rights reserved. - -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: - -// * Redistributions of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// * Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package chunkenc - -import ( - "encoding/binary" - "math" - "math/bits" - - "github.com/prometheus/prometheus/pkg/histogram" - "github.com/prometheus/prometheus/pkg/value" -) - -const () - -// HistoChunk holds sparse histogram encoded sample data. -// Appends a histogram sample -// * schema defines the resolution (number of buckets per power of 2) -// Currently, valid numbers are -4 <= n <= 8. -// They are all for base-2 bucket schemas, where 1 is a bucket boundary in each case, and -// then each power of two is divided into 2^n logarithmic buckets. -// Or in other words, each bucket boundary is the previous boundary times 2^(2^-n). -// In the future, more bucket schemas may be added using numbers < -4 or > 8. -// The bucket with upper boundary of 1 is always bucket 0. -// Then negative numbers for smaller boundaries and positive for uppers. -// -// fields are stored like so: -// field ts count zeroCount sum []posbuckets negbuckets -// observation 1 raw raw raw raw []raw []raw -// observation 2 delta delta delta xor []delta []delta -// observation >2 dod dod dod xor []dod []dod -// TODO zerothreshold -type HistoChunk struct { - b bstream -} - -// NewHistoChunk returns a new chunk with Histo encoding of the given size. -func NewHistoChunk() *HistoChunk { - b := make([]byte, 3, 128) - return &HistoChunk{b: bstream{stream: b, count: 0}} -} - -// Encoding returns the encoding type. -func (c *HistoChunk) Encoding() Encoding { - return EncSHS -} - -// Bytes returns the underlying byte slice of the chunk. -func (c *HistoChunk) Bytes() []byte { - return c.b.bytes() -} - -// NumSamples returns the number of samples in the chunk. -func (c *HistoChunk) NumSamples() int { - return int(binary.BigEndian.Uint16(c.Bytes())) -} - -// Meta returns the histogram metadata. -// callers may only call this on chunks that have at least one sample -func (c *HistoChunk) Meta() (int32, float64, []histogram.Span, []histogram.Span, error) { - if c.NumSamples() == 0 { - panic("HistoChunk.Meta() called on an empty chunk") - } - b := newBReader(c.Bytes()[2:]) - return readHistoChunkMeta(&b) -} - -// CounterResetHeader defines the first 2 bits of the chunk header. -type CounterResetHeader byte - -const ( - CounterReset CounterResetHeader = 0b10000000 - NotCounterReset CounterResetHeader = 0b01000000 - GaugeType CounterResetHeader = 0b11000000 - UnknownCounterReset CounterResetHeader = 0b00000000 -) - -// SetCounterResetHeader sets the counter reset header. -func (c *HistoChunk) SetCounterResetHeader(h CounterResetHeader) { - switch h { - case CounterReset, NotCounterReset, GaugeType, UnknownCounterReset: - bytes := c.Bytes() - bytes[2] = (bytes[2] & 0b00111111) | byte(h) - default: - panic("invalid CounterResetHeader type") - } -} - -// GetCounterResetHeader returns the info about the first 2 bits of the chunk header. -func (c *HistoChunk) GetCounterResetHeader() CounterResetHeader { - return CounterResetHeader(c.Bytes()[2] & 0b11000000) -} - -// Compact implements the Chunk interface. -func (c *HistoChunk) Compact() { - if l := len(c.b.stream); cap(c.b.stream) > l+chunkCompactCapacityThreshold { - buf := make([]byte, l) - copy(buf, c.b.stream) - c.b.stream = buf - } -} - -// Appender implements the Chunk interface. -func (c *HistoChunk) Appender() (Appender, error) { - it := c.iterator(nil) - - // To get an appender we must know the state it would have if we had - // appended all existing data from scratch. - // We iterate through the end and populate via the iterator's state. - for it.Next() { - } - if err := it.Err(); err != nil { - return nil, err - } - - a := &HistoAppender{ - b: &c.b, - - schema: it.schema, - zeroThreshold: it.zeroThreshold, - posSpans: it.posSpans, - negSpans: it.negSpans, - t: it.t, - cnt: it.cnt, - zcnt: it.zcnt, - tDelta: it.tDelta, - cntDelta: it.cntDelta, - zcntDelta: it.zcntDelta, - posbuckets: it.posbuckets, - negbuckets: it.negbuckets, - posbucketsDelta: it.posbucketsDelta, - negbucketsDelta: it.negbucketsDelta, - - sum: it.sum, - leading: it.leading, - trailing: it.trailing, - - buf64: make([]byte, binary.MaxVarintLen64), - } - if binary.BigEndian.Uint16(a.b.bytes()) == 0 { - a.leading = 0xff - } - return a, nil -} - -func countSpans(spans []histogram.Span) int { - var cnt int - for _, s := range spans { - cnt += int(s.Length) - } - return cnt -} - -func newHistoIterator(b []byte) *histoIterator { - it := &histoIterator{ - br: newBReader(b), - numTotal: binary.BigEndian.Uint16(b), - t: math.MinInt64, - } - // The first 2 bytes contain chunk headers. - // We skip that for actual samples. - _, _ = it.br.readBits(16) - return it -} - -func (c *HistoChunk) iterator(it Iterator) *histoIterator { - // TODO fix this. this is taken from xor.go // dieter not sure what the purpose of this is - // Should iterators guarantee to act on a copy of the data so it doesn't lock append? - // When using striped locks to guard access to chunks, probably yes. - // Could only copy data if the chunk is not completed yet. - //if histoIter, ok := it.(*histoIterator); ok { - // histoIter.Reset(c.b.bytes()) - // return histoIter - //} - return newHistoIterator(c.b.bytes()) -} - -// Iterator implements the Chunk interface. -func (c *HistoChunk) Iterator(it Iterator) Iterator { - return c.iterator(it) -} - -// HistoAppender is an Appender implementation for sparse histograms. -type HistoAppender struct { - b *bstream - - // Metadata: - schema int32 - zeroThreshold float64 - posSpans, negSpans []histogram.Span - - // For the fields that are tracked as dod's. Note that we expect to - // handle negative deltas (e.g. resets) by creating new chunks, we still - // want to support it in general hence signed integer types. - t int64 - cnt, zcnt uint64 - tDelta, cntDelta, zcntDelta int64 - - posbuckets, negbuckets []int64 - posbucketsDelta, negbucketsDelta []int64 - - // The sum is Gorilla xor encoded. - sum float64 - leading uint8 - trailing uint8 - - buf64 []byte // For working on varint64's. -} - -func putVarint(b *bstream, buf []byte, x int64) { - for _, byt := range buf[:binary.PutVarint(buf, x)] { - b.writeByte(byt) - } -} - -func putUvarint(b *bstream, buf []byte, x uint64) { - for _, byt := range buf[:binary.PutUvarint(buf, x)] { - b.writeByte(byt) - } -} - -// Append implements Appender. This implementation does nothing for now. -// TODO(beorn7): Implement in a meaningful way, i.e. we need to support -// appending of stale markers, but this should never be used for "real" -// samples. -func (a *HistoAppender) Append(int64, float64) {} - -// Appendable returns whether the chunk can be appended to, and if so -// whether any recoding needs to happen using the provided interjections -// (in case of any new buckets, positive or negative range, respectively) -// The chunk is not appendable if: -// * the schema has changed -// * the zerobucket threshold has changed -// * any buckets disappeared -// * there was a counter reset in the count of observations or in any bucket, including the zero bucket -// * the last sample in the chunk was stale while the current sample is not stale -// It returns an additional boolean set to true if it is not appendable because of a counter reset. -// If the given sample is stale, it will always return true. -// If counterReset is true, okToAppend MUST be false. -func (a *HistoAppender) Appendable(h histogram.SparseHistogram) (posInterjections []Interjection, negInterjections []Interjection, okToAppend bool, counterReset bool) { - if value.IsStaleNaN(h.Sum) { - // This is a stale sample whose buckets and spans don't matter. - okToAppend = true - return - } - if value.IsStaleNaN(a.sum) { - // If the last sample was stale, then we can only accept stale samples in this chunk. - return - } - - if h.Count < a.cnt { - // There has been a counter reset. - counterReset = true - return - } - - if h.Schema != a.schema || h.ZeroThreshold != a.zeroThreshold { - return - } - - if h.ZeroCount < a.zcnt { - // There has been a counter reset since ZeroThreshold didn't change. - counterReset = true - return - } - - var ok bool - posInterjections, ok = compareSpans(a.posSpans, h.PositiveSpans) - if !ok { - counterReset = true - return - } - negInterjections, ok = compareSpans(a.negSpans, h.NegativeSpans) - if !ok { - counterReset = true - return - } - - if counterResetInAnyBucket(a.posbuckets, h.PositiveBuckets, a.posSpans, h.PositiveSpans) || - counterResetInAnyBucket(a.negbuckets, h.NegativeBuckets, a.negSpans, h.NegativeSpans) { - counterReset, posInterjections, negInterjections = true, nil, nil - return - } - - okToAppend = true - return -} - -// counterResetInAnyBucket returns true if there was a counter reset for any bucket. -// This should be called only when buckets are same or new buckets were added, -// and does not handle the case of buckets missing. -func counterResetInAnyBucket(oldBuckets, newBuckets []int64, oldSpans, newSpans []histogram.Span) bool { - if len(oldSpans) == 0 || len(oldBuckets) == 0 { - return false - } - - oldSpanSliceIdx, newSpanSliceIdx := 0, 0 // Index for the span slices. - oldInsideSpanIdx, newInsideSpanIdx := uint32(0), uint32(0) // Index inside a span. - oldIdx, newIdx := oldSpans[0].Offset, newSpans[0].Offset - - oldBucketSliceIdx, newBucketSliceIdx := 0, 0 // Index inside bucket slice. - oldVal, newVal := oldBuckets[0], newBuckets[0] - - // Since we assume that new spans won't have missing buckets, there will never be a case - // where the old index will not find a matching new index. - for { - if oldIdx == newIdx { - if newVal < oldVal { - return true - } - } - - if oldIdx <= newIdx { - // Moving ahead old bucket and span by 1 index. - if oldInsideSpanIdx == oldSpans[oldSpanSliceIdx].Length-1 { - // Current span is over. - oldSpanSliceIdx++ - oldInsideSpanIdx = 0 - if oldSpanSliceIdx >= len(oldSpans) { - // All old spans are over. - break - } - oldIdx += 1 + oldSpans[oldSpanSliceIdx].Offset - } else { - oldInsideSpanIdx++ - oldIdx++ - } - oldBucketSliceIdx++ - oldVal += oldBuckets[oldBucketSliceIdx] - } - - if oldIdx > newIdx { - // Moving ahead new bucket and span by 1 index. - if newInsideSpanIdx == newSpans[newSpanSliceIdx].Length-1 { - // Current span is over. - newSpanSliceIdx++ - newInsideSpanIdx = 0 - if newSpanSliceIdx >= len(newSpans) { - // All new spans are over. - // This should not happen, old spans above should catch this first. - panic("new spans over before old spans in counterReset") - } - newIdx += 1 + newSpans[newSpanSliceIdx].Offset - } else { - newInsideSpanIdx++ - newIdx++ - } - newBucketSliceIdx++ - newVal += newBuckets[newBucketSliceIdx] - } - } - - return false -} - -// AppendHistogram appends a SparseHistogram to the chunk. We assume the -// histogram is properly structured. E.g. that the number of pos/neg buckets -// used corresponds to the number conveyed by the pos/neg span structures. -// callers must call Appendable() first and act accordingly! -func (a *HistoAppender) AppendHistogram(t int64, h histogram.SparseHistogram) { - var tDelta, cntDelta, zcntDelta int64 - num := binary.BigEndian.Uint16(a.b.bytes()) - - if value.IsStaleNaN(h.Sum) { - // Emptying out other fields to write no buckets, and an empty meta in case of - // first histogram in the chunk. - h = histogram.SparseHistogram{Sum: h.Sum} - } - - switch num { - case 0: - // the first append gets the privilege to dictate the metadata - // but it's also responsible for encoding it into the chunk! - - writeHistoChunkMeta(a.b, h.Schema, h.ZeroThreshold, h.PositiveSpans, h.NegativeSpans) - a.schema = h.Schema - a.zeroThreshold = h.ZeroThreshold - a.posSpans, a.negSpans = h.PositiveSpans, h.NegativeSpans - numPosBuckets, numNegBuckets := countSpans(h.PositiveSpans), countSpans(h.NegativeSpans) - a.posbuckets = make([]int64, numPosBuckets) - a.negbuckets = make([]int64, numNegBuckets) - a.posbucketsDelta = make([]int64, numPosBuckets) - a.negbucketsDelta = make([]int64, numNegBuckets) - - // now store actual data - putVarint(a.b, a.buf64, t) - putUvarint(a.b, a.buf64, h.Count) - putUvarint(a.b, a.buf64, h.ZeroCount) - a.b.writeBits(math.Float64bits(h.Sum), 64) - for _, buck := range h.PositiveBuckets { - putVarint(a.b, a.buf64, buck) - } - for _, buck := range h.NegativeBuckets { - putVarint(a.b, a.buf64, buck) - } - case 1: - tDelta = t - a.t - cntDelta = int64(h.Count) - int64(a.cnt) - zcntDelta = int64(h.ZeroCount) - int64(a.zcnt) - - if value.IsStaleNaN(h.Sum) { - cntDelta, zcntDelta = 0, 0 - } - - putVarint(a.b, a.buf64, tDelta) - putVarint(a.b, a.buf64, cntDelta) - putVarint(a.b, a.buf64, zcntDelta) - - a.writeSumDelta(h.Sum) - - for i, buck := range h.PositiveBuckets { - delta := buck - a.posbuckets[i] - putVarint(a.b, a.buf64, delta) - a.posbucketsDelta[i] = delta - } - for i, buck := range h.NegativeBuckets { - delta := buck - a.negbuckets[i] - putVarint(a.b, a.buf64, delta) - a.negbucketsDelta[i] = delta - } - - default: - tDelta = t - a.t - cntDelta = int64(h.Count) - int64(a.cnt) - zcntDelta = int64(h.ZeroCount) - int64(a.zcnt) - - tDod := tDelta - a.tDelta - cntDod := cntDelta - a.cntDelta - zcntDod := zcntDelta - a.zcntDelta - - if value.IsStaleNaN(h.Sum) { - cntDod, zcntDod = 0, 0 - } - - putInt64VBBucket(a.b, tDod) - putInt64VBBucket(a.b, cntDod) - putInt64VBBucket(a.b, zcntDod) - - a.writeSumDelta(h.Sum) - - for i, buck := range h.PositiveBuckets { - delta := buck - a.posbuckets[i] - dod := delta - a.posbucketsDelta[i] - putInt64VBBucket(a.b, dod) - a.posbucketsDelta[i] = delta - } - for i, buck := range h.NegativeBuckets { - delta := buck - a.negbuckets[i] - dod := delta - a.negbucketsDelta[i] - putInt64VBBucket(a.b, dod) - a.negbucketsDelta[i] = delta - } - } - - binary.BigEndian.PutUint16(a.b.bytes(), num+1) - - a.t = t - a.cnt = h.Count - a.zcnt = h.ZeroCount - a.tDelta = tDelta - a.cntDelta = cntDelta - a.zcntDelta = zcntDelta - - a.posbuckets, a.negbuckets = h.PositiveBuckets, h.NegativeBuckets - // note that the bucket deltas were already updated above - a.sum = h.Sum -} - -// Recode converts the current chunk to accommodate an expansion of the set of -// (positive and/or negative) buckets used, according to the provided -// interjections, resulting in the honoring of the provided new posSpans and -// negSpans. -func (a *HistoAppender) Recode(posInterjections, negInterjections []Interjection, posSpans, negSpans []histogram.Span) (Chunk, Appender) { - // TODO(beorn7): This currently just decodes everything and then encodes - // it again with the new span layout. This can probably be done in-place - // by editing the chunk. But let's first see how expensive it is in the - // big picture. - byts := a.b.bytes() - it := newHistoIterator(byts) - hc := NewHistoChunk() - app, err := hc.Appender() - if err != nil { - panic(err) - } - numPosBuckets, numNegBuckets := countSpans(posSpans), countSpans(negSpans) - - for it.Next() { - tOld, hOld := it.AtHistogram() - - // We have to newly allocate slices for the modified buckets - // here because they are kept by the appender until the next - // append. - // TODO(beorn7): We might be able to optimize this. - posBuckets := make([]int64, numPosBuckets) - negBuckets := make([]int64, numNegBuckets) - - // Save the modified histogram to the new chunk. - hOld.PositiveSpans, hOld.NegativeSpans = posSpans, negSpans - if len(posInterjections) > 0 { - hOld.PositiveBuckets = interject(hOld.PositiveBuckets, posBuckets, posInterjections) - } - if len(negInterjections) > 0 { - hOld.NegativeBuckets = interject(hOld.NegativeBuckets, negBuckets, negInterjections) - } - app.AppendHistogram(tOld, hOld) - } - - // Set the flags. - hc.SetCounterResetHeader(CounterResetHeader(byts[2] & 0b11000000)) - return hc, app -} - -func (a *HistoAppender) writeSumDelta(v float64) { - vDelta := math.Float64bits(v) ^ math.Float64bits(a.sum) - - if vDelta == 0 { - a.b.writeBit(zero) - return - } - a.b.writeBit(one) - - leading := uint8(bits.LeadingZeros64(vDelta)) - trailing := uint8(bits.TrailingZeros64(vDelta)) - - // Clamp number of leading zeros to avoid overflow when encoding. - if leading >= 32 { - leading = 31 - } - - if a.leading != 0xff && leading >= a.leading && trailing >= a.trailing { - a.b.writeBit(zero) - a.b.writeBits(vDelta>>a.trailing, 64-int(a.leading)-int(a.trailing)) - } else { - a.leading, a.trailing = leading, trailing - - a.b.writeBit(one) - a.b.writeBits(uint64(leading), 5) - - // Note that if leading == trailing == 0, then sigbits == 64. But that value doesn't actually fit into the 6 bits we have. - // Luckily, we never need to encode 0 significant bits, since that would put us in the other case (vdelta == 0). - // So instead we write out a 0 and adjust it back to 64 on unpacking. - sigbits := 64 - leading - trailing - a.b.writeBits(uint64(sigbits), 6) - a.b.writeBits(vDelta>>trailing, int(sigbits)) - } -} - -type histoIterator struct { - br bstreamReader - numTotal uint16 - numRead uint16 - - // Metadata: - schema int32 - zeroThreshold float64 - posSpans, negSpans []histogram.Span - - // For the fields that are tracked as dod's. - t int64 - cnt, zcnt uint64 - tDelta, cntDelta, zcntDelta int64 - - posbuckets, negbuckets []int64 - posbucketsDelta, negbucketsDelta []int64 - - // The sum is Gorilla xor encoded. - sum float64 - leading uint8 - trailing uint8 - - err error -} - -func (it *histoIterator) Seek(t int64) bool { - if it.err != nil { - return false - } - - for t > it.t || it.numRead == 0 { - if !it.Next() { - return false - } - } - return true -} - -func (it *histoIterator) At() (int64, float64) { - panic("cannot call histoIterator.At().") -} - -func (it *histoIterator) ChunkEncoding() Encoding { - return EncSHS -} - -func (it *histoIterator) AtHistogram() (int64, histogram.SparseHistogram) { - if value.IsStaleNaN(it.sum) { - return it.t, histogram.SparseHistogram{Sum: it.sum} - } - return it.t, histogram.SparseHistogram{ - Count: it.cnt, - ZeroCount: it.zcnt, - Sum: it.sum, - ZeroThreshold: it.zeroThreshold, - Schema: it.schema, - PositiveSpans: it.posSpans, - NegativeSpans: it.negSpans, - PositiveBuckets: it.posbuckets, - NegativeBuckets: it.negbuckets, - } -} - -func (it *histoIterator) Err() error { - return it.err -} - -func (it *histoIterator) Reset(b []byte) { - // The first 2 bytes contain chunk headers. - // We skip that for actual samples. - it.br = newBReader(b[2:]) - it.numTotal = binary.BigEndian.Uint16(b) - it.numRead = 0 - - it.t, it.cnt, it.zcnt = 0, 0, 0 - it.tDelta, it.cntDelta, it.zcntDelta = 0, 0, 0 - - for i := range it.posbuckets { - it.posbuckets[i] = 0 - it.posbucketsDelta[i] = 0 - } - for i := range it.negbuckets { - it.negbuckets[i] = 0 - it.negbucketsDelta[i] = 0 - } - - it.sum = 0 - it.leading = 0 - it.trailing = 0 - it.err = nil -} - -func (it *histoIterator) Next() bool { - if it.err != nil || it.numRead == it.numTotal { - return false - } - - if it.numRead == 0 { - - // first read is responsible for reading chunk metadata and initializing fields that depend on it - // We give counter reset info at chunk level, hence we discard it here. - schema, zeroThreshold, posSpans, negSpans, err := readHistoChunkMeta(&it.br) - if err != nil { - it.err = err - return false - } - it.schema = schema - it.zeroThreshold = zeroThreshold - it.posSpans, it.negSpans = posSpans, negSpans - numPosBuckets, numNegBuckets := countSpans(posSpans), countSpans(negSpans) - if numPosBuckets > 0 { - it.posbuckets = make([]int64, numPosBuckets) - it.posbucketsDelta = make([]int64, numPosBuckets) - } - if numNegBuckets > 0 { - it.negbuckets = make([]int64, numNegBuckets) - it.negbucketsDelta = make([]int64, numNegBuckets) - } - - // now read actual data - - t, err := binary.ReadVarint(&it.br) - if err != nil { - it.err = err - return false - } - it.t = t - - cnt, err := binary.ReadUvarint(&it.br) - if err != nil { - it.err = err - return false - } - it.cnt = cnt - - zcnt, err := binary.ReadUvarint(&it.br) - if err != nil { - it.err = err - return false - } - it.zcnt = zcnt - - sum, err := it.br.readBits(64) - if err != nil { - it.err = err - return false - } - it.sum = math.Float64frombits(sum) - - for i := range it.posbuckets { - v, err := binary.ReadVarint(&it.br) - if err != nil { - it.err = err - return false - } - it.posbuckets[i] = v - } - for i := range it.negbuckets { - v, err := binary.ReadVarint(&it.br) - if err != nil { - it.err = err - return false - } - it.negbuckets[i] = v - } - - it.numRead++ - return true - } - - if it.numRead == 1 { - tDelta, err := binary.ReadVarint(&it.br) - if err != nil { - it.err = err - return false - } - it.tDelta = tDelta - it.t += int64(it.tDelta) - - cntDelta, err := binary.ReadVarint(&it.br) - if err != nil { - it.err = err - return false - } - it.cntDelta = cntDelta - it.cnt = uint64(int64(it.cnt) + it.cntDelta) - - zcntDelta, err := binary.ReadVarint(&it.br) - if err != nil { - it.err = err - return false - } - it.zcntDelta = zcntDelta - it.zcnt = uint64(int64(it.zcnt) + it.zcntDelta) - - ok := it.readSum() - if !ok { - return false - } - - if value.IsStaleNaN(it.sum) { - it.numRead++ - return true - } - - for i := range it.posbuckets { - delta, err := binary.ReadVarint(&it.br) - if err != nil { - it.err = err - return false - } - it.posbucketsDelta[i] = delta - it.posbuckets[i] = it.posbuckets[i] + delta - } - - for i := range it.negbuckets { - delta, err := binary.ReadVarint(&it.br) - if err != nil { - it.err = err - return false - } - it.negbucketsDelta[i] = delta - it.negbuckets[i] = it.negbuckets[i] + delta - } - - it.numRead++ - return true - } - - tDod, err := readInt64VBBucket(&it.br) - if err != nil { - it.err = err - return false - } - it.tDelta = it.tDelta + tDod - it.t += it.tDelta - - cntDod, err := readInt64VBBucket(&it.br) - if err != nil { - it.err = err - return false - } - it.cntDelta = it.cntDelta + cntDod - it.cnt = uint64(int64(it.cnt) + it.cntDelta) - - zcntDod, err := readInt64VBBucket(&it.br) - if err != nil { - it.err = err - return false - } - it.zcntDelta = it.zcntDelta + zcntDod - it.zcnt = uint64(int64(it.zcnt) + it.zcntDelta) - - ok := it.readSum() - if !ok { - return false - } - - if value.IsStaleNaN(it.sum) { - it.numRead++ - return true - } - - for i := range it.posbuckets { - dod, err := readInt64VBBucket(&it.br) - if err != nil { - it.err = err - return false - } - it.posbucketsDelta[i] = it.posbucketsDelta[i] + dod - it.posbuckets[i] = it.posbuckets[i] + it.posbucketsDelta[i] - } - - for i := range it.negbuckets { - dod, err := readInt64VBBucket(&it.br) - if err != nil { - it.err = err - return false - } - it.negbucketsDelta[i] = it.negbucketsDelta[i] + dod - it.negbuckets[i] = it.negbuckets[i] + it.negbucketsDelta[i] - } - - it.numRead++ - return true -} - -func (it *histoIterator) readSum() bool { - bit, err := it.br.readBitFast() - if err != nil { - bit, err = it.br.readBit() - } - if err != nil { - it.err = err - return false - } - - if bit == zero { - // it.sum = it.sum - } else { - bit, err := it.br.readBitFast() - if err != nil { - bit, err = it.br.readBit() - } - if err != nil { - it.err = err - return false - } - if bit == zero { - // reuse leading/trailing zero bits - // it.leading, it.trailing = it.leading, it.trailing - } else { - bits, err := it.br.readBitsFast(5) - if err != nil { - bits, err = it.br.readBits(5) - } - if err != nil { - it.err = err - return false - } - it.leading = uint8(bits) - - bits, err = it.br.readBitsFast(6) - if err != nil { - bits, err = it.br.readBits(6) - } - if err != nil { - it.err = err - return false - } - mbits := uint8(bits) - // 0 significant bits here means we overflowed and we actually need 64; see comment in encoder - if mbits == 0 { - mbits = 64 - } - it.trailing = 64 - it.leading - mbits - } - - mbits := 64 - it.leading - it.trailing - bits, err := it.br.readBitsFast(mbits) - if err != nil { - bits, err = it.br.readBits(mbits) - } - if err != nil { - it.err = err - return false - } - vbits := math.Float64bits(it.sum) - vbits ^= bits << it.trailing - it.sum = math.Float64frombits(vbits) - } - - return true -} diff --git a/tsdb/chunkenc/histo_meta.go b/tsdb/chunkenc/histo_meta.go deleted file mode 100644 index b685035fb8..0000000000 --- a/tsdb/chunkenc/histo_meta.go +++ /dev/null @@ -1,261 +0,0 @@ -// Copyright 2021 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package chunkenc - -import ( - "github.com/prometheus/prometheus/pkg/histogram" -) - -func writeHistoChunkMeta(b *bstream, schema int32, zeroThreshold float64, posSpans, negSpans []histogram.Span) { - putInt64VBBucket(b, int64(schema)) - putFloat64VBBucket(b, zeroThreshold) - putHistoChunkMetaSpans(b, posSpans) - putHistoChunkMetaSpans(b, negSpans) -} - -func putHistoChunkMetaSpans(b *bstream, spans []histogram.Span) { - putInt64VBBucket(b, int64(len(spans))) - for _, s := range spans { - putInt64VBBucket(b, int64(s.Length)) - putInt64VBBucket(b, int64(s.Offset)) - } -} - -func readHistoChunkMeta(b *bstreamReader) (schema int32, zeroThreshold float64, posSpans []histogram.Span, negSpans []histogram.Span, err error) { - _, err = b.ReadByte() // The header. - if err != nil { - return - } - - v, err := readInt64VBBucket(b) - if err != nil { - return - } - schema = int32(v) - - zeroThreshold, err = readFloat64VBBucket(b) - if err != nil { - return - } - - posSpans, err = readHistoChunkMetaSpans(b) - if err != nil { - return - } - - negSpans, err = readHistoChunkMetaSpans(b) - if err != nil { - return - } - - return -} - -func readHistoChunkMetaSpans(b *bstreamReader) ([]histogram.Span, error) { - var spans []histogram.Span - num, err := readInt64VBBucket(b) - if err != nil { - return nil, err - } - for i := 0; i < int(num); i++ { - - length, err := readInt64VBBucket(b) - if err != nil { - return nil, err - } - - offset, err := readInt64VBBucket(b) - if err != nil { - return nil, err - } - - spans = append(spans, histogram.Span{ - Length: uint32(length), - Offset: int32(offset), - }) - } - return spans, nil -} - -type bucketIterator struct { - spans []histogram.Span - span int // span position of last yielded bucket - bucket int // bucket position within span of last yielded bucket - idx int // bucket index (globally across all spans) of last yielded bucket -} - -func newBucketIterator(spans []histogram.Span) *bucketIterator { - b := bucketIterator{ - spans: spans, - span: 0, - bucket: -1, - idx: -1, - } - if len(spans) > 0 { - b.idx += int(spans[0].Offset) - } - return &b -} - -func (b *bucketIterator) Next() (int, bool) { - // we're already out of bounds - if b.span >= len(b.spans) { - return 0, false - } -try: - if b.bucket < int(b.spans[b.span].Length-1) { // try to move within same span. - b.bucket++ - b.idx++ - return b.idx, true - } else if b.span < len(b.spans)-1 { // try to move from one span to the next - b.span++ - b.idx += int(b.spans[b.span].Offset + 1) - b.bucket = 0 - if b.spans[b.span].Length == 0 { - // pathological case that should never happen. We can't use this span, let's try again. - goto try - } - return b.idx, true - } - // we're out of options - return 0, false -} - -// Interjection describes that num new buckets are introduced before processing the pos'th delta from the original slice -type Interjection struct { - pos int - num int -} - -// compareSpans returns the interjections to convert a slice of deltas to a new slice representing an expanded set of buckets, or false if incompatible (e.g. if buckets were removed) -// For example: -// Let's say the old buckets look like this: -// span syntax: [offset, length] -// spans : [ 0 , 2 ] [2,1] [ 3 , 2 ] [3,1] [1,1] -// bucket idx : [0] [1] 2 3 [4] 5 6 7 [8] [9] 10 11 12 [13] 14 [15] -// raw values 6 3 3 2 4 5 1 -// deltas 6 -3 0 -1 2 1 -4 - -// But now we introduce a new bucket layout. (carefully chosen example where we have a span appended, one unchanged[*], one prepended, and two merge - in that order) -// [*] unchanged in terms of which bucket indices they represent. but to achieve that, their offset needs to change if "disrupted" by spans changing ahead of them -// \/ this one is "unchanged" -// spans : [ 0 , 3 ] [1,1] [ 1 , 4 ] [ 3 , 3 ] -// bucket idx : [0] [1] [2] 3 [4] 5 [6] [7] [8] [9] 10 11 12 [13] [14] [15] -// raw values 6 3 0 3 0 0 2 4 5 0 1 -// deltas 6 -3 -3 3 -3 0 2 2 1 -5 1 -// delta mods: / \ / \ / \ -// note that whenever any new buckets are introduced, the subsequent "old" bucket needs to readjust its delta to the new base of 0 -// thus, for the caller, who wants to transform the set of original deltas to a new set of deltas to match a new span layout that adds buckets, we simply -// need to generate a list of interjections -// note: within compareSpans we don't have to worry about the changes to the spans themselves, -// thanks to the iterators, we get to work with the more useful bucket indices (which of course directly correspond to the buckets we have to adjust) -func compareSpans(a, b []histogram.Span) ([]Interjection, bool) { - ai := newBucketIterator(a) - bi := newBucketIterator(b) - - var interjections []Interjection - - // when inter.num becomes > 0, this becomes a valid interjection that should be yielded when we finish a streak of new buckets - var inter Interjection - - av, aok := ai.Next() - bv, bok := bi.Next() -loop: - for { - switch { - case aok && bok: - switch { - case av == bv: // Both have an identical value. move on! - // Finish WIP interjection and reset. - if inter.num > 0 { - interjections = append(interjections, inter) - } - inter.num = 0 - av, aok = ai.Next() - bv, bok = bi.Next() - inter.pos++ - case av < bv: // b misses a value that is in a. - return interjections, false - case av > bv: // a misses a value that is in b. Forward b and recompare. - inter.num++ - bv, bok = bi.Next() - } - case aok && !bok: // b misses a value that is in a. - return interjections, false - case !aok && bok: // a misses a value that is in b. Forward b and recompare. - inter.num++ - bv, bok = bi.Next() - default: // Both iterators ran out. We're done. - if inter.num > 0 { - interjections = append(interjections, inter) - } - break loop - } - } - - return interjections, true -} - -// interject merges 'in' with the provided interjections and writes them into -// 'out', which must already have the appropriate length. -func interject(in, out []int64, interjections []Interjection) []int64 { - var j int // Position in out. - var v int64 // The last value seen. - var interj int // The next interjection to process. - for i, d := range in { - if interj < len(interjections) && i == interjections[interj].pos { - - // We have an interjection! - // Add interjection.num new delta values such that their - // bucket values equate 0. - out[j] = int64(-v) - j++ - for x := 1; x < interjections[interj].num; x++ { - out[j] = 0 - j++ - } - interj++ - - // Now save the value from the input. The delta value we - // should save is the original delta value + the last - // value of the point before the interjection (to undo - // the delta that was introduced by the interjection). - out[j] = d + v - j++ - v = d + v - continue - } - - // If there was no interjection, the original delta is still - // valid. - out[j] = d - j++ - v += d - } - switch interj { - case len(interjections): - // All interjections processed. Nothing more to do. - case len(interjections) - 1: - // One more interjection to process at the end. - out[j] = int64(-v) - j++ - for x := 1; x < interjections[interj].num; x++ { - out[j] = 0 - j++ - } - default: - panic("unprocessed interjections left") - } - return out -} diff --git a/tsdb/chunkenc/histogram.go b/tsdb/chunkenc/histogram.go new file mode 100644 index 0000000000..ad0dbe9f37 --- /dev/null +++ b/tsdb/chunkenc/histogram.go @@ -0,0 +1,934 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package chunkenc + +import ( + "encoding/binary" + "math" + "math/bits" + + "github.com/prometheus/prometheus/model/histogram" + "github.com/prometheus/prometheus/pkg/value" +) + +const () + +// HistogramChunk holds encoded sample data for a sparse, high-resolution +// histogram. +// +// TODO(beorn7): Document the layout of chunk metadata. +// +// Each sample has multiple "fields", stored in the following way (raw = store +// number directly, delta = store delta to the previous number, dod = store +// delta of the delta to the previous number, xor = what we do for regular +// sample values): +// +// field → ts count zeroCount sum []posbuckets []negbuckets +// sample 1 raw raw raw raw []raw []raw +// sample 2 delta delta delta xor []delta []delta +// sample >2 dod dod dod xor []dod []dod +type HistogramChunk struct { + b bstream +} + +// NewHistogramChunk returns a new chunk with histogram encoding of the given +// size. +func NewHistogramChunk() *HistogramChunk { + b := make([]byte, 3, 128) + return &HistogramChunk{b: bstream{stream: b, count: 0}} +} + +// Encoding returns the encoding type. +func (c *HistogramChunk) Encoding() Encoding { + return EncHistogram +} + +// Bytes returns the underlying byte slice of the chunk. +func (c *HistogramChunk) Bytes() []byte { + return c.b.bytes() +} + +// NumSamples returns the number of samples in the chunk. +func (c *HistogramChunk) NumSamples() int { + return int(binary.BigEndian.Uint16(c.Bytes())) +} + +// Meta returns the histogram metadata. Only call this on chunks that have at +// least one sample. +func (c *HistogramChunk) Meta() ( + schema int32, zeroThreshold float64, + negativeSpans, positiveSpans []histogram.Span, + err error, +) { + if c.NumSamples() == 0 { + panic("HistoChunk.Meta() called on an empty chunk") + } + b := newBReader(c.Bytes()[2:]) + return readHistogramChunkMeta(&b) +} + +// CounterResetHeader defines the first 2 bits of the chunk header. +type CounterResetHeader byte + +const ( + CounterReset CounterResetHeader = 0b10000000 + NotCounterReset CounterResetHeader = 0b01000000 + GaugeType CounterResetHeader = 0b11000000 + UnknownCounterReset CounterResetHeader = 0b00000000 +) + +// SetCounterResetHeader sets the counter reset header. +func (c *HistogramChunk) SetCounterResetHeader(h CounterResetHeader) { + switch h { + case CounterReset, NotCounterReset, GaugeType, UnknownCounterReset: + bytes := c.Bytes() + bytes[2] = (bytes[2] & 0b00111111) | byte(h) + default: + panic("invalid CounterResetHeader type") + } +} + +// GetCounterResetHeader returns the info about the first 2 bits of the chunk +// header. +func (c *HistogramChunk) GetCounterResetHeader() CounterResetHeader { + return CounterResetHeader(c.Bytes()[2] & 0b11000000) +} + +// Compact implements the Chunk interface. +func (c *HistogramChunk) Compact() { + if l := len(c.b.stream); cap(c.b.stream) > l+chunkCompactCapacityThreshold { + buf := make([]byte, l) + copy(buf, c.b.stream) + c.b.stream = buf + } +} + +// Appender implements the Chunk interface. +func (c *HistogramChunk) Appender() (Appender, error) { + it := c.iterator(nil) + + // To get an appender, we must know the state it would have if we had + // appended all existing data from scratch. We iterate through the end + // and populate via the iterator's state. + for it.Next() { + } + if err := it.Err(); err != nil { + return nil, err + } + + a := &HistogramAppender{ + b: &c.b, + + schema: it.schema, + zThreshold: it.zThreshold, + pSpans: it.pSpans, + nSpans: it.nSpans, + t: it.t, + cnt: it.cnt, + zCnt: it.zCnt, + tDelta: it.tDelta, + cntDelta: it.cntDelta, + zCntDelta: it.zCntDelta, + pBuckets: it.pBuckets, + nBuckets: it.nBuckets, + pBucketsDelta: it.pBucketsDelta, + nBucketsDelta: it.nBucketsDelta, + + sum: it.sum, + leading: it.leading, + trailing: it.trailing, + + buf64: make([]byte, binary.MaxVarintLen64), + } + if binary.BigEndian.Uint16(a.b.bytes()) == 0 { + a.leading = 0xff + } + return a, nil +} + +func countSpans(spans []histogram.Span) int { + var cnt int + for _, s := range spans { + cnt += int(s.Length) + } + return cnt +} + +func newHistogramIterator(b []byte) *histogramIterator { + it := &histogramIterator{ + br: newBReader(b), + numTotal: binary.BigEndian.Uint16(b), + t: math.MinInt64, + } + // The first 2 bytes contain chunk headers. + // We skip that for actual samples. + _, _ = it.br.readBits(16) + return it +} + +func (c *HistogramChunk) iterator(it Iterator) *histogramIterator { + // This commet is copied from XORChunk.iterator: + // Should iterators guarantee to act on a copy of the data so it doesn't lock append? + // When using striped locks to guard access to chunks, probably yes. + // Could only copy data if the chunk is not completed yet. + if histogramIter, ok := it.(*histogramIterator); ok { + histogramIter.Reset(c.b.bytes()) + return histogramIter + } + return newHistogramIterator(c.b.bytes()) +} + +// Iterator implements the Chunk interface. +func (c *HistogramChunk) Iterator(it Iterator) Iterator { + return c.iterator(it) +} + +// HistogramAppender is an Appender implementation for sparse histograms. +type HistogramAppender struct { + b *bstream + + // Metadata: + schema int32 + zThreshold float64 + pSpans, nSpans []histogram.Span + + // Although we intend to start new chunks on counter resets, we still + // have to handle negative deltas for gauge histograms. Therefore, even + // deltas are signed types here (even for tDelta to not treat that one + // specially). + t int64 + cnt, zCnt uint64 + tDelta, cntDelta, zCntDelta int64 + pBuckets, nBuckets []int64 + pBucketsDelta, nBucketsDelta []int64 + + // The sum is Gorilla xor encoded. + sum float64 + leading uint8 + trailing uint8 + + buf64 []byte // For working on varint64's. +} + +func putVarint(b *bstream, buf []byte, x int64) { + for _, byt := range buf[:binary.PutVarint(buf, x)] { + b.writeByte(byt) + } +} + +func putUvarint(b *bstream, buf []byte, x uint64) { + for _, byt := range buf[:binary.PutUvarint(buf, x)] { + b.writeByte(byt) + } +} + +// Append implements Appender. This implementation panics because normal float +// samples must never be appended to a histogram chunk. +func (a *HistogramAppender) Append(int64, float64) { + panic("appended a float sample to a histogram chunk") +} + +// Appendable returns whether the chunk can be appended to, and if so +// whether any recoding needs to happen using the provided interjections +// (in case of any new buckets, positive or negative range, respectively). +// +// The chunk is not appendable in the following cases: +// +// • The schema has changed. +// +// • The threshold for the zero bucket has changed. +// +// • Any buckets have disappeared. +// +// • There was a counter reset in the count of observations or in any bucket, +// including the zero bucket. +// +// • The last sample in the chunk was stale while the current sample is not stale. +// +// The method returns an additional boolean set to true if it is not appendable +// because of a counter reset. If the given sample is stale, it is always ok to +// append. If counterReset is true, okToAppend is always false. +func (a *HistogramAppender) Appendable(h histogram.Histogram) ( + positiveInterjections, negativeInterjections []Interjection, + okToAppend bool, counterReset bool, +) { + if value.IsStaleNaN(h.Sum) { + // This is a stale sample whose buckets and spans don't matter. + okToAppend = true + return + } + if value.IsStaleNaN(a.sum) { + // If the last sample was stale, then we can only accept stale + // samples in this chunk. + return + } + + if h.Count < a.cnt { + // There has been a counter reset. + counterReset = true + return + } + + if h.Schema != a.schema || h.ZeroThreshold != a.zThreshold { + return + } + + if h.ZeroCount < a.zCnt { + // There has been a counter reset since ZeroThreshold didn't change. + counterReset = true + return + } + + var ok bool + positiveInterjections, ok = compareSpans(a.pSpans, h.PositiveSpans) + if !ok { + counterReset = true + return + } + negativeInterjections, ok = compareSpans(a.nSpans, h.NegativeSpans) + if !ok { + counterReset = true + return + } + + if counterResetInAnyBucket(a.pBuckets, h.PositiveBuckets, a.pSpans, h.PositiveSpans) || + counterResetInAnyBucket(a.nBuckets, h.NegativeBuckets, a.nSpans, h.NegativeSpans) { + counterReset, positiveInterjections, negativeInterjections = true, nil, nil + return + } + + okToAppend = true + return +} + +// counterResetInAnyBucket returns true if there was a counter reset for any +// bucket. This should be called only when the bucket layout is the same or new +// buckets were added. It does not handle the case of buckets missing. +func counterResetInAnyBucket(oldBuckets, newBuckets []int64, oldSpans, newSpans []histogram.Span) bool { + if len(oldSpans) == 0 || len(oldBuckets) == 0 { + return false + } + + oldSpanSliceIdx, newSpanSliceIdx := 0, 0 // Index for the span slices. + oldInsideSpanIdx, newInsideSpanIdx := uint32(0), uint32(0) // Index inside a span. + oldIdx, newIdx := oldSpans[0].Offset, newSpans[0].Offset + + oldBucketSliceIdx, newBucketSliceIdx := 0, 0 // Index inside bucket slice. + oldVal, newVal := oldBuckets[0], newBuckets[0] + + // Since we assume that new spans won't have missing buckets, there will never be a case + // where the old index will not find a matching new index. + for { + if oldIdx == newIdx { + if newVal < oldVal { + return true + } + } + + if oldIdx <= newIdx { + // Moving ahead old bucket and span by 1 index. + if oldInsideSpanIdx == oldSpans[oldSpanSliceIdx].Length-1 { + // Current span is over. + oldSpanSliceIdx++ + oldInsideSpanIdx = 0 + if oldSpanSliceIdx >= len(oldSpans) { + // All old spans are over. + break + } + oldIdx += 1 + oldSpans[oldSpanSliceIdx].Offset + } else { + oldInsideSpanIdx++ + oldIdx++ + } + oldBucketSliceIdx++ + oldVal += oldBuckets[oldBucketSliceIdx] + } + + if oldIdx > newIdx { + // Moving ahead new bucket and span by 1 index. + if newInsideSpanIdx == newSpans[newSpanSliceIdx].Length-1 { + // Current span is over. + newSpanSliceIdx++ + newInsideSpanIdx = 0 + if newSpanSliceIdx >= len(newSpans) { + // All new spans are over. + // This should not happen, old spans above should catch this first. + panic("new spans over before old spans in counterReset") + } + newIdx += 1 + newSpans[newSpanSliceIdx].Offset + } else { + newInsideSpanIdx++ + newIdx++ + } + newBucketSliceIdx++ + newVal += newBuckets[newBucketSliceIdx] + } + } + + return false +} + +// AppendHistogram appends a histogram to the chunk. The caller must ensure that +// the histogram is properly structured, e.g. the number of buckets used +// corresponds to the number conveyed by the span structures. First call +// Appendable() and act accordingly! +func (a *HistogramAppender) AppendHistogram(t int64, h histogram.Histogram) { + var tDelta, cntDelta, zCntDelta int64 + num := binary.BigEndian.Uint16(a.b.bytes()) + + if value.IsStaleNaN(h.Sum) { + // Emptying out other fields to write no buckets, and an empty + // meta in case of first histogram in the chunk. + h = histogram.Histogram{Sum: h.Sum} + } + + switch num { + case 0: + // The first append gets the privilege to dictate the metadata + // but it's also responsible for encoding it into the chunk! + writeHistogramChunkMeta(a.b, h.Schema, h.ZeroThreshold, h.PositiveSpans, h.NegativeSpans) + a.schema = h.Schema + a.zThreshold = h.ZeroThreshold + a.pSpans, a.nSpans = h.PositiveSpans, h.NegativeSpans + numPBuckets, numNBuckets := countSpans(h.PositiveSpans), countSpans(h.NegativeSpans) + a.pBuckets = make([]int64, numPBuckets) + a.nBuckets = make([]int64, numNBuckets) + a.pBucketsDelta = make([]int64, numPBuckets) + a.nBucketsDelta = make([]int64, numNBuckets) + + // Now store the actual data. + putVarint(a.b, a.buf64, t) + putUvarint(a.b, a.buf64, h.Count) // TODO(beorn7): Use putVarbitInt? + putUvarint(a.b, a.buf64, h.ZeroCount) // TODO(beorn7): Use putVarbitInt? + a.b.writeBits(math.Float64bits(h.Sum), 64) + for _, buck := range h.PositiveBuckets { + putVarint(a.b, a.buf64, buck) // TODO(beorn7): Use putVarbitInt? + } + for _, buck := range h.NegativeBuckets { + putVarint(a.b, a.buf64, buck) // TODO(beorn7): Use putVarbitInt? + } + case 1: + tDelta = t - a.t + cntDelta = int64(h.Count) - int64(a.cnt) + zCntDelta = int64(h.ZeroCount) - int64(a.zCnt) + + if value.IsStaleNaN(h.Sum) { + cntDelta, zCntDelta = 0, 0 + } + + putVarint(a.b, a.buf64, tDelta) // TODO(beorn7): This should probably be putUvarint. + putVarint(a.b, a.buf64, cntDelta) // TODO(beorn7): Use putVarbitInt? + putVarint(a.b, a.buf64, zCntDelta) // TODO(beorn7): Use putVarbitInt? + + a.writeSumDelta(h.Sum) + + for i, buck := range h.PositiveBuckets { + delta := buck - a.pBuckets[i] + putVarint(a.b, a.buf64, delta) // TODO(beorn7): Use putVarbitInt? + a.pBucketsDelta[i] = delta + } + for i, buck := range h.NegativeBuckets { + delta := buck - a.nBuckets[i] + putVarint(a.b, a.buf64, delta) // TODO(beorn7): Use putVarbitInt? + a.nBucketsDelta[i] = delta + } + + default: + tDelta = t - a.t + cntDelta = int64(h.Count) - int64(a.cnt) + zCntDelta = int64(h.ZeroCount) - int64(a.zCnt) + + tDod := tDelta - a.tDelta + cntDod := cntDelta - a.cntDelta + zCntDod := zCntDelta - a.zCntDelta + + if value.IsStaleNaN(h.Sum) { + cntDod, zCntDod = 0, 0 + } + + putVarbitInt(a.b, tDod) + putVarbitInt(a.b, cntDod) + putVarbitInt(a.b, zCntDod) + + a.writeSumDelta(h.Sum) + + for i, buck := range h.PositiveBuckets { + delta := buck - a.pBuckets[i] + dod := delta - a.pBucketsDelta[i] + putVarbitInt(a.b, dod) + a.pBucketsDelta[i] = delta + } + for i, buck := range h.NegativeBuckets { + delta := buck - a.nBuckets[i] + dod := delta - a.nBucketsDelta[i] + putVarbitInt(a.b, dod) + a.nBucketsDelta[i] = delta + } + } + + binary.BigEndian.PutUint16(a.b.bytes(), num+1) + + a.t = t + a.cnt = h.Count + a.zCnt = h.ZeroCount + a.tDelta = tDelta + a.cntDelta = cntDelta + a.zCntDelta = zCntDelta + + a.pBuckets, a.nBuckets = h.PositiveBuckets, h.NegativeBuckets + // Note that the bucket deltas were already updated above. + a.sum = h.Sum +} + +// Recode converts the current chunk to accommodate an expansion of the set of +// (positive and/or negative) buckets used, according to the provided +// interjections, resulting in the honoring of the provided new positive and +// negative spans. +func (a *HistogramAppender) Recode( + positiveInterjections, negativeInterjections []Interjection, + positiveSpans, negativeSpans []histogram.Span, +) (Chunk, Appender) { + // TODO(beorn7): This currently just decodes everything and then encodes + // it again with the new span layout. This can probably be done in-place + // by editing the chunk. But let's first see how expensive it is in the + // big picture. + byts := a.b.bytes() + it := newHistogramIterator(byts) + hc := NewHistogramChunk() + app, err := hc.Appender() + if err != nil { + panic(err) + } + numPositiveBuckets, numNegativeBuckets := countSpans(positiveSpans), countSpans(negativeSpans) + + for it.Next() { + tOld, hOld := it.AtHistogram() + + // We have to newly allocate slices for the modified buckets + // here because they are kept by the appender until the next + // append. + // TODO(beorn7): We might be able to optimize this. + positiveBuckets := make([]int64, numPositiveBuckets) + negativeBuckets := make([]int64, numNegativeBuckets) + + // Save the modified histogram to the new chunk. + hOld.PositiveSpans, hOld.NegativeSpans = positiveSpans, negativeSpans + if len(positiveInterjections) > 0 { + hOld.PositiveBuckets = interject(hOld.PositiveBuckets, positiveBuckets, positiveInterjections) + } + if len(negativeInterjections) > 0 { + hOld.NegativeBuckets = interject(hOld.NegativeBuckets, negativeBuckets, negativeInterjections) + } + app.AppendHistogram(tOld, hOld) + } + + hc.SetCounterResetHeader(CounterResetHeader(byts[2] & 0b11000000)) + return hc, app +} + +func (a *HistogramAppender) writeSumDelta(v float64) { + vDelta := math.Float64bits(v) ^ math.Float64bits(a.sum) + + if vDelta == 0 { + a.b.writeBit(zero) + return + } + a.b.writeBit(one) + + leading := uint8(bits.LeadingZeros64(vDelta)) + trailing := uint8(bits.TrailingZeros64(vDelta)) + + // Clamp number of leading zeros to avoid overflow when encoding. + if leading >= 32 { + leading = 31 + } + + if a.leading != 0xff && leading >= a.leading && trailing >= a.trailing { + a.b.writeBit(zero) + a.b.writeBits(vDelta>>a.trailing, 64-int(a.leading)-int(a.trailing)) + } else { + a.leading, a.trailing = leading, trailing + + a.b.writeBit(one) + a.b.writeBits(uint64(leading), 5) + + // Note that if leading == trailing == 0, then sigbits == 64. + // But that value doesn't actually fit into the 6 bits we have. + // Luckily, we never need to encode 0 significant bits, since + // that would put us in the other case (vdelta == 0). So + // instead we write out a 0 and adjust it back to 64 on + // unpacking. + sigbits := 64 - leading - trailing + a.b.writeBits(uint64(sigbits), 6) + a.b.writeBits(vDelta>>trailing, int(sigbits)) + } +} + +type histogramIterator struct { + br bstreamReader + numTotal uint16 + numRead uint16 + + // Metadata: + schema int32 + zThreshold float64 + pSpans, nSpans []histogram.Span + + // For the fields that are tracked as deltas and ultimately dod's. + t int64 + cnt, zCnt uint64 + tDelta, cntDelta, zCntDelta int64 + pBuckets, nBuckets []int64 + pBucketsDelta, nBucketsDelta []int64 + + // The sum is Gorilla xor encoded. + sum float64 + leading uint8 + trailing uint8 + + err error +} + +func (it *histogramIterator) Seek(t int64) bool { + if it.err != nil { + return false + } + + for t > it.t || it.numRead == 0 { + if !it.Next() { + return false + } + } + return true +} + +func (it *histogramIterator) At() (int64, float64) { + panic("cannot call histogramIterator.At") +} + +func (it *histogramIterator) ChunkEncoding() Encoding { + return EncHistogram +} + +func (it *histogramIterator) AtHistogram() (int64, histogram.Histogram) { + if value.IsStaleNaN(it.sum) { + return it.t, histogram.Histogram{Sum: it.sum} + } + return it.t, histogram.Histogram{ + Count: it.cnt, + ZeroCount: it.zCnt, + Sum: it.sum, + ZeroThreshold: it.zThreshold, + Schema: it.schema, + PositiveSpans: it.pSpans, + NegativeSpans: it.nSpans, + PositiveBuckets: it.pBuckets, + NegativeBuckets: it.nBuckets, + } +} + +func (it *histogramIterator) Err() error { + return it.err +} + +func (it *histogramIterator) Reset(b []byte) { + // The first 2 bytes contain chunk headers. + // We skip that for actual samples. + it.br = newBReader(b[2:]) + it.numTotal = binary.BigEndian.Uint16(b) + it.numRead = 0 + + it.t, it.cnt, it.zCnt = 0, 0, 0 + it.tDelta, it.cntDelta, it.zCntDelta = 0, 0, 0 + + // TODO(beorn7): Those will be recreated anyway. + // Either delete them here entirely or recycle them + // below if big enough. + for i := range it.pBuckets { + it.pBuckets[i] = 0 + it.pBucketsDelta[i] = 0 + } + for i := range it.nBuckets { + it.nBuckets[i] = 0 + it.nBucketsDelta[i] = 0 + } + + it.sum = 0 + it.leading = 0 + it.trailing = 0 + it.err = nil +} + +func (it *histogramIterator) Next() bool { + if it.err != nil || it.numRead == it.numTotal { + return false + } + + if it.numRead == 0 { + + // The first read is responsible for reading the chunk metadata + // and for initializing fields that depend on it. We give + // counter reset info at chunk level, hence we discard it here. + schema, zeroThreshold, posSpans, negSpans, err := readHistogramChunkMeta(&it.br) + if err != nil { + it.err = err + return false + } + it.schema = schema + it.zThreshold = zeroThreshold + it.pSpans, it.nSpans = posSpans, negSpans + numPosBuckets, numNegBuckets := countSpans(posSpans), countSpans(negSpans) + if numPosBuckets > 0 { + it.pBuckets = make([]int64, numPosBuckets) + it.pBucketsDelta = make([]int64, numPosBuckets) + } + if numNegBuckets > 0 { + it.nBuckets = make([]int64, numNegBuckets) + it.nBucketsDelta = make([]int64, numNegBuckets) + } + + // Now read the actual data. + t, err := binary.ReadVarint(&it.br) + if err != nil { + it.err = err + return false + } + it.t = t + + cnt, err := binary.ReadUvarint(&it.br) + if err != nil { + it.err = err + return false + } + it.cnt = cnt + + zcnt, err := binary.ReadUvarint(&it.br) + if err != nil { + it.err = err + return false + } + it.zCnt = zcnt + + sum, err := it.br.readBits(64) + if err != nil { + it.err = err + return false + } + it.sum = math.Float64frombits(sum) + + for i := range it.pBuckets { + v, err := binary.ReadVarint(&it.br) + if err != nil { + it.err = err + return false + } + it.pBuckets[i] = v + } + for i := range it.nBuckets { + v, err := binary.ReadVarint(&it.br) + if err != nil { + it.err = err + return false + } + it.nBuckets[i] = v + } + + it.numRead++ + return true + } + + if it.numRead == 1 { + tDelta, err := binary.ReadVarint(&it.br) + if err != nil { + it.err = err + return false + } + it.tDelta = tDelta + it.t += int64(it.tDelta) + + cntDelta, err := binary.ReadVarint(&it.br) + if err != nil { + it.err = err + return false + } + it.cntDelta = cntDelta + it.cnt = uint64(int64(it.cnt) + it.cntDelta) + + zcntDelta, err := binary.ReadVarint(&it.br) + if err != nil { + it.err = err + return false + } + it.zCntDelta = zcntDelta + it.zCnt = uint64(int64(it.zCnt) + it.zCntDelta) + + ok := it.readSum() + if !ok { + return false + } + + if value.IsStaleNaN(it.sum) { + it.numRead++ + return true + } + + for i := range it.pBuckets { + delta, err := binary.ReadVarint(&it.br) + if err != nil { + it.err = err + return false + } + it.pBucketsDelta[i] = delta + it.pBuckets[i] = it.pBuckets[i] + delta + } + + for i := range it.nBuckets { + delta, err := binary.ReadVarint(&it.br) + if err != nil { + it.err = err + return false + } + it.nBucketsDelta[i] = delta + it.nBuckets[i] = it.nBuckets[i] + delta + } + + it.numRead++ + return true + } + + tDod, err := readVarbitInt(&it.br) + if err != nil { + it.err = err + return false + } + it.tDelta = it.tDelta + tDod + it.t += it.tDelta + + cntDod, err := readVarbitInt(&it.br) + if err != nil { + it.err = err + return false + } + it.cntDelta = it.cntDelta + cntDod + it.cnt = uint64(int64(it.cnt) + it.cntDelta) + + zcntDod, err := readVarbitInt(&it.br) + if err != nil { + it.err = err + return false + } + it.zCntDelta = it.zCntDelta + zcntDod + it.zCnt = uint64(int64(it.zCnt) + it.zCntDelta) + + ok := it.readSum() + if !ok { + return false + } + + if value.IsStaleNaN(it.sum) { + it.numRead++ + return true + } + + for i := range it.pBuckets { + dod, err := readVarbitInt(&it.br) + if err != nil { + it.err = err + return false + } + it.pBucketsDelta[i] = it.pBucketsDelta[i] + dod + it.pBuckets[i] = it.pBuckets[i] + it.pBucketsDelta[i] + } + + for i := range it.nBuckets { + dod, err := readVarbitInt(&it.br) + if err != nil { + it.err = err + return false + } + it.nBucketsDelta[i] = it.nBucketsDelta[i] + dod + it.nBuckets[i] = it.nBuckets[i] + it.nBucketsDelta[i] + } + + it.numRead++ + return true +} + +func (it *histogramIterator) readSum() bool { + bit, err := it.br.readBitFast() + if err != nil { + bit, err = it.br.readBit() + } + if err != nil { + it.err = err + return false + } + + if bit == zero { + return true // it.sum = it.sum + } + + bit, err = it.br.readBitFast() + if err != nil { + bit, err = it.br.readBit() + } + if err != nil { + it.err = err + return false + } + if bit == zero { + // Reuse leading/trailing zero bits. + // it.leading, it.trailing = it.leading, it.trailing + } else { + bits, err := it.br.readBitsFast(5) + if err != nil { + bits, err = it.br.readBits(5) + } + if err != nil { + it.err = err + return false + } + it.leading = uint8(bits) + + bits, err = it.br.readBitsFast(6) + if err != nil { + bits, err = it.br.readBits(6) + } + if err != nil { + it.err = err + return false + } + mbits := uint8(bits) + // 0 significant bits here means we overflowed and we actually + // need 64; see comment in encoder. + if mbits == 0 { + mbits = 64 + } + it.trailing = 64 - it.leading - mbits + } + + mbits := 64 - it.leading - it.trailing + bits, err := it.br.readBitsFast(mbits) + if err != nil { + bits, err = it.br.readBits(mbits) + } + if err != nil { + it.err = err + return false + } + vbits := math.Float64bits(it.sum) + vbits ^= bits << it.trailing + it.sum = math.Float64frombits(vbits) + return true +} diff --git a/tsdb/chunkenc/histogram_meta.go b/tsdb/chunkenc/histogram_meta.go new file mode 100644 index 0000000000..ac4badee12 --- /dev/null +++ b/tsdb/chunkenc/histogram_meta.go @@ -0,0 +1,286 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package chunkenc + +import ( + "github.com/prometheus/prometheus/model/histogram" +) + +func writeHistogramChunkMeta(b *bstream, schema int32, zeroThreshold float64, positiveSpans, negativeSpans []histogram.Span) { + putVarbitInt(b, int64(schema)) + putVarbitFloat(b, zeroThreshold) + putHistogramChunkMetaSpans(b, positiveSpans) + putHistogramChunkMetaSpans(b, negativeSpans) +} + +func putHistogramChunkMetaSpans(b *bstream, spans []histogram.Span) { + putVarbitInt(b, int64(len(spans))) + for _, s := range spans { + putVarbitInt(b, int64(s.Length)) + putVarbitInt(b, int64(s.Offset)) + } +} + +func readHistogramChunkMeta(b *bstreamReader) ( + schema int32, zeroThreshold float64, + positiveSpans, negativeSpans []histogram.Span, + err error, +) { + _, err = b.ReadByte() // The header. + if err != nil { + return + } + + v, err := readVarbitInt(b) + if err != nil { + return + } + schema = int32(v) + + zeroThreshold, err = readVarbitFloat(b) + if err != nil { + return + } + + positiveSpans, err = readHistogramChunkMetaSpans(b) + if err != nil { + return + } + + negativeSpans, err = readHistogramChunkMetaSpans(b) + if err != nil { + return + } + + return +} + +func readHistogramChunkMetaSpans(b *bstreamReader) ([]histogram.Span, error) { + var spans []histogram.Span + num, err := readVarbitInt(b) + if err != nil { + return nil, err + } + for i := 0; i < int(num); i++ { + + length, err := readVarbitInt(b) + if err != nil { + return nil, err + } + + offset, err := readVarbitInt(b) + if err != nil { + return nil, err + } + + spans = append(spans, histogram.Span{ + Length: uint32(length), + Offset: int32(offset), + }) + } + return spans, nil +} + +type bucketIterator struct { + spans []histogram.Span + span int // Span position of last yielded bucket. + bucket int // Bucket position within span of last yielded bucket. + idx int // Bucket index (globally across all spans) of last yielded bucket. +} + +func newBucketIterator(spans []histogram.Span) *bucketIterator { + b := bucketIterator{ + spans: spans, + span: 0, + bucket: -1, + idx: -1, + } + if len(spans) > 0 { + b.idx += int(spans[0].Offset) + } + return &b +} + +func (b *bucketIterator) Next() (int, bool) { + // We're already out of bounds. + if b.span >= len(b.spans) { + return 0, false + } +try: + if b.bucket < int(b.spans[b.span].Length-1) { // Try to move within same span. + b.bucket++ + b.idx++ + return b.idx, true + } else if b.span < len(b.spans)-1 { // Try to move from one span to the next. + b.span++ + b.idx += int(b.spans[b.span].Offset + 1) + b.bucket = 0 + if b.spans[b.span].Length == 0 { + // Pathological case that should never happen. We can't use this span, let's try again. + goto try + } + return b.idx, true + } + // We're out of options. + return 0, false +} + +// An Interjection describes how many new buckets have to be introduced before +// processing the pos'th delta from the original slice. +type Interjection struct { + pos int + num int +} + +// compareSpans returns the interjections to convert a slice of deltas to a new +// slice representing an expanded set of buckets, or false if incompatible +// (e.g. if buckets were removed). +// +// Example: +// +// Let's say the old buckets look like this: +// +// span syntax: [offset, length] +// spans : [ 0 , 2 ] [2,1] [ 3 , 2 ] [3,1] [1,1] +// bucket idx : [0] [1] 2 3 [4] 5 6 7 [8] [9] 10 11 12 [13] 14 [15] +// raw values 6 3 3 2 4 5 1 +// deltas 6 -3 0 -1 2 1 -4 +// +// But now we introduce a new bucket layout. (Carefully chosen example where we +// have a span appended, one unchanged[*], one prepended, and two merge - in +// that order.) +// +// [*] unchanged in terms of which bucket indices they represent. but to achieve +// that, their offset needs to change if "disrupted" by spans changing ahead of +// them +// +// \/ this one is "unchanged" +// spans : [ 0 , 3 ] [1,1] [ 1 , 4 ] [ 3 , 3 ] +// bucket idx : [0] [1] [2] 3 [4] 5 [6] [7] [8] [9] 10 11 12 [13] [14] [15] +// raw values 6 3 0 3 0 0 2 4 5 0 1 +// deltas 6 -3 -3 3 -3 0 2 2 1 -5 1 +// delta mods: / \ / \ / \ +// +// Note that whenever any new buckets are introduced, the subsequent "old" +// bucket needs to readjust its delta to the new base of 0. Thus, for the caller +// who wants to transform the set of original deltas to a new set of deltas to +// match a new span layout that adds buckets, we simply need to generate a list +// of interjections. +// +// Note: Within compareSpans we don't have to worry about the changes to the +// spans themselves, thanks to the iterators we get to work with the more useful +// bucket indices (which of course directly correspond to the buckets we have to +// adjust). +func compareSpans(a, b []histogram.Span) ([]Interjection, bool) { + ai := newBucketIterator(a) + bi := newBucketIterator(b) + + var interjections []Interjection + + // When inter.num becomes > 0, this becomes a valid interjection that + // should be yielded when we finish a streak of new buckets. + var inter Interjection + + av, aOK := ai.Next() + bv, bOK := bi.Next() +loop: + for { + switch { + case aOK && bOK: + switch { + case av == bv: // Both have an identical value. move on! + // Finish WIP interjection and reset. + if inter.num > 0 { + interjections = append(interjections, inter) + } + inter.num = 0 + av, aOK = ai.Next() + bv, bOK = bi.Next() + inter.pos++ + case av < bv: // b misses a value that is in a. + return interjections, false + case av > bv: // a misses a value that is in b. Forward b and recompare. + inter.num++ + bv, bOK = bi.Next() + } + case aOK && !bOK: // b misses a value that is in a. + return interjections, false + case !aOK && bOK: // a misses a value that is in b. Forward b and recompare. + inter.num++ + bv, bOK = bi.Next() + default: // Both iterators ran out. We're done. + if inter.num > 0 { + interjections = append(interjections, inter) + } + break loop + } + } + + return interjections, true +} + +// interject merges 'in' with the provided interjections and writes them into +// 'out', which must already have the appropriate length. +func interject(in, out []int64, interjections []Interjection) []int64 { + var ( + j int // Position in out. + v int64 // The last value seen. + interj int // The next interjection to process. + ) + for i, d := range in { + if interj < len(interjections) && i == interjections[interj].pos { + + // We have an interjection! + // Add interjection.num new delta values such that their + // bucket values equate 0. + out[j] = int64(-v) + j++ + for x := 1; x < interjections[interj].num; x++ { + out[j] = 0 + j++ + } + interj++ + + // Now save the value from the input. The delta value we + // should save is the original delta value + the last + // value of the point before the interjection (to undo + // the delta that was introduced by the interjection). + out[j] = d + v + j++ + v = d + v + continue + } + + // If there was no interjection, the original delta is still + // valid. + out[j] = d + j++ + v += d + } + switch interj { + case len(interjections): + // All interjections processed. Nothing more to do. + case len(interjections) - 1: + // One more interjection to process at the end. + out[j] = int64(-v) + j++ + for x := 1; x < interjections[interj].num; x++ { + out[j] = 0 + j++ + } + default: + panic("unprocessed interjections left") + } + return out +} diff --git a/tsdb/chunkenc/histo_meta_test.go b/tsdb/chunkenc/histogram_meta_test.go similarity index 93% rename from tsdb/chunkenc/histo_meta_test.go rename to tsdb/chunkenc/histogram_meta_test.go index a8339ca2f2..9c98b72959 100644 --- a/tsdb/chunkenc/histo_meta_test.go +++ b/tsdb/chunkenc/histogram_meta_test.go @@ -21,13 +21,15 @@ package chunkenc import ( "testing" - "github.com/prometheus/prometheus/pkg/histogram" + "github.com/prometheus/prometheus/model/histogram" "github.com/stretchr/testify/require" ) -// example of a span layout and resulting bucket indices (_idx_ is used in this histogram, others are shown just for context) -// spans : [offset: 0, length: 2] [offset 1, length 1] -// bucket idx : _0_ _1_ 2 [3] 4 ... +// Example of a span layout and resulting bucket indices (_idx_ is used in this +// histogram, others are shown just for context): +// +// spans : [offset: 0, length: 2] [offset 1, length 1] +// bucket idx : _0_ _1_ 2 [3] 4 ... func TestBucketIterator(t *testing.T) { type test struct { @@ -74,7 +76,7 @@ func TestBucketIterator(t *testing.T) { }, idxs: []int{100, 101, 102, 103, 112, 113, 114, 115, 116, 117, 118, 119}, }, - // the below 2 sets ore the ones described in compareSpans's comments + // The below 2 sets ore the ones described in compareSpans's comments. { spans: []histogram.Span{ {Offset: 0, Length: 2}, diff --git a/tsdb/chunkenc/histo_test.go b/tsdb/chunkenc/histogram_test.go similarity index 81% rename from tsdb/chunkenc/histo_test.go rename to tsdb/chunkenc/histogram_test.go index 71210e57a7..ca5ea72395 100644 --- a/tsdb/chunkenc/histo_test.go +++ b/tsdb/chunkenc/histogram_test.go @@ -16,21 +16,21 @@ package chunkenc import ( "testing" - "github.com/prometheus/prometheus/pkg/histogram" + "github.com/prometheus/prometheus/model/histogram" "github.com/stretchr/testify/require" ) -func TestHistoChunkSameBuckets(t *testing.T) { - c := NewHistoChunk() +func TestHistogramChunkSameBuckets(t *testing.T) { + c := NewHistogramChunk() var exp []res - // Create fresh appender and add the first histogram + // Create fresh appender and add the first histogram. app, err := c.Appender() require.NoError(t, err) require.Equal(t, 0, c.NumSamples()) ts := int64(1234567890) - h := histogram.SparseHistogram{ + h := histogram.Histogram{ Count: 5, ZeroCount: 2, Sum: 18.4, @@ -56,7 +56,7 @@ func TestHistoChunkSameBuckets(t *testing.T) { exp = append(exp, res{t: ts, h: h}) require.Equal(t, 2, c.NumSamples()) - // Add update with new appender + // Add update with new appender. app, err = c.Appender() require.NoError(t, err) @@ -113,12 +113,12 @@ func TestHistoChunkSameBuckets(t *testing.T) { type res struct { t int64 - h histogram.SparseHistogram + h histogram.Histogram } -// mimics the scenario described for compareSpans() -func TestHistoChunkBucketChanges(t *testing.T) { - c := Chunk(NewHistoChunk()) +// Mimics the scenario described for compareSpans(). +func TestHistogramChunkBucketChanges(t *testing.T) { + c := Chunk(NewHistogramChunk()) // Create fresh appender and add the first histogram. app, err := c.Appender() @@ -126,7 +126,7 @@ func TestHistoChunkBucketChanges(t *testing.T) { require.Equal(t, 0, c.NumSamples()) ts1 := int64(1234567890) - h1 := histogram.SparseHistogram{ + h1 := histogram.Histogram{ Count: 5, ZeroCount: 2, Sum: 18.4, @@ -157,24 +157,26 @@ func TestHistoChunkBucketChanges(t *testing.T) { h2.Count += 9 h2.ZeroCount++ h2.Sum = 30 - // Existing histogram should get values converted from the above to: 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between) + // Existing histogram should get values converted from the above to: + // 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between) // so the new histogram should have new counts >= these per-bucket counts, e.g.: h2.PositiveBuckets = []int64{7, -2, -4, 2, -2, -1, 2, 3, 0, -5, 1} // 7 5 1 3 1 0 2 5 5 0 1 (total 30) // This is how span changes will be handled. - histoApp, _ := app.(*HistoAppender) - posInterjections, negInterjections, ok, cr := histoApp.Appendable(h2) + hApp, _ := app.(*HistogramAppender) + posInterjections, negInterjections, ok, cr := hApp.Appendable(h2) require.Greater(t, len(posInterjections), 0) require.Equal(t, 0, len(negInterjections)) require.True(t, ok) // Only new buckets came in. require.False(t, cr) - c, app = histoApp.Recode(posInterjections, negInterjections, h2.PositiveSpans, h2.NegativeSpans) + c, app = hApp.Recode(posInterjections, negInterjections, h2.PositiveSpans, h2.NegativeSpans) app.AppendHistogram(ts2, h2) require.Equal(t, 2, c.NumSamples()) - // Because the 2nd histogram has expanded buckets, we should expect all histograms (in particular the first) - // to come back using the new spans metadata as well as the expanded buckets. + // Because the 2nd histogram has expanded buckets, we should expect all + // histograms (in particular the first) to come back using the new spans + // metadata as well as the expanded buckets. h1.PositiveSpans = h2.PositiveSpans h1.PositiveBuckets = []int64{6, -3, -3, 3, -3, 0, 2, 2, 1, -5, 1} exp := []res{ @@ -192,7 +194,7 @@ func TestHistoChunkBucketChanges(t *testing.T) { } func TestHistoChunkAppendable(t *testing.T) { - c := Chunk(NewHistoChunk()) + c := Chunk(NewHistogramChunk()) // Create fresh appender and add the first histogram. app, err := c.Appender() @@ -200,7 +202,7 @@ func TestHistoChunkAppendable(t *testing.T) { require.Equal(t, 0, c.NumSamples()) ts := int64(1234567890) - h1 := histogram.SparseHistogram{ + h1 := histogram.Histogram{ Count: 5, ZeroCount: 2, Sum: 18.4, @@ -230,12 +232,13 @@ func TestHistoChunkAppendable(t *testing.T) { h2.Count += 9 h2.ZeroCount++ h2.Sum = 30 - // Existing histogram should get values converted from the above to: 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between) + // Existing histogram should get values converted from the above to: + // 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between) // so the new histogram should have new counts >= these per-bucket counts, e.g.: h2.PositiveBuckets = []int64{7, -2, -4, 2, -2, -1, 2, 3, 0, -5, 1} // 7 5 1 3 1 0 2 5 5 0 1 (total 30) - histoApp, _ := app.(*HistoAppender) - posInterjections, negInterjections, ok, cr := histoApp.Appendable(h2) + hApp, _ := app.(*HistogramAppender) + posInterjections, negInterjections, ok, cr := hApp.Appendable(h2) require.Greater(t, len(posInterjections), 0) require.Equal(t, 0, len(negInterjections)) require.True(t, ok) // Only new buckets came in. @@ -253,8 +256,8 @@ func TestHistoChunkAppendable(t *testing.T) { h2.Sum = 21 h2.PositiveBuckets = []int64{6, -3, -1, 2, 1, -4} // counts: 6, 3, 2, 4, 5, 1 (total 21) - histoApp, _ := app.(*HistoAppender) - posInterjections, negInterjections, ok, cr := histoApp.Appendable(h2) + hApp, _ := app.(*HistogramAppender) + posInterjections, negInterjections, ok, cr := hApp.Appendable(h2) require.Equal(t, 0, len(posInterjections)) require.Equal(t, 0, len(negInterjections)) require.False(t, ok) // Need to cut a new chunk. @@ -266,8 +269,8 @@ func TestHistoChunkAppendable(t *testing.T) { h2.Sum = 23 h2.PositiveBuckets = []int64{6, -4, 1, -1, 2, 1, -4} // counts: 6, 2, 3, 2, 4, 5, 1 (total 23) - histoApp, _ := app.(*HistoAppender) - posInterjections, negInterjections, ok, cr := histoApp.Appendable(h2) + hApp, _ := app.(*HistogramAppender) + posInterjections, negInterjections, ok, cr := hApp.Appendable(h2) require.Equal(t, 0, len(posInterjections)) require.Equal(t, 0, len(negInterjections)) require.False(t, ok) // Need to cut a new chunk. @@ -283,20 +286,24 @@ func TestHistoChunkAppendable(t *testing.T) { {Offset: 3, Length: 3}, } h2.Sum = 29 - // Existing histogram should get values converted from the above to: 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between) + // Existing histogram should get values converted from the above to: + // 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between) // so the new histogram should have new counts >= these per-bucket counts, e.g.: h2.PositiveBuckets = []int64{7, -2, -4, 2, -2, -1, 2, 3, 0, -5, 0} // 7 5 1 3 1 0 2 5 5 0 0 (total 29) - histoApp, _ := app.(*HistoAppender) - posInterjections, negInterjections, ok, cr := histoApp.Appendable(h2) + hApp, _ := app.(*HistogramAppender) + posInterjections, negInterjections, ok, cr := hApp.Appendable(h2) require.Equal(t, 0, len(posInterjections)) require.Equal(t, 0, len(negInterjections)) require.False(t, ok) // Need to cut a new chunk. require.True(t, cr) } - { // New histogram that has a counter reset while new buckets were added before the first bucket and reset on first bucket. - // (to catch the edge case where the new bucket should be forwarded ahead until first old bucket at start) + { + // New histogram that has a counter reset while new buckets were + // added before the first bucket and reset on first bucket. (to + // catch the edge case where the new bucket should be forwarded + // ahead until first old bucket at start) h2 := h1 h2.PositiveSpans = []histogram.Span{ {Offset: -3, Length: 2}, @@ -307,12 +314,13 @@ func TestHistoChunkAppendable(t *testing.T) { {Offset: 1, Length: 1}, } h2.Sum = 26 - // Existing histogram should get values converted from the above to: 0, 0, 6, 3, 3, 2, 4, 5, 1 + // Existing histogram should get values converted from the above to: + // 0, 0, 6, 3, 3, 2, 4, 5, 1 // so the new histogram should have new counts >= these per-bucket counts, e.g.: h2.PositiveBuckets = []int64{1, 1, 3, -2, 0, -1, 2, 1, -4} // counts: 1, 2, 5, 3, 3, 2, 4, 5, 1 (total 26) - histoApp, _ := app.(*HistoAppender) - posInterjections, negInterjections, ok, cr := histoApp.Appendable(h2) + hApp, _ := app.(*HistogramAppender) + posInterjections, negInterjections, ok, cr := hApp.Appendable(h2) require.Equal(t, 0, len(posInterjections)) require.Equal(t, 0, len(negInterjections)) require.False(t, ok) // Need to cut a new chunk. diff --git a/tsdb/chunkenc/varbit.go b/tsdb/chunkenc/varbit.go new file mode 100644 index 0000000000..3465c1af1a --- /dev/null +++ b/tsdb/chunkenc/varbit.go @@ -0,0 +1,143 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package chunkenc + +import ( + "math" +) + +// putVarbitFloat writes a float64 using varbit encoding. It does so by +// converting the underlying bits into an int64. +func putVarbitFloat(b *bstream, val float64) { + // TODO(beorn7): The resulting int64 here will almost never be a small + // integer. Thus, the varbit encoding doesn't really make sense + // here. This function is only used to encode the zero threshold in + // histograms. Based on that, here is an idea to improve the encoding: + // + // It is recommended to use (usually negative) powers of two as + // threshoulds. The default value for the zero threshald is in fact + // 2^-128, or 0.5*2^-127, as it is represented by IEEE 754. It is + // therefore worth a try to test if the threshold is a power of 2 and + // then just store the exponent. 0 is also a commen threshold for those + // use cases where only observations of precisely zero should go to the + // zero bucket. This results in the following proposal: + // - First we store 1 byte. + // - Iff that byte is 255 (all bits set), it is followed by a direct + // 8byte representation of the float. + // - If the byte is 0, the threshold is 0. + // - In all other cases, take the number represented by the byte, + // subtract 246, and that's the exponent (i.e. between -245 and + // +8, covering thresholds that are powers of 2 between 2^-246 + // to 128). + putVarbitInt(b, int64(math.Float64bits(val))) +} + +// readVarbitFloat reads a float64 encoded with putVarbitFloat +func readVarbitFloat(b *bstreamReader) (float64, error) { + val, err := readVarbitInt(b) + if err != nil { + return 0, err + } + return math.Float64frombits(uint64(val)), nil +} + +// putVarbitInt writes an int64 using varbit encoding with a bit bucketing +// optimized for the dod's observed in histogram buckets. +// +// TODO(Dieterbe): We could improve this further: Each branch doesn't need to +// support any values of any of the prior branches. So we can expand the range +// of each branch. Do more with fewer bits. It comes at the price of more +// expensive encoding and decoding (cutting out and later adding back that +// center-piece we skip). +func putVarbitInt(b *bstream, val int64) { + switch { + case val == 0: + b.writeBit(zero) + case bitRange(val, 3): // -3 <= val <= 4 + b.writeBits(0b10, 2) + b.writeBits(uint64(val), 3) + case bitRange(val, 6): // -31 <= val <= 32 + b.writeBits(0b110, 3) + b.writeBits(uint64(val), 6) + case bitRange(val, 9): // -255 <= val <= 256 + b.writeBits(0b1110, 4) + b.writeBits(uint64(val), 9) + case bitRange(val, 12): // -2047 <= val <= 2048 + b.writeBits(0b11110, 5) + b.writeBits(uint64(val), 12) + default: + b.writeBits(0b11111, 5) + b.writeBits(uint64(val), 64) + } +} + +// readVarbitInt reads an int64 encoced with putVarbitInt. +func readVarbitInt(b *bstreamReader) (int64, error) { + var d byte + for i := 0; i < 5; i++ { + d <<= 1 + bit, err := b.readBitFast() + if err != nil { + bit, err = b.readBit() + } + if err != nil { + return 0, err + } + if bit == zero { + break + } + d |= 1 + } + + var val int64 + var sz uint8 + + switch d { + case 0b0: + // val == 0 + case 0b10: + sz = 3 + case 0b110: + sz = 6 + case 0b1110: + sz = 9 + case 0b11110: + sz = 12 + case 0b11111: + // Do not use fast because it's very unlikely it will succeed. + bits, err := b.readBits(64) + if err != nil { + return 0, err + } + + val = int64(bits) + } + + if sz != 0 { + bits, err := b.readBitsFast(sz) + if err != nil { + bits, err = b.readBits(sz) + } + if err != nil { + return 0, err + } + if bits > (1 << (sz - 1)) { + // Or something. + bits = bits - (1 << sz) + } + val = int64(bits) + } + + return val, nil +} diff --git a/tsdb/chunkenc/varbit_buckets.go b/tsdb/chunkenc/varbit_buckets.go deleted file mode 100644 index 5bae350f6d..0000000000 --- a/tsdb/chunkenc/varbit_buckets.go +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright 2021 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The code in this file was largely written by Damian Gryski as part of -// https://github.com/dgryski/go-tsz and published under the license below. -// It was modified to accommodate reading from byte slices without modifying -// the underlying bytes, which would panic when reading from mmap'd -// read-only byte slices. - -// Copyright (c) 2015,2016 Damian Gryski -// All rights reserved. - -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: - -// * Redistributions of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// * Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package chunkenc - -import ( - "math" -) - -// putFloat64VBBucket writes a float64 using varbit optimized for SHS buckets. -// It does so by converting the underlying bits into an int64. -func putFloat64VBBucket(b *bstream, val float64) { - // TODO: Since this is used for the zero threshold, this almost always goes into the default - // bit range (i.e. using 5+64 bits). So we can consider skipping `putInt64VBBucket` and directly - // write the float and save 5 bits here. - putInt64VBBucket(b, int64(math.Float64bits(val))) -} - -// readFloat64VBBucket reads a float64 using varbit optimized for SHS buckets -func readFloat64VBBucket(b *bstreamReader) (float64, error) { - val, err := readInt64VBBucket(b) - if err != nil { - return 0, err - } - return math.Float64frombits(uint64(val)), nil -} - -// putInt64VBBucket writes an int64 using varbit optimized for SHS buckets. -// -// TODO(Dieterbe): We could improve this further: Each branch doesn't need to -// support any values of any of the prior branches. So we can expand the range -// of each branch. Do more with fewer bits. It comes at the price of more -// expensive encoding and decoding (cutting out and later adding back that -// center-piece we skip). -func putInt64VBBucket(b *bstream, val int64) { - switch { - case val == 0: - b.writeBit(zero) - case bitRange(val, 3): // -3 <= val <= 4 - b.writeBits(0b10, 2) - b.writeBits(uint64(val), 3) - case bitRange(val, 6): // -31 <= val <= 32 - b.writeBits(0b110, 3) - b.writeBits(uint64(val), 6) - case bitRange(val, 9): // -255 <= val <= 256 - b.writeBits(0b1110, 4) - b.writeBits(uint64(val), 9) - case bitRange(val, 12): // -2047 <= val <= 2048 - b.writeBits(0b11110, 5) - b.writeBits(uint64(val), 12) - default: - b.writeBits(0b11111, 5) - b.writeBits(uint64(val), 64) - } -} - -// readInt64VBBucket reads an int64 using varbit optimized for SHS buckets -func readInt64VBBucket(b *bstreamReader) (int64, error) { - var d byte - for i := 0; i < 5; i++ { - d <<= 1 - bit, err := b.readBitFast() - if err != nil { - bit, err = b.readBit() - } - if err != nil { - return 0, err - } - if bit == zero { - break - } - d |= 1 - } - - var val int64 - var sz uint8 - - switch d { - case 0b0: - // val == 0 - case 0b10: - sz = 3 - case 0b110: - sz = 6 - case 0b1110: - sz = 9 - case 0b11110: - sz = 12 - case 0b11111: - // Do not use fast because it's very unlikely it will succeed. - bits, err := b.readBits(64) - if err != nil { - return 0, err - } - - val = int64(bits) - } - - if sz != 0 { - bits, err := b.readBitsFast(sz) - if err != nil { - bits, err = b.readBits(sz) - } - if err != nil { - return 0, err - } - if bits > (1 << (sz - 1)) { - // or something - bits = bits - (1 << sz) - } - val = int64(bits) - } - - return val, nil -} diff --git a/tsdb/chunkenc/xor.go b/tsdb/chunkenc/xor.go index 24cec61cbd..21c35d3c1b 100644 --- a/tsdb/chunkenc/xor.go +++ b/tsdb/chunkenc/xor.go @@ -48,7 +48,7 @@ import ( "math" "math/bits" - "github.com/prometheus/prometheus/pkg/histogram" + "github.com/prometheus/prometheus/model/histogram" ) const ( @@ -150,8 +150,8 @@ type xorAppender struct { trailing uint8 } -func (a *xorAppender) AppendHistogram(t int64, h histogram.SparseHistogram) { - //panic("cannot call xorAppender.AppendHistogram().") +func (a *xorAppender) AppendHistogram(t int64, h histogram.Histogram) { + panic("appended a histogram to an xor chunk") } func (a *xorAppender) Append(t int64, v float64) { @@ -181,6 +181,12 @@ func (a *xorAppender) Append(t int64, v float64) { // Gorilla has a max resolution of seconds, Prometheus milliseconds. // Thus we use higher value range steps with larger bit size. + // + // TODO(beorn7): This seems to needlessly jump to large bit + // sizes even for very small deviations from zero. Timestamp + // compression can probably benefit from some smaller bit + // buckets. See also what was done for histogram encoding in + // varbit.go. switch { case dod == 0: a.b.writeBit(zero) @@ -278,7 +284,7 @@ func (it *xorIterator) At() (int64, float64) { return it.t, it.val } -func (it *xorIterator) AtHistogram() (int64, histogram.SparseHistogram) { +func (it *xorIterator) AtHistogram() (int64, histogram.Histogram) { panic("cannot call xorIterator.AtHistogram().") } diff --git a/tsdb/compact_test.go b/tsdb/compact_test.go index 02c9bd3426..e9ac42077e 100644 --- a/tsdb/compact_test.go +++ b/tsdb/compact_test.go @@ -32,7 +32,7 @@ import ( prom_testutil "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" - "github.com/prometheus/prometheus/pkg/histogram" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/chunkenc" @@ -1328,7 +1328,7 @@ func TestHeadCompactionWithHistograms(t *testing.T) { type timedHist struct { t int64 - h histogram.SparseHistogram + h histogram.Histogram } // Ingest samples. @@ -1384,7 +1384,7 @@ func TestHeadCompactionWithHistograms(t *testing.T) { // Depending on numSeriesPerSchema, it can take few gigs of memory; // the test adds all samples to appender before committing instead of // buffering the writes to make it run faster. -func TestSparseHistoSpaceSavings(t *testing.T) { +func TestSparseHistogramSpaceSavings(t *testing.T) { t.Skip() cases := []struct { @@ -1455,7 +1455,7 @@ func TestSparseHistoSpaceSavings(t *testing.T) { var allSparseSeries []struct { baseLabels labels.Labels - hists []histogram.SparseHistogram + hists []histogram.Histogram } for sid, schema := range allSchemas { @@ -1467,7 +1467,7 @@ func TestSparseHistoSpaceSavings(t *testing.T) { } allSparseSeries = append(allSparseSeries, struct { baseLabels labels.Labels - hists []histogram.SparseHistogram + hists []histogram.Histogram }{baseLabels: lbls, hists: generateCustomHistograms(numHistograms, c.numBuckets, c.numSpans, c.gapBetweenSpans, schema)}) } } @@ -1522,13 +1522,13 @@ func TestSparseHistoSpaceSavings(t *testing.T) { h := ah.hists[i] numOldSeriesPerHistogram = 0 - it := histogram.CumulativeExpandSparseHistogram(h) + it := h.CumulativeBucketIterator() itIdx := 0 var err error for it.Next() { numOldSeriesPerHistogram++ b := it.At() - lbls := append(ah.baseLabels, labels.Label{Name: "le", Value: fmt.Sprintf("%.16f", b.Le)}) + lbls := append(ah.baseLabels, labels.Label{Name: "le", Value: fmt.Sprintf("%.16f", b.Upper)}) refs[itIdx], err = oldApp.Append(refs[itIdx], lbls, ts, float64(b.Count)) require.NoError(t, err) itIdx++ @@ -1614,9 +1614,9 @@ Savings: Index=%.2f%%, Chunks=%.2f%%, Total=%.2f%% } } -func generateCustomHistograms(numHists, numBuckets, numSpans, gapBetweenSpans, schema int) (r []histogram.SparseHistogram) { +func generateCustomHistograms(numHists, numBuckets, numSpans, gapBetweenSpans, schema int) (r []histogram.Histogram) { // First histogram with all the settings. - h := histogram.SparseHistogram{ + h := histogram.Histogram{ Sum: 1000 * rand.Float64(), Schema: int32(schema), } @@ -1711,7 +1711,7 @@ func TestSparseHistogramCompactionAndQuery(t *testing.T) { type timedHist struct { t int64 - h histogram.SparseHistogram + h histogram.Histogram } expHists := make(map[string][]timedHist) diff --git a/tsdb/docs/format/chunks.md b/tsdb/docs/format/chunks.md index 5a8b9edc77..54b8b000e1 100644 --- a/tsdb/docs/format/chunks.md +++ b/tsdb/docs/format/chunks.md @@ -34,8 +34,14 @@ in-file offset (lower 4 bytes) and segment sequence number (upper 4 bytes). └───────────────┴───────────────────┴──────────────┴────────────────┘ ``` +## XOR chunk + +TODO(beorn7): Add. + ## Histogram chunk +TODO(beorn7): This is out of date. Update once settled on the (more or less) final format. + ``` ┌──────────────┬─────────────────┬──────────────────────────┬──────────────────────────┬──────────────┐ │ len │ schema │ pos-spans │ neg-spans │ data │ diff --git a/tsdb/head.go b/tsdb/head.go index 6cda5e537c..fffee9f704 100644 --- a/tsdb/head.go +++ b/tsdb/head.go @@ -29,8 +29,8 @@ import ( "go.uber.org/atomic" "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/pkg/exemplar" - "github.com/prometheus/prometheus/pkg/histogram" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/chunkenc" @@ -614,7 +614,7 @@ func (h *Head) Init(minValidTime int64) error { sparseHistogramSeries := 0 for _, m := range h.series.series { for _, ms := range m { - if ms.sparseHistogramSeries { + if ms.histogramSeries { sparseHistogramSeries++ } } @@ -1397,7 +1397,7 @@ func (s *stripeSeries) gc(mint int64) (map[uint64]struct{}, int, int64, int) { s.locks[j].Lock() } - if series.sparseHistogramSeries { + if series.histogramSeries { sparseHistogramSeriesDeleted++ } deleted[series.ref] = struct{}{} @@ -1488,9 +1488,9 @@ func (s *stripeSeries) getOrSet(hash uint64, lset labels.Labels, createSeries fu return series, true, nil } -type hist struct { +type histogramSample struct { t int64 - h histogram.SparseHistogram + h histogram.Histogram } type sample struct { @@ -1517,7 +1517,7 @@ type memSeries struct { nextAt int64 // Timestamp at which to cut the next chunk. sampleBuf [4]sample - histBuf [4]hist + histogramBuf [4]histogramSample pendingCommit bool // Whether there are samples waiting to be committed to this series. app chunkenc.Appender // Current appender for the chunk. @@ -1527,7 +1527,7 @@ type memSeries struct { txs *txRing // Temporary variable for sparsehistogram experiment. - sparseHistogramSeries bool + histogramSeries bool } func newMemSeries(lset labels.Labels, id uint64, chunkRange int64, memChunkPool *sync.Pool) *memSeries { diff --git a/tsdb/head_append.go b/tsdb/head_append.go index 40431d9041..7364d1d94c 100644 --- a/tsdb/head_append.go +++ b/tsdb/head_append.go @@ -21,8 +21,8 @@ import ( "github.com/go-kit/log/level" "github.com/pkg/errors" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/pkg/exemplar" - "github.com/prometheus/prometheus/pkg/histogram" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/value" "github.com/prometheus/prometheus/storage" @@ -67,14 +67,14 @@ func (a *initAppender) AppendExemplar(ref uint64, l labels.Labels, e exemplar.Ex return a.app.AppendExemplar(ref, l, e) } -func (a *initAppender) AppendHistogram(ref uint64, l labels.Labels, t int64, sh histogram.SparseHistogram) (uint64, error) { +func (a *initAppender) AppendHistogram(ref uint64, l labels.Labels, t int64, h histogram.Histogram) (uint64, error) { if a.app != nil { - return a.app.AppendHistogram(ref, l, t, sh) + return a.app.AppendHistogram(ref, l, t, h) } a.head.initTime(t) a.app = a.head.appender() - return a.app.AppendHistogram(ref, l, t, sh) + return a.app.AppendHistogram(ref, l, t, h) } // initTime initializes a head with the first timestamp. This only needs to be called @@ -269,8 +269,8 @@ func (a *headAppender) Append(ref uint64, lset labels.Labels, t int64, v float64 } } - if value.IsStaleNaN(v) && s.sparseHistogramSeries { - return a.AppendHistogram(ref, lset, t, histogram.SparseHistogram{Sum: v}) + if value.IsStaleNaN(v) && s.histogramSeries { + return a.AppendHistogram(ref, lset, t, histogram.Histogram{Sum: v}) } s.Lock() @@ -322,7 +322,7 @@ func (s *memSeries) appendable(t int64, v float64) error { } // appendableHistogram checks whether the given sample is valid for appending to the series. -func (s *memSeries) appendableHistogram(t int64, sh histogram.SparseHistogram) error { +func (s *memSeries) appendableHistogram(t int64, sh histogram.Histogram) error { c := s.head() if c == nil { return nil @@ -372,7 +372,7 @@ func (a *headAppender) AppendExemplar(ref uint64, _ labels.Labels, e exemplar.Ex return s.ref, nil } -func (a *headAppender) AppendHistogram(ref uint64, lset labels.Labels, t int64, sh histogram.SparseHistogram) (uint64, error) { +func (a *headAppender) AppendHistogram(ref uint64, lset labels.Labels, t int64, h histogram.Histogram) (uint64, error) { if t < a.minValidTime { a.head.metrics.outOfBoundSamples.Inc() return 0, storage.ErrOutOfBounds @@ -396,7 +396,7 @@ func (a *headAppender) AppendHistogram(ref uint64, lset labels.Labels, t int64, if err != nil { return 0, err } - s.sparseHistogramSeries = true + s.histogramSeries = true if created { a.head.metrics.sparseHistogramSeries.Inc() a.series = append(a.series, record.RefSeries{ @@ -407,7 +407,7 @@ func (a *headAppender) AppendHistogram(ref uint64, lset labels.Labels, t int64, } s.Lock() - if err := s.appendableHistogram(t, sh); err != nil { + if err := s.appendableHistogram(t, h); err != nil { s.Unlock() if err == storage.ErrOutOfOrderSample { a.head.metrics.outOfOrderSamples.Inc() @@ -427,7 +427,7 @@ func (a *headAppender) AppendHistogram(ref uint64, lset labels.Labels, t int64, a.histograms = append(a.histograms, record.RefHistogram{ Ref: s.ref, T: t, - H: sh, + H: h, }) a.histogramSeries = append(a.histogramSeries, s) return s.ref, nil @@ -604,37 +604,39 @@ func (s *memSeries) append(t int64, v float64, appendID uint64, chunkDiskMapper return true, chunkCreated } -// appendHistogram adds the sparse histogram. +// appendHistogram adds the histogram. // It is unsafe to call this concurrently with s.iterator(...) without holding the series lock. -func (s *memSeries) appendHistogram(t int64, sh histogram.SparseHistogram, appendID uint64, chunkDiskMapper *chunks.ChunkDiskMapper) (sampleInOrder, chunkCreated bool) { +func (s *memSeries) appendHistogram(t int64, sh histogram.Histogram, appendID uint64, chunkDiskMapper *chunks.ChunkDiskMapper) (sampleInOrder, chunkCreated bool) { // Head controls the execution of recoding, so that we own the proper chunk reference afterwards. // We check for Appendable before appendPreprocessor because in case it ends up creating a new chunk, // we need to know if there was also a counter reset or not to set the meta properly. - app, _ := s.app.(*chunkenc.HistoAppender) + app, _ := s.app.(*chunkenc.HistogramAppender) var ( - posInterjections, negInterjections []chunkenc.Interjection - okToAppend, counterReset bool + positiveInterjections, negativeInterjections []chunkenc.Interjection + okToAppend, counterReset bool ) if app != nil { - posInterjections, negInterjections, okToAppend, counterReset = app.Appendable(sh) + positiveInterjections, negativeInterjections, okToAppend, counterReset = app.Appendable(sh) } - c, sampleInOrder, chunkCreated := s.appendPreprocessor(t, chunkenc.EncSHS, chunkDiskMapper) + c, sampleInOrder, chunkCreated := s.appendPreprocessor(t, chunkenc.EncHistogram, chunkDiskMapper) if !sampleInOrder { return sampleInOrder, chunkCreated } if !chunkCreated { // We have 3 cases here - // !okToAppend -> we need to cut a new chunk - // okToAppend but we have interjections -> existing chunk needs recoding before we can append our histogram - // okToAppend and no interjections -> chunk is ready to support our histogram + // - !okToAppend -> We need to cut a new chunk. + // - okToAppend but we have interjections -> Existing chunk needs recoding before we can append our histogram. + // - okToAppend and no interjections -> Chunk is ready to support our histogram. if !okToAppend || counterReset { - c = s.cutNewHeadChunk(t, chunkenc.EncSHS, chunkDiskMapper) + c = s.cutNewHeadChunk(t, chunkenc.EncHistogram, chunkDiskMapper) chunkCreated = true - } else if len(posInterjections) > 0 || len(negInterjections) > 0 { - // new buckets have appeared. we need to recode all prior histograms within the chunk before we can process this one. - chunk, app := app.Recode(posInterjections, negInterjections, sh.PositiveSpans, sh.NegativeSpans) + } else if len(positiveInterjections) > 0 || len(negativeInterjections) > 0 { + // New buckets have appeared. We need to recode all + // prior histogram samples within the chunk before we + // can process this one. + chunk, app := app.Recode(positiveInterjections, negativeInterjections, sh.PositiveSpans, sh.NegativeSpans) s.headChunk = &memChunk{ minTime: s.headChunk.minTime, maxTime: s.headChunk.maxTime, @@ -645,7 +647,7 @@ func (s *memSeries) appendHistogram(t int64, sh histogram.SparseHistogram, appen } if chunkCreated { - hc := s.headChunk.chunk.(*chunkenc.HistoChunk) + hc := s.headChunk.chunk.(*chunkenc.HistogramChunk) header := chunkenc.UnknownCounterReset if counterReset { header = chunkenc.CounterReset @@ -656,14 +658,14 @@ func (s *memSeries) appendHistogram(t int64, sh histogram.SparseHistogram, appen } s.app.AppendHistogram(t, sh) - s.sparseHistogramSeries = true + s.histogramSeries = true c.maxTime = t - s.histBuf[0] = s.histBuf[1] - s.histBuf[1] = s.histBuf[2] - s.histBuf[2] = s.histBuf[3] - s.histBuf[3] = hist{t: t, h: sh} + s.histogramBuf[0] = s.histogramBuf[1] + s.histogramBuf[1] = s.histogramBuf[2] + s.histogramBuf[2] = s.histogramBuf[3] + s.histogramBuf[3] = histogramSample{t: t, h: sh} if appendID > 0 { s.txs.add(appendID) diff --git a/tsdb/head_read.go b/tsdb/head_read.go index 2710de8fad..c9d496f784 100644 --- a/tsdb/head_read.go +++ b/tsdb/head_read.go @@ -21,7 +21,7 @@ import ( "github.com/go-kit/log/level" "github.com/pkg/errors" - "github.com/prometheus/prometheus/pkg/histogram" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/chunkenc" @@ -443,7 +443,7 @@ func (s *memSeries) iterator(id int, isoState *isolationState, chunkDiskMapper * msIter.total = numSamples msIter.stopAfter = stopAfter msIter.buf = s.sampleBuf - msIter.histBuf = s.histBuf + msIter.histogramBuf = s.histogramBuf return msIter } return &memSafeIterator{ @@ -452,18 +452,18 @@ func (s *memSeries) iterator(id int, isoState *isolationState, chunkDiskMapper * i: -1, stopAfter: stopAfter, }, - total: numSamples, - buf: s.sampleBuf, - histBuf: s.histBuf, + total: numSamples, + buf: s.sampleBuf, + histogramBuf: s.histogramBuf, } } type memSafeIterator struct { stopIterator - total int - buf [4]sample - histBuf [4]hist + total int + buf [4]sample + histogramBuf [4]histogramSample } func (it *memSafeIterator) Seek(t int64) bool { @@ -502,11 +502,11 @@ func (it *memSafeIterator) At() (int64, float64) { return s.t, s.v } -func (it *memSafeIterator) AtHistogram() (int64, histogram.SparseHistogram) { +func (it *memSafeIterator) AtHistogram() (int64, histogram.Histogram) { if it.total-it.i > 4 { return it.Iterator.AtHistogram() } - s := it.histBuf[4-(it.total-it.i)] + s := it.histogramBuf[4-(it.total-it.i)] return s.t, s.h } diff --git a/tsdb/head_test.go b/tsdb/head_test.go index f9655eff53..852e3d847a 100644 --- a/tsdb/head_test.go +++ b/tsdb/head_test.go @@ -37,8 +37,8 @@ import ( "go.uber.org/atomic" "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/pkg/exemplar" - "github.com/prometheus/prometheus/pkg/histogram" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/value" "github.com/prometheus/prometheus/storage" @@ -2539,15 +2539,15 @@ func TestAppendHistogram(t *testing.T) { require.NoError(t, head.Init(0)) app := head.Appender(context.Background()) - type timedHist struct { + type timedHistogram struct { t int64 - h histogram.SparseHistogram + h histogram.Histogram } - expHists := make([]timedHist, 0, numHistograms) + expHistograms := make([]timedHistogram, 0, numHistograms) for i, h := range generateHistograms(numHistograms) { _, err := app.AppendHistogram(0, l, int64(i), h) require.NoError(t, err) - expHists = append(expHists, timedHist{int64(i), h}) + expHistograms = append(expHistograms, timedHistogram{int64(i), h}) } require.NoError(t, app.Commit()) @@ -2564,13 +2564,13 @@ func TestAppendHistogram(t *testing.T) { require.False(t, ss.Next()) it := s.Iterator() - actHists := make([]timedHist, 0, len(expHists)) + actHistograms := make([]timedHistogram, 0, len(expHistograms)) for it.Next() { t, h := it.AtHistogram() - actHists = append(actHists, timedHist{t, h.Copy()}) + actHistograms = append(actHistograms, timedHistogram{t, h.Copy()}) } - require.Equal(t, expHists, actHists) + require.Equal(t, expHistograms, actHistograms) }) } } @@ -2586,17 +2586,17 @@ func TestHistogramInWAL(t *testing.T) { require.NoError(t, head.Init(0)) app := head.Appender(context.Background()) - type timedHist struct { + type timedHistogram struct { t int64 - h histogram.SparseHistogram + h histogram.Histogram } - expHists := make([]timedHist, 0, numHistograms) + expHistograms := make([]timedHistogram, 0, numHistograms) for i, h := range generateHistograms(numHistograms) { h.NegativeSpans = h.PositiveSpans h.NegativeBuckets = h.PositiveBuckets _, err := app.AppendHistogram(0, l, int64(i), h) require.NoError(t, err) - expHists = append(expHists, timedHist{int64(i), h}) + expHistograms = append(expHistograms, timedHistogram{int64(i), h}) } require.NoError(t, app.Commit()) @@ -2621,18 +2621,18 @@ func TestHistogramInWAL(t *testing.T) { require.False(t, ss.Next()) it := s.Iterator() - actHists := make([]timedHist, 0, len(expHists)) + actHistograms := make([]timedHistogram, 0, len(expHistograms)) for it.Next() { t, h := it.AtHistogram() - actHists = append(actHists, timedHist{t, h.Copy()}) + actHistograms = append(actHistograms, timedHistogram{t, h.Copy()}) } - require.Equal(t, expHists, actHists) + require.Equal(t, expHistograms, actHistograms) } -func generateHistograms(n int) (r []histogram.SparseHistogram) { +func generateHistograms(n int) (r []histogram.Histogram) { for i := 0; i < n; i++ { - r = append(r, histogram.SparseHistogram{ + r = append(r, histogram.Histogram{ Count: 5 + uint64(i*4), ZeroCount: 2 + uint64(i), ZeroThreshold: 0.001, @@ -2957,22 +2957,22 @@ func TestSparseHistogramMetrics(t *testing.T) { }) require.NoError(t, head.Init(0)) - expHistSeries, expHistSamples := 0, 0 + expHSeries, expHSamples := 0, 0 for x := 0; x < 5; x++ { - expHistSeries++ + expHSeries++ l := labels.Labels{{Name: "a", Value: fmt.Sprintf("b%d", x)}} for i, h := range generateHistograms(10) { app := head.Appender(context.Background()) _, err := app.AppendHistogram(0, l, int64(i), h) require.NoError(t, err) require.NoError(t, app.Commit()) - expHistSamples++ + expHSamples++ } } - require.Equal(t, float64(expHistSeries), prom_testutil.ToFloat64(head.metrics.sparseHistogramSeries)) - require.Equal(t, float64(expHistSamples), prom_testutil.ToFloat64(head.metrics.sparseHistogramSamplesTotal)) + require.Equal(t, float64(expHSeries), prom_testutil.ToFloat64(head.metrics.sparseHistogramSeries)) + require.Equal(t, float64(expHSamples), prom_testutil.ToFloat64(head.metrics.sparseHistogramSamplesTotal)) require.NoError(t, head.Close()) w, err := wal.NewSize(nil, nil, head.wal.Dir(), 32768, false) @@ -2981,7 +2981,7 @@ func TestSparseHistogramMetrics(t *testing.T) { require.NoError(t, err) require.NoError(t, head.Init(0)) - require.Equal(t, float64(expHistSeries), prom_testutil.ToFloat64(head.metrics.sparseHistogramSeries)) + require.Equal(t, float64(expHSeries), prom_testutil.ToFloat64(head.metrics.sparseHistogramSeries)) require.Equal(t, float64(0), prom_testutil.ToFloat64(head.metrics.sparseHistogramSamplesTotal)) // Counter reset. } @@ -2994,11 +2994,11 @@ func TestSparseHistogramStaleSample(t *testing.T) { }) require.NoError(t, head.Init(0)) - type timedHist struct { + type timedHistogram struct { t int64 - h histogram.SparseHistogram + h histogram.Histogram } - expHists := make([]timedHist, 0, numHistograms) + expHistograms := make([]timedHistogram, 0, numHistograms) testQuery := func(numStale int) { q, err := NewBlockQuerier(head, head.MinTime(), head.MaxTime()) @@ -3014,17 +3014,17 @@ func TestSparseHistogramStaleSample(t *testing.T) { require.False(t, ss.Next()) it := s.Iterator() - actHists := make([]timedHist, 0, len(expHists)) + actHistograms := make([]timedHistogram, 0, len(expHistograms)) for it.Next() { t, h := it.AtHistogram() - actHists = append(actHists, timedHist{t, h.Copy()}) + actHistograms = append(actHistograms, timedHistogram{t, h.Copy()}) } // We cannot compare StaleNAN with require.Equal, hence checking each histogram manually. - require.Equal(t, len(expHists), len(actHists)) + require.Equal(t, len(expHistograms), len(actHistograms)) actNumStale := 0 - for i, eh := range expHists { - ah := actHists[i] + for i, eh := range expHistograms { + ah := actHistograms[i] if value.IsStaleNaN(eh.h.Sum) { actNumStale++ require.True(t, value.IsStaleNaN(ah.h.Sum)) @@ -3040,14 +3040,14 @@ func TestSparseHistogramStaleSample(t *testing.T) { // Adding stale in the same appender. app := head.Appender(context.Background()) for _, h := range generateHistograms(numHistograms) { - _, err := app.AppendHistogram(0, l, 100*int64(len(expHists)), h) + _, err := app.AppendHistogram(0, l, 100*int64(len(expHistograms)), h) require.NoError(t, err) - expHists = append(expHists, timedHist{100 * int64(len(expHists)), h}) + expHistograms = append(expHistograms, timedHistogram{100 * int64(len(expHistograms)), h}) } // +1 so that delta-of-delta is not 0. - _, err := app.Append(0, l, 100*int64(len(expHists))+1, math.Float64frombits(value.StaleNaN)) + _, err := app.Append(0, l, 100*int64(len(expHistograms))+1, math.Float64frombits(value.StaleNaN)) require.NoError(t, err) - expHists = append(expHists, timedHist{100*int64(len(expHists)) + 1, histogram.SparseHistogram{Sum: math.Float64frombits(value.StaleNaN)}}) + expHistograms = append(expHistograms, timedHistogram{100*int64(len(expHistograms)) + 1, histogram.Histogram{Sum: math.Float64frombits(value.StaleNaN)}}) require.NoError(t, app.Commit()) // Only 1 chunk in the memory, no m-mapped chunk. @@ -3059,17 +3059,17 @@ func TestSparseHistogramStaleSample(t *testing.T) { // Adding stale in different appender and continuing series after a stale sample. app = head.Appender(context.Background()) for _, h := range generateHistograms(2 * numHistograms)[numHistograms:] { - _, err := app.AppendHistogram(0, l, 100*int64(len(expHists)), h) + _, err := app.AppendHistogram(0, l, 100*int64(len(expHistograms)), h) require.NoError(t, err) - expHists = append(expHists, timedHist{100 * int64(len(expHists)), h}) + expHistograms = append(expHistograms, timedHistogram{100 * int64(len(expHistograms)), h}) } require.NoError(t, app.Commit()) app = head.Appender(context.Background()) // +1 so that delta-of-delta is not 0. - _, err = app.Append(0, l, 100*int64(len(expHists))+1, math.Float64frombits(value.StaleNaN)) + _, err = app.Append(0, l, 100*int64(len(expHistograms))+1, math.Float64frombits(value.StaleNaN)) require.NoError(t, err) - expHists = append(expHists, timedHist{100*int64(len(expHists)) + 1, histogram.SparseHistogram{Sum: math.Float64frombits(value.StaleNaN)}}) + expHistograms = append(expHistograms, timedHistogram{100*int64(len(expHistograms)) + 1, histogram.Histogram{Sum: math.Float64frombits(value.StaleNaN)}}) require.NoError(t, app.Commit()) // Total 2 chunks, 1 m-mapped. diff --git a/tsdb/head_wal.go b/tsdb/head_wal.go index b41dc34f3a..e7e18fdb2b 100644 --- a/tsdb/head_wal.go +++ b/tsdb/head_wal.go @@ -15,11 +15,6 @@ package tsdb import ( "fmt" - "github.com/prometheus/prometheus/pkg/labels" - "github.com/prometheus/prometheus/tsdb/chunkenc" - "github.com/prometheus/prometheus/tsdb/encoding" - tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" - "github.com/prometheus/prometheus/tsdb/fileutil" "io/ioutil" "math" "os" @@ -30,6 +25,12 @@ import ( "sync" "time" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/prometheus/prometheus/tsdb/encoding" + tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" + "github.com/prometheus/prometheus/tsdb/fileutil" + "github.com/go-kit/log/level" "github.com/pkg/errors" "go.uber.org/atomic" @@ -159,7 +160,7 @@ func (h *Head) loadWAL(r *wal.Reader, multiRef map[uint64]uint64, mmappedChunks if ms.head() == nil { // First histogram for the series. Count this in metrics. - ms.sparseHistogramSeries = true + ms.histogramSeries = true } if rh.T < h.minValidTime.Load() { diff --git a/tsdb/querier.go b/tsdb/querier.go index 7e4c3269b5..ec6310da52 100644 --- a/tsdb/querier.go +++ b/tsdb/querier.go @@ -21,7 +21,7 @@ import ( "github.com/pkg/errors" - "github.com/prometheus/prometheus/pkg/histogram" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/chunkenc" @@ -649,7 +649,7 @@ func (p *populateWithDelSeriesIterator) Seek(t int64) bool { } func (p *populateWithDelSeriesIterator) At() (int64, float64) { return p.curr.At() } -func (p *populateWithDelSeriesIterator) AtHistogram() (int64, histogram.SparseHistogram) { +func (p *populateWithDelSeriesIterator) AtHistogram() (int64, histogram.Histogram) { return p.curr.AtHistogram() } func (p *populateWithDelSeriesIterator) ChunkEncoding() chunkenc.Encoding { @@ -688,8 +688,8 @@ func (p *populateWithDelChunkSeriesIterator) Next() bool { app chunkenc.Appender err error ) - if p.currDelIter.ChunkEncoding() == chunkenc.EncSHS { - newChunk = chunkenc.NewHistoChunk() + if p.currDelIter.ChunkEncoding() == chunkenc.EncHistogram { + newChunk = chunkenc.NewHistogramChunk() app, err = newChunk.Appender() } else { newChunk = chunkenc.NewXORChunk() @@ -714,11 +714,11 @@ func (p *populateWithDelChunkSeriesIterator) Next() bool { var ( t int64 v float64 - h histogram.SparseHistogram + h histogram.Histogram ) - if p.currDelIter.ChunkEncoding() == chunkenc.EncSHS { - if hc, ok := p.currChkMeta.Chunk.(*chunkenc.HistoChunk); ok { - newChunk.(*chunkenc.HistoChunk).SetCounterResetHeader(hc.GetCounterResetHeader()) + if p.currDelIter.ChunkEncoding() == chunkenc.EncHistogram { + if hc, ok := p.currChkMeta.Chunk.(*chunkenc.HistogramChunk); ok { + newChunk.(*chunkenc.HistogramChunk).SetCounterResetHeader(hc.GetCounterResetHeader()) } t, h = p.currDelIter.AtHistogram() p.curr.MinTime = t @@ -870,7 +870,7 @@ func (it *DeletedIterator) At() (int64, float64) { return it.Iter.At() } -func (it *DeletedIterator) AtHistogram() (int64, histogram.SparseHistogram) { +func (it *DeletedIterator) AtHistogram() (int64, histogram.Histogram) { t, h := it.Iter.AtHistogram() return t, h } @@ -889,7 +889,7 @@ func (it *DeletedIterator) Seek(t int64) bool { // Now double check if the entry falls into a deleted interval. var ts int64 - if it.ChunkEncoding() == chunkenc.EncSHS { + if it.ChunkEncoding() == chunkenc.EncHistogram { ts, _ = it.AtHistogram() } else { ts, _ = it.At() @@ -916,7 +916,7 @@ func (it *DeletedIterator) Next() bool { Outer: for it.Iter.Next() { var ts int64 - if it.ChunkEncoding() == chunkenc.EncSHS { + if it.ChunkEncoding() == chunkenc.EncHistogram { ts, _ = it.AtHistogram() } else { ts, _ = it.At() diff --git a/tsdb/record/record.go b/tsdb/record/record.go index cd0ff058a3..25c2107b82 100644 --- a/tsdb/record/record.go +++ b/tsdb/record/record.go @@ -20,7 +20,7 @@ import ( "github.com/pkg/errors" - "github.com/prometheus/prometheus/pkg/histogram" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb/encoding" "github.com/prometheus/prometheus/tsdb/tombstones" @@ -74,7 +74,7 @@ type RefExemplar struct { type RefHistogram struct { Ref uint64 T int64 - H histogram.SparseHistogram + H histogram.Histogram } // Decoder decodes series, sample, and tombstone records. @@ -233,14 +233,14 @@ func (d *Decoder) ExemplarsFromBuffer(dec *encoding.Decbuf, exemplars []RefExemp return exemplars, nil } -func (d *Decoder) Histograms(rec []byte, hists []RefHistogram) ([]RefHistogram, error) { +func (d *Decoder) Histograms(rec []byte, histograms []RefHistogram) ([]RefHistogram, error) { dec := encoding.Decbuf{B: rec} t := Type(dec.Byte()) if t != Histograms { return nil, errors.New("invalid record type") } if dec.Len() == 0 { - return hists, nil + return histograms, nil } var ( baseRef = dec.Be64() @@ -253,7 +253,7 @@ func (d *Decoder) Histograms(rec []byte, hists []RefHistogram) ([]RefHistogram, rh := RefHistogram{ Ref: baseRef + uint64(dref), T: baseTime + dtime, - H: histogram.SparseHistogram{ + H: histogram.Histogram{ Schema: 0, ZeroThreshold: 0, ZeroCount: 0, @@ -303,16 +303,16 @@ func (d *Decoder) Histograms(rec []byte, hists []RefHistogram) ([]RefHistogram, rh.H.NegativeBuckets[i] = dec.Varint64() } - hists = append(hists, rh) + histograms = append(histograms, rh) } if dec.Err() != nil { - return nil, errors.Wrapf(dec.Err(), "decode error after %d histograms", len(hists)) + return nil, errors.Wrapf(dec.Err(), "decode error after %d histograms", len(histograms)) } if len(dec.B) > 0 { return nil, errors.Errorf("unexpected %d bytes left in entry", len(dec.B)) } - return hists, nil + return histograms, nil } // Encoder encodes series, sample, and tombstones records. @@ -410,21 +410,21 @@ func (e *Encoder) EncodeExemplarsIntoBuffer(exemplars []RefExemplar, buf *encodi } } -func (e *Encoder) Histograms(hists []RefHistogram, b []byte) []byte { +func (e *Encoder) Histograms(histograms []RefHistogram, b []byte) []byte { buf := encoding.Encbuf{B: b} buf.PutByte(byte(Histograms)) - if len(hists) == 0 { + if len(histograms) == 0 { return buf.Get() } // Store base timestamp and base reference number of first histogram. // All histograms encode their timestamp and ref as delta to those. - first := hists[0] + first := histograms[0] buf.PutBE64(first.Ref) buf.PutBE64int64(first.T) - for _, h := range hists { + for _, h := range histograms { buf.PutVarint64(int64(h.Ref) - int64(first.Ref)) buf.PutVarint64(h.T - first.T) diff --git a/tsdb/tsdbutil/buffer.go b/tsdb/tsdbutil/buffer.go index e96267f38e..2c9bbb10bd 100644 --- a/tsdb/tsdbutil/buffer.go +++ b/tsdb/tsdbutil/buffer.go @@ -16,7 +16,7 @@ package tsdbutil import ( "math" - "github.com/prometheus/prometheus/pkg/histogram" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/tsdb/chunkenc" ) @@ -160,8 +160,8 @@ func (it *sampleRingIterator) At() (int64, float64) { return it.r.at(it.i) } -func (it *sampleRingIterator) AtHistogram() (int64, histogram.SparseHistogram) { - return 0, histogram.SparseHistogram{} +func (it *sampleRingIterator) AtHistogram() (int64, histogram.Histogram) { + return 0, histogram.Histogram{} } func (it *sampleRingIterator) ChunkEncoding() chunkenc.Encoding { diff --git a/tsdb/tsdbutil/buffer_test.go b/tsdb/tsdbutil/buffer_test.go index ef9c061020..0401e6c879 100644 --- a/tsdb/tsdbutil/buffer_test.go +++ b/tsdb/tsdbutil/buffer_test.go @@ -18,7 +18,7 @@ import ( "sort" "testing" - "github.com/prometheus/prometheus/pkg/histogram" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/stretchr/testify/require" ) @@ -152,8 +152,8 @@ func (it *listSeriesIterator) At() (int64, float64) { return s.t, s.v } -func (it *listSeriesIterator) AtHistogram() (int64, histogram.SparseHistogram) { - return 0, histogram.SparseHistogram{} +func (it *listSeriesIterator) AtHistogram() (int64, histogram.Histogram) { + return 0, histogram.Histogram{} } func (it *listSeriesIterator) ChunkEncoding() chunkenc.Encoding { diff --git a/web/api/v1/api_test.go b/web/api/v1/api_test.go index a1649912b7..b839e61cd5 100644 --- a/web/api/v1/api_test.go +++ b/web/api/v1/api_test.go @@ -296,7 +296,6 @@ var sampleFlagMap = map[string]string{ } func TestEndpoints(t *testing.T) { - t.Skip() suite, err := promql.NewTest(t, ` load 1m test_metric1{foo="bar"} 0+100x100