2018-05-17 06:02:47 -07:00
|
|
|
// Copyright 2018 The Prometheus Authors
|
|
|
|
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
2021-11-17 02:21:27 -08:00
|
|
|
// Package record contains the various record types used for encoding various Head block data in the WAL and in-memory snapshot.
|
2019-09-19 02:15:41 -07:00
|
|
|
package record
|
2018-05-17 06:02:47 -07:00
|
|
|
|
|
|
|
import (
|
|
|
|
"math"
|
|
|
|
|
|
|
|
"github.com/pkg/errors"
|
2020-10-22 02:00:08 -07:00
|
|
|
|
Style cleanup of all the changes in sparsehistogram so far
A lot of this code was hacked together, literally during a
hackathon. This commit intends not to change the code substantially,
but just make the code obey the usual style practices.
A (possibly incomplete) list of areas:
* Generally address linter warnings.
* The `pgk` directory is deprecated as per dev-summit. No new packages should
be added to it. I moved the new `pkg/histogram` package to `model`
anticipating what's proposed in #9478.
* Make the naming of the Sparse Histogram more consistent. Including
abbreviations, there were just too many names for it: SparseHistogram,
Histogram, Histo, hist, his, shs, h. The idea is to call it "Histogram" in
general. Only add "Sparse" if it is needed to avoid confusion with
conventional Histograms (which is rare because the TSDB really has no notion
of conventional Histograms). Use abbreviations only in local scope, and then
really abbreviate (not just removing three out of seven letters like in
"Histo"). This is in the spirit of
https://github.com/golang/go/wiki/CodeReviewComments#variable-names
* Several other minor name changes.
* A lot of formatting of doc comments. For one, following
https://github.com/golang/go/wiki/CodeReviewComments#comment-sentences
, but also layout question, anticipating how things will look like
when rendered by `godoc` (even where `godoc` doesn't render them
right now because they are for unexported types or not a doc comment
at all but just a normal code comment - consistency is queen!).
* Re-enabled `TestQueryLog` and `TestEndopints` (they pass now,
leaving them disabled was presumably an oversight).
* Bucket iterator for histogram.Histogram is now created with a
method.
* HistogramChunk.iterator now allows iterator recycling. (I think
@dieterbe only commented it out because he was confused by the
question in the comment.)
* HistogramAppender.Append panics now because we decided to treat
staleness marker differently.
Signed-off-by: beorn7 <beorn@grafana.com>
2021-10-09 06:57:07 -07:00
|
|
|
"github.com/prometheus/prometheus/model/histogram"
|
2021-11-08 06:23:17 -08:00
|
|
|
"github.com/prometheus/prometheus/model/labels"
|
2022-07-19 01:58:52 -07:00
|
|
|
"github.com/prometheus/prometheus/model/textparse"
|
2021-11-06 03:10:04 -07:00
|
|
|
"github.com/prometheus/prometheus/storage"
|
|
|
|
"github.com/prometheus/prometheus/tsdb/chunks"
|
2019-08-13 01:34:14 -07:00
|
|
|
"github.com/prometheus/prometheus/tsdb/encoding"
|
2019-09-19 02:15:41 -07:00
|
|
|
"github.com/prometheus/prometheus/tsdb/tombstones"
|
2018-05-17 06:02:47 -07:00
|
|
|
)
|
|
|
|
|
2019-09-19 02:15:41 -07:00
|
|
|
// Type represents the data type of a record.
|
|
|
|
type Type uint8
|
2018-05-17 06:02:47 -07:00
|
|
|
|
|
|
|
const (
|
2020-10-05 02:09:59 -07:00
|
|
|
// Unknown is returned for unrecognised WAL record types.
|
|
|
|
Unknown Type = 255
|
2019-09-19 02:15:41 -07:00
|
|
|
// Series is used to match WAL records of type Series.
|
|
|
|
Series Type = 1
|
|
|
|
// Samples is used to match WAL records of type Samples.
|
|
|
|
Samples Type = 2
|
|
|
|
// Tombstones is used to match WAL records of type Tombstones.
|
|
|
|
Tombstones Type = 3
|
2021-05-06 13:53:52 -07:00
|
|
|
// Exemplars is used to match WAL records of type Exemplars.
|
|
|
|
Exemplars Type = 4
|
2022-09-20 10:05:50 -07:00
|
|
|
// MmapMarkers is used to match OOO WBL records of type MmapMarkers.
|
|
|
|
MmapMarkers Type = 5
|
2022-07-19 01:58:52 -07:00
|
|
|
// Metadata is used to match WAL records of type Metadata.
|
|
|
|
Metadata Type = 6
|
2022-10-05 13:14:49 -07:00
|
|
|
// HistogramSamples is used to match WAL records of type Histograms.
|
2022-08-29 05:08:36 -07:00
|
|
|
HistogramSamples Type = 7
|
2022-12-28 00:55:07 -08:00
|
|
|
// FloatHistogramSamples is used to match WAL records of type Float Histograms.
|
|
|
|
FloatHistogramSamples Type = 8
|
2018-05-17 06:02:47 -07:00
|
|
|
)
|
|
|
|
|
2022-07-18 03:24:11 -07:00
|
|
|
func (rt Type) String() string {
|
|
|
|
switch rt {
|
|
|
|
case Series:
|
|
|
|
return "series"
|
|
|
|
case Samples:
|
|
|
|
return "samples"
|
|
|
|
case Tombstones:
|
|
|
|
return "tombstones"
|
2022-08-10 08:54:37 -07:00
|
|
|
case Exemplars:
|
|
|
|
return "exemplars"
|
2022-08-29 05:08:36 -07:00
|
|
|
case HistogramSamples:
|
|
|
|
return "histogram_samples"
|
2022-12-28 00:55:07 -08:00
|
|
|
case FloatHistogramSamples:
|
|
|
|
return "float_histogram_samples"
|
2022-09-20 10:05:50 -07:00
|
|
|
case MmapMarkers:
|
|
|
|
return "mmapmarkers"
|
2022-07-19 01:58:52 -07:00
|
|
|
case Metadata:
|
|
|
|
return "metadata"
|
2022-07-18 03:24:11 -07:00
|
|
|
default:
|
|
|
|
return "unknown"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-07-19 01:58:52 -07:00
|
|
|
// MetricType represents the type of a series.
|
|
|
|
type MetricType uint8
|
|
|
|
|
|
|
|
const (
|
2022-08-29 05:08:36 -07:00
|
|
|
UnknownMT MetricType = 0
|
|
|
|
Counter MetricType = 1
|
|
|
|
Gauge MetricType = 2
|
|
|
|
HistogramSample MetricType = 3
|
|
|
|
GaugeHistogram MetricType = 4
|
|
|
|
Summary MetricType = 5
|
|
|
|
Info MetricType = 6
|
|
|
|
Stateset MetricType = 7
|
2022-07-19 01:58:52 -07:00
|
|
|
)
|
|
|
|
|
|
|
|
func GetMetricType(t textparse.MetricType) uint8 {
|
|
|
|
switch t {
|
|
|
|
case textparse.MetricTypeCounter:
|
|
|
|
return uint8(Counter)
|
|
|
|
case textparse.MetricTypeGauge:
|
|
|
|
return uint8(Gauge)
|
|
|
|
case textparse.MetricTypeHistogram:
|
2022-08-29 05:08:36 -07:00
|
|
|
return uint8(HistogramSample)
|
2022-07-19 01:58:52 -07:00
|
|
|
case textparse.MetricTypeGaugeHistogram:
|
|
|
|
return uint8(GaugeHistogram)
|
|
|
|
case textparse.MetricTypeSummary:
|
|
|
|
return uint8(Summary)
|
|
|
|
case textparse.MetricTypeInfo:
|
|
|
|
return uint8(Info)
|
|
|
|
case textparse.MetricTypeStateset:
|
|
|
|
return uint8(Stateset)
|
|
|
|
default:
|
|
|
|
return uint8(UnknownMT)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func ToTextparseMetricType(m uint8) textparse.MetricType {
|
|
|
|
switch m {
|
|
|
|
case uint8(Counter):
|
|
|
|
return textparse.MetricTypeCounter
|
|
|
|
case uint8(Gauge):
|
|
|
|
return textparse.MetricTypeGauge
|
2022-08-29 05:08:36 -07:00
|
|
|
case uint8(HistogramSample):
|
2022-07-19 01:58:52 -07:00
|
|
|
return textparse.MetricTypeHistogram
|
|
|
|
case uint8(GaugeHistogram):
|
|
|
|
return textparse.MetricTypeGaugeHistogram
|
|
|
|
case uint8(Summary):
|
|
|
|
return textparse.MetricTypeSummary
|
|
|
|
case uint8(Info):
|
|
|
|
return textparse.MetricTypeInfo
|
|
|
|
case uint8(Stateset):
|
|
|
|
return textparse.MetricTypeStateset
|
|
|
|
default:
|
|
|
|
return textparse.MetricTypeUnknown
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
const (
|
|
|
|
unitMetaName = "UNIT"
|
|
|
|
helpMetaName = "HELP"
|
2018-05-17 06:02:47 -07:00
|
|
|
)
|
|
|
|
|
2021-10-22 01:06:44 -07:00
|
|
|
// ErrNotFound is returned if a looked up resource was not found. Duplicate ErrNotFound from head.go.
|
|
|
|
var ErrNotFound = errors.New("not found")
|
2019-09-19 02:15:41 -07:00
|
|
|
|
|
|
|
// RefSeries is the series labels with the series ID.
|
|
|
|
type RefSeries struct {
|
2021-11-06 03:10:04 -07:00
|
|
|
Ref chunks.HeadSeriesRef
|
2019-09-19 02:15:41 -07:00
|
|
|
Labels labels.Labels
|
|
|
|
}
|
|
|
|
|
|
|
|
// RefSample is a timestamp/value pair associated with a reference to a series.
|
2022-08-29 05:08:36 -07:00
|
|
|
// TODO(beorn7): Perhaps make this "polymorphic", including histogram and float-histogram pointers? Then get rid of RefHistogramSample.
|
2019-09-19 02:15:41 -07:00
|
|
|
type RefSample struct {
|
2021-11-06 03:10:04 -07:00
|
|
|
Ref chunks.HeadSeriesRef
|
2019-09-19 02:15:41 -07:00
|
|
|
T int64
|
|
|
|
V float64
|
|
|
|
}
|
|
|
|
|
2022-07-19 01:58:52 -07:00
|
|
|
// RefMetadata is the metadata associated with a series ID.
|
|
|
|
type RefMetadata struct {
|
|
|
|
Ref chunks.HeadSeriesRef
|
|
|
|
Type uint8
|
|
|
|
Unit string
|
|
|
|
Help string
|
|
|
|
}
|
|
|
|
|
2021-05-06 13:53:52 -07:00
|
|
|
// RefExemplar is an exemplar with it's labels, timestamp, value the exemplar was collected/observed with, and a reference to a series.
|
|
|
|
type RefExemplar struct {
|
2021-11-06 03:10:04 -07:00
|
|
|
Ref chunks.HeadSeriesRef
|
2021-05-06 13:53:52 -07:00
|
|
|
T int64
|
|
|
|
V float64
|
|
|
|
Labels labels.Labels
|
|
|
|
}
|
|
|
|
|
2022-08-29 05:08:36 -07:00
|
|
|
// RefHistogramSample is a histogram.
|
|
|
|
type RefHistogramSample struct {
|
2021-11-17 10:57:31 -08:00
|
|
|
Ref chunks.HeadSeriesRef
|
2021-06-29 07:38:46 -07:00
|
|
|
T int64
|
2021-11-12 10:07:41 -08:00
|
|
|
H *histogram.Histogram
|
2021-06-29 07:38:46 -07:00
|
|
|
}
|
|
|
|
|
2022-12-28 00:55:07 -08:00
|
|
|
// RefFloatHistogramSample is a float histogram.
|
|
|
|
type RefFloatHistogramSample struct {
|
|
|
|
Ref chunks.HeadSeriesRef
|
|
|
|
T int64
|
|
|
|
FH *histogram.FloatHistogram
|
|
|
|
}
|
|
|
|
|
2022-09-20 10:05:50 -07:00
|
|
|
// RefMmapMarker marks that the all the samples of the given series until now have been m-mapped to disk.
|
|
|
|
type RefMmapMarker struct {
|
|
|
|
Ref chunks.HeadSeriesRef
|
|
|
|
MmapRef chunks.ChunkDiskMapperRef
|
|
|
|
}
|
|
|
|
|
2022-07-19 01:58:52 -07:00
|
|
|
// Decoder decodes series, sample, metadata and tombstone records.
|
2018-05-17 06:02:47 -07:00
|
|
|
// The zero value is ready to use.
|
2022-06-28 08:02:08 -07:00
|
|
|
type Decoder struct {
|
|
|
|
builder labels.ScratchBuilder
|
|
|
|
}
|
2018-05-17 06:02:47 -07:00
|
|
|
|
|
|
|
// Type returns the type of the record.
|
2020-10-05 02:09:59 -07:00
|
|
|
// Returns RecordUnknown if no valid record type is found.
|
2019-09-19 02:15:41 -07:00
|
|
|
func (d *Decoder) Type(rec []byte) Type {
|
2018-05-17 06:02:47 -07:00
|
|
|
if len(rec) < 1 {
|
2020-10-05 02:09:59 -07:00
|
|
|
return Unknown
|
2018-05-17 06:02:47 -07:00
|
|
|
}
|
2019-09-19 02:15:41 -07:00
|
|
|
switch t := Type(rec[0]); t {
|
2022-12-28 00:55:07 -08:00
|
|
|
case Series, Samples, Tombstones, Exemplars, MmapMarkers, Metadata, HistogramSamples, FloatHistogramSamples:
|
2018-05-17 06:02:47 -07:00
|
|
|
return t
|
|
|
|
}
|
2020-10-05 02:09:59 -07:00
|
|
|
return Unknown
|
2018-05-17 06:02:47 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
// Series appends series in rec to the given slice.
|
2019-09-19 02:15:41 -07:00
|
|
|
func (d *Decoder) Series(rec []byte, series []RefSeries) ([]RefSeries, error) {
|
2019-02-22 09:11:11 -08:00
|
|
|
dec := encoding.Decbuf{B: rec}
|
2018-05-17 06:02:47 -07:00
|
|
|
|
2019-09-19 02:15:41 -07:00
|
|
|
if Type(dec.Byte()) != Series {
|
2018-05-17 06:02:47 -07:00
|
|
|
return nil, errors.New("invalid record type")
|
|
|
|
}
|
2019-02-22 09:11:11 -08:00
|
|
|
for len(dec.B) > 0 && dec.Err() == nil {
|
2021-11-06 03:10:04 -07:00
|
|
|
ref := storage.SeriesRef(dec.Be64())
|
2022-07-26 07:42:00 -07:00
|
|
|
lset := d.DecodeLabels(&dec)
|
2018-05-17 06:02:47 -07:00
|
|
|
|
|
|
|
series = append(series, RefSeries{
|
2021-11-06 03:10:04 -07:00
|
|
|
Ref: chunks.HeadSeriesRef(ref),
|
2018-05-17 06:02:47 -07:00
|
|
|
Labels: lset,
|
|
|
|
})
|
|
|
|
}
|
2019-02-22 09:11:11 -08:00
|
|
|
if dec.Err() != nil {
|
|
|
|
return nil, dec.Err()
|
2018-05-17 06:02:47 -07:00
|
|
|
}
|
2019-02-22 09:11:11 -08:00
|
|
|
if len(dec.B) > 0 {
|
|
|
|
return nil, errors.Errorf("unexpected %d bytes left in entry", len(dec.B))
|
2018-05-17 06:02:47 -07:00
|
|
|
}
|
|
|
|
return series, nil
|
|
|
|
}
|
|
|
|
|
2022-07-19 01:58:52 -07:00
|
|
|
// Metadata appends metadata in rec to the given slice.
|
|
|
|
func (d *Decoder) Metadata(rec []byte, metadata []RefMetadata) ([]RefMetadata, error) {
|
|
|
|
dec := encoding.Decbuf{B: rec}
|
|
|
|
|
|
|
|
if Type(dec.Byte()) != Metadata {
|
|
|
|
return nil, errors.New("invalid record type")
|
|
|
|
}
|
|
|
|
for len(dec.B) > 0 && dec.Err() == nil {
|
|
|
|
ref := dec.Uvarint64()
|
|
|
|
typ := dec.Byte()
|
|
|
|
numFields := dec.Uvarint()
|
|
|
|
|
|
|
|
// We're currently aware of two more metadata fields other than TYPE; that is UNIT and HELP.
|
|
|
|
// We can skip the rest of the fields (if we encounter any), but we must decode them anyway
|
|
|
|
// so we can correctly align with the start with the next metadata record.
|
|
|
|
var unit, help string
|
|
|
|
for i := 0; i < numFields; i++ {
|
|
|
|
fieldName := dec.UvarintStr()
|
|
|
|
fieldValue := dec.UvarintStr()
|
|
|
|
switch fieldName {
|
|
|
|
case unitMetaName:
|
|
|
|
unit = fieldValue
|
|
|
|
case helpMetaName:
|
|
|
|
help = fieldValue
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
metadata = append(metadata, RefMetadata{
|
|
|
|
Ref: chunks.HeadSeriesRef(ref),
|
|
|
|
Type: typ,
|
|
|
|
Unit: unit,
|
|
|
|
Help: help,
|
|
|
|
})
|
|
|
|
}
|
|
|
|
if dec.Err() != nil {
|
|
|
|
return nil, dec.Err()
|
|
|
|
}
|
|
|
|
if len(dec.B) > 0 {
|
|
|
|
return nil, errors.Errorf("unexpected %d bytes left in entry", len(dec.B))
|
|
|
|
}
|
|
|
|
return metadata, nil
|
|
|
|
}
|
|
|
|
|
2022-07-26 07:42:00 -07:00
|
|
|
// DecodeLabels decodes one set of labels from buf.
|
|
|
|
func (d *Decoder) DecodeLabels(dec *encoding.Decbuf) labels.Labels {
|
2022-06-28 08:02:08 -07:00
|
|
|
// TODO: reconsider if this function could be pushed down into labels.Labels to be more efficient.
|
|
|
|
d.builder.Reset()
|
|
|
|
nLabels := dec.Uvarint()
|
|
|
|
for i := 0; i < nLabels; i++ {
|
|
|
|
lName := dec.UvarintStr()
|
|
|
|
lValue := dec.UvarintStr()
|
|
|
|
d.builder.Add(lName, lValue)
|
|
|
|
}
|
|
|
|
return d.builder.Labels()
|
2022-07-26 07:42:00 -07:00
|
|
|
}
|
|
|
|
|
2018-05-17 06:02:47 -07:00
|
|
|
// Samples appends samples in rec to the given slice.
|
2019-09-19 02:15:41 -07:00
|
|
|
func (d *Decoder) Samples(rec []byte, samples []RefSample) ([]RefSample, error) {
|
2019-02-22 09:11:11 -08:00
|
|
|
dec := encoding.Decbuf{B: rec}
|
2018-05-17 06:02:47 -07:00
|
|
|
|
2019-09-19 02:15:41 -07:00
|
|
|
if Type(dec.Byte()) != Samples {
|
2018-05-17 06:02:47 -07:00
|
|
|
return nil, errors.New("invalid record type")
|
|
|
|
}
|
2019-02-22 09:11:11 -08:00
|
|
|
if dec.Len() == 0 {
|
2018-05-17 06:02:47 -07:00
|
|
|
return samples, nil
|
|
|
|
}
|
|
|
|
var (
|
2019-02-22 09:11:11 -08:00
|
|
|
baseRef = dec.Be64()
|
|
|
|
baseTime = dec.Be64int64()
|
2018-05-17 06:02:47 -07:00
|
|
|
)
|
2019-02-22 09:11:11 -08:00
|
|
|
for len(dec.B) > 0 && dec.Err() == nil {
|
|
|
|
dref := dec.Varint64()
|
|
|
|
dtime := dec.Varint64()
|
|
|
|
val := dec.Be64()
|
2018-05-17 06:02:47 -07:00
|
|
|
|
|
|
|
samples = append(samples, RefSample{
|
2021-11-06 03:10:04 -07:00
|
|
|
Ref: chunks.HeadSeriesRef(int64(baseRef) + dref),
|
2018-05-17 06:02:47 -07:00
|
|
|
T: baseTime + dtime,
|
|
|
|
V: math.Float64frombits(val),
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2019-02-22 09:11:11 -08:00
|
|
|
if dec.Err() != nil {
|
|
|
|
return nil, errors.Wrapf(dec.Err(), "decode error after %d samples", len(samples))
|
2018-05-17 06:02:47 -07:00
|
|
|
}
|
2019-02-22 09:11:11 -08:00
|
|
|
if len(dec.B) > 0 {
|
|
|
|
return nil, errors.Errorf("unexpected %d bytes left in entry", len(dec.B))
|
2018-05-17 06:02:47 -07:00
|
|
|
}
|
|
|
|
return samples, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Tombstones appends tombstones in rec to the given slice.
|
2019-09-19 02:15:41 -07:00
|
|
|
func (d *Decoder) Tombstones(rec []byte, tstones []tombstones.Stone) ([]tombstones.Stone, error) {
|
2019-02-22 09:11:11 -08:00
|
|
|
dec := encoding.Decbuf{B: rec}
|
2018-05-17 06:02:47 -07:00
|
|
|
|
2019-09-19 02:15:41 -07:00
|
|
|
if Type(dec.Byte()) != Tombstones {
|
2018-05-17 06:02:47 -07:00
|
|
|
return nil, errors.New("invalid record type")
|
|
|
|
}
|
2019-02-22 09:11:11 -08:00
|
|
|
for dec.Len() > 0 && dec.Err() == nil {
|
2019-09-19 02:15:41 -07:00
|
|
|
tstones = append(tstones, tombstones.Stone{
|
2021-11-06 03:10:04 -07:00
|
|
|
Ref: storage.SeriesRef(dec.Be64()),
|
2019-09-19 02:15:41 -07:00
|
|
|
Intervals: tombstones.Intervals{
|
2019-02-22 09:11:11 -08:00
|
|
|
{Mint: dec.Varint64(), Maxt: dec.Varint64()},
|
2018-05-17 06:02:47 -07:00
|
|
|
},
|
|
|
|
})
|
|
|
|
}
|
2019-02-22 09:11:11 -08:00
|
|
|
if dec.Err() != nil {
|
|
|
|
return nil, dec.Err()
|
2018-05-17 06:02:47 -07:00
|
|
|
}
|
2019-02-22 09:11:11 -08:00
|
|
|
if len(dec.B) > 0 {
|
|
|
|
return nil, errors.Errorf("unexpected %d bytes left in entry", len(dec.B))
|
2018-05-17 06:02:47 -07:00
|
|
|
}
|
|
|
|
return tstones, nil
|
|
|
|
}
|
|
|
|
|
2021-05-06 13:53:52 -07:00
|
|
|
func (d *Decoder) Exemplars(rec []byte, exemplars []RefExemplar) ([]RefExemplar, error) {
|
|
|
|
dec := encoding.Decbuf{B: rec}
|
|
|
|
t := Type(dec.Byte())
|
|
|
|
if t != Exemplars {
|
|
|
|
return nil, errors.New("invalid record type")
|
|
|
|
}
|
2021-08-30 07:04:38 -07:00
|
|
|
|
|
|
|
return d.ExemplarsFromBuffer(&dec, exemplars)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (d *Decoder) ExemplarsFromBuffer(dec *encoding.Decbuf, exemplars []RefExemplar) ([]RefExemplar, error) {
|
2021-05-06 13:53:52 -07:00
|
|
|
if dec.Len() == 0 {
|
|
|
|
return exemplars, nil
|
|
|
|
}
|
|
|
|
var (
|
|
|
|
baseRef = dec.Be64()
|
|
|
|
baseTime = dec.Be64int64()
|
|
|
|
)
|
|
|
|
for len(dec.B) > 0 && dec.Err() == nil {
|
|
|
|
dref := dec.Varint64()
|
|
|
|
dtime := dec.Varint64()
|
|
|
|
val := dec.Be64()
|
2022-07-26 07:42:00 -07:00
|
|
|
lset := d.DecodeLabels(dec)
|
2021-05-06 13:53:52 -07:00
|
|
|
|
|
|
|
exemplars = append(exemplars, RefExemplar{
|
2021-11-06 03:10:04 -07:00
|
|
|
Ref: chunks.HeadSeriesRef(baseRef + uint64(dref)),
|
2021-05-06 13:53:52 -07:00
|
|
|
T: baseTime + dtime,
|
|
|
|
V: math.Float64frombits(val),
|
|
|
|
Labels: lset,
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
if dec.Err() != nil {
|
|
|
|
return nil, errors.Wrapf(dec.Err(), "decode error after %d exemplars", len(exemplars))
|
|
|
|
}
|
|
|
|
if len(dec.B) > 0 {
|
|
|
|
return nil, errors.Errorf("unexpected %d bytes left in entry", len(dec.B))
|
|
|
|
}
|
|
|
|
return exemplars, nil
|
|
|
|
}
|
|
|
|
|
2022-09-20 10:05:50 -07:00
|
|
|
func (d *Decoder) MmapMarkers(rec []byte, markers []RefMmapMarker) ([]RefMmapMarker, error) {
|
|
|
|
dec := encoding.Decbuf{B: rec}
|
|
|
|
t := Type(dec.Byte())
|
|
|
|
if t != MmapMarkers {
|
|
|
|
return nil, errors.New("invalid record type")
|
|
|
|
}
|
|
|
|
|
|
|
|
if dec.Len() == 0 {
|
|
|
|
return markers, nil
|
|
|
|
}
|
|
|
|
for len(dec.B) > 0 && dec.Err() == nil {
|
|
|
|
ref := chunks.HeadSeriesRef(dec.Be64())
|
|
|
|
mmapRef := chunks.ChunkDiskMapperRef(dec.Be64())
|
|
|
|
markers = append(markers, RefMmapMarker{
|
|
|
|
Ref: ref,
|
|
|
|
MmapRef: mmapRef,
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
if dec.Err() != nil {
|
|
|
|
return nil, errors.Wrapf(dec.Err(), "decode error after %d mmap markers", len(markers))
|
|
|
|
}
|
|
|
|
if len(dec.B) > 0 {
|
|
|
|
return nil, errors.Errorf("unexpected %d bytes left in entry", len(dec.B))
|
|
|
|
}
|
|
|
|
return markers, nil
|
|
|
|
}
|
|
|
|
|
2022-08-29 05:08:36 -07:00
|
|
|
func (d *Decoder) HistogramSamples(rec []byte, histograms []RefHistogramSample) ([]RefHistogramSample, error) {
|
2021-08-11 05:08:48 -07:00
|
|
|
dec := encoding.Decbuf{B: rec}
|
|
|
|
t := Type(dec.Byte())
|
2022-08-29 05:08:36 -07:00
|
|
|
if t != HistogramSamples {
|
2021-08-11 05:08:48 -07:00
|
|
|
return nil, errors.New("invalid record type")
|
|
|
|
}
|
|
|
|
if dec.Len() == 0 {
|
Style cleanup of all the changes in sparsehistogram so far
A lot of this code was hacked together, literally during a
hackathon. This commit intends not to change the code substantially,
but just make the code obey the usual style practices.
A (possibly incomplete) list of areas:
* Generally address linter warnings.
* The `pgk` directory is deprecated as per dev-summit. No new packages should
be added to it. I moved the new `pkg/histogram` package to `model`
anticipating what's proposed in #9478.
* Make the naming of the Sparse Histogram more consistent. Including
abbreviations, there were just too many names for it: SparseHistogram,
Histogram, Histo, hist, his, shs, h. The idea is to call it "Histogram" in
general. Only add "Sparse" if it is needed to avoid confusion with
conventional Histograms (which is rare because the TSDB really has no notion
of conventional Histograms). Use abbreviations only in local scope, and then
really abbreviate (not just removing three out of seven letters like in
"Histo"). This is in the spirit of
https://github.com/golang/go/wiki/CodeReviewComments#variable-names
* Several other minor name changes.
* A lot of formatting of doc comments. For one, following
https://github.com/golang/go/wiki/CodeReviewComments#comment-sentences
, but also layout question, anticipating how things will look like
when rendered by `godoc` (even where `godoc` doesn't render them
right now because they are for unexported types or not a doc comment
at all but just a normal code comment - consistency is queen!).
* Re-enabled `TestQueryLog` and `TestEndopints` (they pass now,
leaving them disabled was presumably an oversight).
* Bucket iterator for histogram.Histogram is now created with a
method.
* HistogramChunk.iterator now allows iterator recycling. (I think
@dieterbe only commented it out because he was confused by the
question in the comment.)
* HistogramAppender.Append panics now because we decided to treat
staleness marker differently.
Signed-off-by: beorn7 <beorn@grafana.com>
2021-10-09 06:57:07 -07:00
|
|
|
return histograms, nil
|
2021-08-11 05:08:48 -07:00
|
|
|
}
|
|
|
|
var (
|
|
|
|
baseRef = dec.Be64()
|
|
|
|
baseTime = dec.Be64int64()
|
|
|
|
)
|
|
|
|
for len(dec.B) > 0 && dec.Err() == nil {
|
|
|
|
dref := dec.Varint64()
|
|
|
|
dtime := dec.Varint64()
|
|
|
|
|
2022-08-29 05:08:36 -07:00
|
|
|
rh := RefHistogramSample{
|
2021-11-17 10:57:31 -08:00
|
|
|
Ref: chunks.HeadSeriesRef(baseRef + uint64(dref)),
|
2021-08-11 05:08:48 -07:00
|
|
|
T: baseTime + dtime,
|
2022-12-28 00:55:07 -08:00
|
|
|
H: &histogram.Histogram{},
|
2021-08-11 05:08:48 -07:00
|
|
|
}
|
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
DecodeHistogram(&dec, rh.H)
|
|
|
|
histograms = append(histograms, rh)
|
|
|
|
}
|
2023-01-04 01:58:18 -08:00
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
if dec.Err() != nil {
|
|
|
|
return nil, errors.Wrapf(dec.Err(), "decode error after %d histograms", len(histograms))
|
|
|
|
}
|
|
|
|
if len(dec.B) > 0 {
|
|
|
|
return nil, errors.Errorf("unexpected %d bytes left in entry", len(dec.B))
|
|
|
|
}
|
|
|
|
return histograms, nil
|
|
|
|
}
|
2021-08-11 05:08:48 -07:00
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
// DecodeHistogram decodes a Histogram from a byte slice.
|
|
|
|
func DecodeHistogram(buf *encoding.Decbuf, h *histogram.Histogram) {
|
|
|
|
h.CounterResetHint = histogram.CounterResetHint(buf.Byte())
|
2021-08-11 05:08:48 -07:00
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
h.Schema = int32(buf.Varint64())
|
|
|
|
h.ZeroThreshold = math.Float64frombits(buf.Be64())
|
2021-08-11 05:08:48 -07:00
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
h.ZeroCount = buf.Uvarint64()
|
|
|
|
h.Count = buf.Uvarint64()
|
|
|
|
h.Sum = math.Float64frombits(buf.Be64())
|
2021-08-11 05:08:48 -07:00
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
l := buf.Uvarint()
|
|
|
|
if l > 0 {
|
|
|
|
h.PositiveSpans = make([]histogram.Span, l)
|
|
|
|
}
|
|
|
|
for i := range h.PositiveSpans {
|
|
|
|
h.PositiveSpans[i].Offset = int32(buf.Varint64())
|
|
|
|
h.PositiveSpans[i].Length = buf.Uvarint32()
|
|
|
|
}
|
2021-08-11 05:08:48 -07:00
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
l = buf.Uvarint()
|
|
|
|
if l > 0 {
|
|
|
|
h.NegativeSpans = make([]histogram.Span, l)
|
|
|
|
}
|
|
|
|
for i := range h.NegativeSpans {
|
|
|
|
h.NegativeSpans[i].Offset = int32(buf.Varint64())
|
|
|
|
h.NegativeSpans[i].Length = buf.Uvarint32()
|
|
|
|
}
|
2021-08-11 05:08:48 -07:00
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
l = buf.Uvarint()
|
|
|
|
if l > 0 {
|
|
|
|
h.PositiveBuckets = make([]int64, l)
|
|
|
|
}
|
|
|
|
for i := range h.PositiveBuckets {
|
|
|
|
h.PositiveBuckets[i] = buf.Varint64()
|
2021-08-11 05:08:48 -07:00
|
|
|
}
|
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
l = buf.Uvarint()
|
|
|
|
if l > 0 {
|
|
|
|
h.NegativeBuckets = make([]int64, l)
|
2021-08-11 05:08:48 -07:00
|
|
|
}
|
2023-07-05 02:44:13 -07:00
|
|
|
for i := range h.NegativeBuckets {
|
|
|
|
h.NegativeBuckets[i] = buf.Varint64()
|
2021-08-11 05:08:48 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-12-28 00:55:07 -08:00
|
|
|
func (d *Decoder) FloatHistogramSamples(rec []byte, histograms []RefFloatHistogramSample) ([]RefFloatHistogramSample, error) {
|
|
|
|
dec := encoding.Decbuf{B: rec}
|
|
|
|
t := Type(dec.Byte())
|
|
|
|
if t != FloatHistogramSamples {
|
|
|
|
return nil, errors.New("invalid record type")
|
|
|
|
}
|
|
|
|
if dec.Len() == 0 {
|
|
|
|
return histograms, nil
|
|
|
|
}
|
|
|
|
var (
|
|
|
|
baseRef = dec.Be64()
|
|
|
|
baseTime = dec.Be64int64()
|
|
|
|
)
|
|
|
|
for len(dec.B) > 0 && dec.Err() == nil {
|
|
|
|
dref := dec.Varint64()
|
|
|
|
dtime := dec.Varint64()
|
|
|
|
|
|
|
|
rh := RefFloatHistogramSample{
|
|
|
|
Ref: chunks.HeadSeriesRef(baseRef + uint64(dref)),
|
|
|
|
T: baseTime + dtime,
|
|
|
|
FH: &histogram.FloatHistogram{},
|
|
|
|
}
|
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
DecodeFloatHistogram(&dec, rh.FH)
|
|
|
|
histograms = append(histograms, rh)
|
|
|
|
}
|
2023-01-04 01:58:18 -08:00
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
if dec.Err() != nil {
|
|
|
|
return nil, errors.Wrapf(dec.Err(), "decode error after %d histograms", len(histograms))
|
|
|
|
}
|
|
|
|
if len(dec.B) > 0 {
|
|
|
|
return nil, errors.Errorf("unexpected %d bytes left in entry", len(dec.B))
|
|
|
|
}
|
|
|
|
return histograms, nil
|
|
|
|
}
|
2022-12-28 00:55:07 -08:00
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
// Decode decodes a Histogram from a byte slice.
|
|
|
|
func DecodeFloatHistogram(buf *encoding.Decbuf, fh *histogram.FloatHistogram) {
|
|
|
|
fh.CounterResetHint = histogram.CounterResetHint(buf.Byte())
|
2022-12-28 00:55:07 -08:00
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
fh.Schema = int32(buf.Varint64())
|
|
|
|
fh.ZeroThreshold = buf.Be64Float64()
|
2022-12-28 00:55:07 -08:00
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
fh.ZeroCount = buf.Be64Float64()
|
|
|
|
fh.Count = buf.Be64Float64()
|
|
|
|
fh.Sum = buf.Be64Float64()
|
2022-12-28 00:55:07 -08:00
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
l := buf.Uvarint()
|
|
|
|
if l > 0 {
|
|
|
|
fh.PositiveSpans = make([]histogram.Span, l)
|
|
|
|
}
|
|
|
|
for i := range fh.PositiveSpans {
|
|
|
|
fh.PositiveSpans[i].Offset = int32(buf.Varint64())
|
|
|
|
fh.PositiveSpans[i].Length = buf.Uvarint32()
|
|
|
|
}
|
2022-12-28 00:55:07 -08:00
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
l = buf.Uvarint()
|
|
|
|
if l > 0 {
|
|
|
|
fh.NegativeSpans = make([]histogram.Span, l)
|
|
|
|
}
|
|
|
|
for i := range fh.NegativeSpans {
|
|
|
|
fh.NegativeSpans[i].Offset = int32(buf.Varint64())
|
|
|
|
fh.NegativeSpans[i].Length = buf.Uvarint32()
|
|
|
|
}
|
2022-12-28 00:55:07 -08:00
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
l = buf.Uvarint()
|
|
|
|
if l > 0 {
|
|
|
|
fh.PositiveBuckets = make([]float64, l)
|
|
|
|
}
|
|
|
|
for i := range fh.PositiveBuckets {
|
|
|
|
fh.PositiveBuckets[i] = buf.Be64Float64()
|
2022-12-28 00:55:07 -08:00
|
|
|
}
|
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
l = buf.Uvarint()
|
|
|
|
if l > 0 {
|
|
|
|
fh.NegativeBuckets = make([]float64, l)
|
2022-12-28 00:55:07 -08:00
|
|
|
}
|
2023-07-05 02:44:13 -07:00
|
|
|
for i := range fh.NegativeBuckets {
|
|
|
|
fh.NegativeBuckets[i] = buf.Be64Float64()
|
2022-12-28 00:55:07 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-09-19 02:15:41 -07:00
|
|
|
// Encoder encodes series, sample, and tombstones records.
|
2018-05-17 06:02:47 -07:00
|
|
|
// The zero value is ready to use.
|
2021-10-22 01:06:44 -07:00
|
|
|
type Encoder struct{}
|
2018-05-17 06:02:47 -07:00
|
|
|
|
|
|
|
// Series appends the encoded series to b and returns the resulting slice.
|
2019-09-19 02:15:41 -07:00
|
|
|
func (e *Encoder) Series(series []RefSeries, b []byte) []byte {
|
2019-02-22 09:11:11 -08:00
|
|
|
buf := encoding.Encbuf{B: b}
|
2019-09-19 02:15:41 -07:00
|
|
|
buf.PutByte(byte(Series))
|
2018-05-17 06:02:47 -07:00
|
|
|
|
|
|
|
for _, s := range series {
|
2021-11-06 03:10:04 -07:00
|
|
|
buf.PutBE64(uint64(s.Ref))
|
2022-07-26 07:42:00 -07:00
|
|
|
EncodeLabels(&buf, s.Labels)
|
2018-05-17 06:02:47 -07:00
|
|
|
}
|
2019-02-22 09:11:11 -08:00
|
|
|
return buf.Get()
|
2018-05-17 06:02:47 -07:00
|
|
|
}
|
|
|
|
|
2022-07-19 01:58:52 -07:00
|
|
|
// Metadata appends the encoded metadata to b and returns the resulting slice.
|
|
|
|
func (e *Encoder) Metadata(metadata []RefMetadata, b []byte) []byte {
|
|
|
|
buf := encoding.Encbuf{B: b}
|
|
|
|
buf.PutByte(byte(Metadata))
|
|
|
|
|
|
|
|
for _, m := range metadata {
|
|
|
|
buf.PutUvarint64(uint64(m.Ref))
|
|
|
|
|
|
|
|
buf.PutByte(m.Type)
|
|
|
|
|
|
|
|
buf.PutUvarint(2) // num_fields: We currently have two more metadata fields, UNIT and HELP.
|
|
|
|
buf.PutUvarintStr(unitMetaName)
|
|
|
|
buf.PutUvarintStr(m.Unit)
|
|
|
|
buf.PutUvarintStr(helpMetaName)
|
|
|
|
buf.PutUvarintStr(m.Help)
|
2018-05-17 06:02:47 -07:00
|
|
|
}
|
2022-07-19 01:58:52 -07:00
|
|
|
|
2019-02-22 09:11:11 -08:00
|
|
|
return buf.Get()
|
2018-05-17 06:02:47 -07:00
|
|
|
}
|
|
|
|
|
2022-07-26 07:42:00 -07:00
|
|
|
// EncodeLabels encodes the contents of labels into buf.
|
|
|
|
func EncodeLabels(buf *encoding.Encbuf, lbls labels.Labels) {
|
2022-06-28 08:02:08 -07:00
|
|
|
// TODO: reconsider if this function could be pushed down into labels.Labels to be more efficient.
|
|
|
|
buf.PutUvarint(lbls.Len())
|
2022-07-26 07:42:00 -07:00
|
|
|
|
2022-06-28 08:02:08 -07:00
|
|
|
lbls.Range(func(l labels.Label) {
|
2022-07-26 07:42:00 -07:00
|
|
|
buf.PutUvarintStr(l.Name)
|
|
|
|
buf.PutUvarintStr(l.Value)
|
2022-06-28 08:02:08 -07:00
|
|
|
})
|
2022-07-26 07:42:00 -07:00
|
|
|
}
|
|
|
|
|
2018-05-17 06:02:47 -07:00
|
|
|
// Samples appends the encoded samples to b and returns the resulting slice.
|
2019-09-19 02:15:41 -07:00
|
|
|
func (e *Encoder) Samples(samples []RefSample, b []byte) []byte {
|
2019-02-22 09:11:11 -08:00
|
|
|
buf := encoding.Encbuf{B: b}
|
2019-09-19 02:15:41 -07:00
|
|
|
buf.PutByte(byte(Samples))
|
2018-05-17 06:02:47 -07:00
|
|
|
|
|
|
|
if len(samples) == 0 {
|
2019-02-22 09:11:11 -08:00
|
|
|
return buf.Get()
|
2018-05-17 06:02:47 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
// Store base timestamp and base reference number of first sample.
|
|
|
|
// All samples encode their timestamp and ref as delta to those.
|
|
|
|
first := samples[0]
|
|
|
|
|
2021-11-06 03:10:04 -07:00
|
|
|
buf.PutBE64(uint64(first.Ref))
|
2019-02-22 09:11:11 -08:00
|
|
|
buf.PutBE64int64(first.T)
|
2018-05-17 06:02:47 -07:00
|
|
|
|
|
|
|
for _, s := range samples {
|
2019-02-22 09:11:11 -08:00
|
|
|
buf.PutVarint64(int64(s.Ref) - int64(first.Ref))
|
|
|
|
buf.PutVarint64(s.T - first.T)
|
|
|
|
buf.PutBE64(math.Float64bits(s.V))
|
2018-05-17 06:02:47 -07:00
|
|
|
}
|
2019-02-22 09:11:11 -08:00
|
|
|
return buf.Get()
|
2018-05-17 06:02:47 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
// Tombstones appends the encoded tombstones to b and returns the resulting slice.
|
2019-09-19 02:15:41 -07:00
|
|
|
func (e *Encoder) Tombstones(tstones []tombstones.Stone, b []byte) []byte {
|
2019-02-22 09:11:11 -08:00
|
|
|
buf := encoding.Encbuf{B: b}
|
2019-09-19 02:15:41 -07:00
|
|
|
buf.PutByte(byte(Tombstones))
|
2018-05-17 06:02:47 -07:00
|
|
|
|
|
|
|
for _, s := range tstones {
|
2019-09-19 02:15:41 -07:00
|
|
|
for _, iv := range s.Intervals {
|
2021-11-06 03:10:04 -07:00
|
|
|
buf.PutBE64(uint64(s.Ref))
|
2019-02-22 09:11:11 -08:00
|
|
|
buf.PutVarint64(iv.Mint)
|
|
|
|
buf.PutVarint64(iv.Maxt)
|
2018-05-17 06:02:47 -07:00
|
|
|
}
|
|
|
|
}
|
2019-02-22 09:11:11 -08:00
|
|
|
return buf.Get()
|
2018-05-17 06:02:47 -07:00
|
|
|
}
|
2021-05-06 13:53:52 -07:00
|
|
|
|
|
|
|
func (e *Encoder) Exemplars(exemplars []RefExemplar, b []byte) []byte {
|
|
|
|
buf := encoding.Encbuf{B: b}
|
|
|
|
buf.PutByte(byte(Exemplars))
|
|
|
|
|
|
|
|
if len(exemplars) == 0 {
|
|
|
|
return buf.Get()
|
|
|
|
}
|
|
|
|
|
2021-08-30 07:04:38 -07:00
|
|
|
e.EncodeExemplarsIntoBuffer(exemplars, &buf)
|
|
|
|
|
|
|
|
return buf.Get()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (e *Encoder) EncodeExemplarsIntoBuffer(exemplars []RefExemplar, buf *encoding.Encbuf) {
|
2021-05-06 13:53:52 -07:00
|
|
|
// Store base timestamp and base reference number of first sample.
|
|
|
|
// All samples encode their timestamp and ref as delta to those.
|
|
|
|
first := exemplars[0]
|
|
|
|
|
2021-11-06 03:10:04 -07:00
|
|
|
buf.PutBE64(uint64(first.Ref))
|
2021-05-06 13:53:52 -07:00
|
|
|
buf.PutBE64int64(first.T)
|
|
|
|
|
|
|
|
for _, ex := range exemplars {
|
|
|
|
buf.PutVarint64(int64(ex.Ref) - int64(first.Ref))
|
|
|
|
buf.PutVarint64(ex.T - first.T)
|
|
|
|
buf.PutBE64(math.Float64bits(ex.V))
|
2022-07-26 07:42:00 -07:00
|
|
|
EncodeLabels(buf, ex.Labels)
|
2021-05-06 13:53:52 -07:00
|
|
|
}
|
|
|
|
}
|
2021-08-11 05:08:48 -07:00
|
|
|
|
2022-09-20 10:05:50 -07:00
|
|
|
func (e *Encoder) MmapMarkers(markers []RefMmapMarker, b []byte) []byte {
|
|
|
|
buf := encoding.Encbuf{B: b}
|
|
|
|
buf.PutByte(byte(MmapMarkers))
|
|
|
|
|
|
|
|
for _, s := range markers {
|
|
|
|
buf.PutBE64(uint64(s.Ref))
|
|
|
|
buf.PutBE64(uint64(s.MmapRef))
|
|
|
|
}
|
|
|
|
|
|
|
|
return buf.Get()
|
|
|
|
}
|
2022-10-05 13:14:49 -07:00
|
|
|
|
2022-08-29 05:08:36 -07:00
|
|
|
func (e *Encoder) HistogramSamples(histograms []RefHistogramSample, b []byte) []byte {
|
2021-08-11 05:08:48 -07:00
|
|
|
buf := encoding.Encbuf{B: b}
|
2022-08-29 05:08:36 -07:00
|
|
|
buf.PutByte(byte(HistogramSamples))
|
2021-08-11 05:08:48 -07:00
|
|
|
|
Style cleanup of all the changes in sparsehistogram so far
A lot of this code was hacked together, literally during a
hackathon. This commit intends not to change the code substantially,
but just make the code obey the usual style practices.
A (possibly incomplete) list of areas:
* Generally address linter warnings.
* The `pgk` directory is deprecated as per dev-summit. No new packages should
be added to it. I moved the new `pkg/histogram` package to `model`
anticipating what's proposed in #9478.
* Make the naming of the Sparse Histogram more consistent. Including
abbreviations, there were just too many names for it: SparseHistogram,
Histogram, Histo, hist, his, shs, h. The idea is to call it "Histogram" in
general. Only add "Sparse" if it is needed to avoid confusion with
conventional Histograms (which is rare because the TSDB really has no notion
of conventional Histograms). Use abbreviations only in local scope, and then
really abbreviate (not just removing three out of seven letters like in
"Histo"). This is in the spirit of
https://github.com/golang/go/wiki/CodeReviewComments#variable-names
* Several other minor name changes.
* A lot of formatting of doc comments. For one, following
https://github.com/golang/go/wiki/CodeReviewComments#comment-sentences
, but also layout question, anticipating how things will look like
when rendered by `godoc` (even where `godoc` doesn't render them
right now because they are for unexported types or not a doc comment
at all but just a normal code comment - consistency is queen!).
* Re-enabled `TestQueryLog` and `TestEndopints` (they pass now,
leaving them disabled was presumably an oversight).
* Bucket iterator for histogram.Histogram is now created with a
method.
* HistogramChunk.iterator now allows iterator recycling. (I think
@dieterbe only commented it out because he was confused by the
question in the comment.)
* HistogramAppender.Append panics now because we decided to treat
staleness marker differently.
Signed-off-by: beorn7 <beorn@grafana.com>
2021-10-09 06:57:07 -07:00
|
|
|
if len(histograms) == 0 {
|
2021-08-11 05:08:48 -07:00
|
|
|
return buf.Get()
|
|
|
|
}
|
|
|
|
|
|
|
|
// Store base timestamp and base reference number of first histogram.
|
|
|
|
// All histograms encode their timestamp and ref as delta to those.
|
Style cleanup of all the changes in sparsehistogram so far
A lot of this code was hacked together, literally during a
hackathon. This commit intends not to change the code substantially,
but just make the code obey the usual style practices.
A (possibly incomplete) list of areas:
* Generally address linter warnings.
* The `pgk` directory is deprecated as per dev-summit. No new packages should
be added to it. I moved the new `pkg/histogram` package to `model`
anticipating what's proposed in #9478.
* Make the naming of the Sparse Histogram more consistent. Including
abbreviations, there were just too many names for it: SparseHistogram,
Histogram, Histo, hist, his, shs, h. The idea is to call it "Histogram" in
general. Only add "Sparse" if it is needed to avoid confusion with
conventional Histograms (which is rare because the TSDB really has no notion
of conventional Histograms). Use abbreviations only in local scope, and then
really abbreviate (not just removing three out of seven letters like in
"Histo"). This is in the spirit of
https://github.com/golang/go/wiki/CodeReviewComments#variable-names
* Several other minor name changes.
* A lot of formatting of doc comments. For one, following
https://github.com/golang/go/wiki/CodeReviewComments#comment-sentences
, but also layout question, anticipating how things will look like
when rendered by `godoc` (even where `godoc` doesn't render them
right now because they are for unexported types or not a doc comment
at all but just a normal code comment - consistency is queen!).
* Re-enabled `TestQueryLog` and `TestEndopints` (they pass now,
leaving them disabled was presumably an oversight).
* Bucket iterator for histogram.Histogram is now created with a
method.
* HistogramChunk.iterator now allows iterator recycling. (I think
@dieterbe only commented it out because he was confused by the
question in the comment.)
* HistogramAppender.Append panics now because we decided to treat
staleness marker differently.
Signed-off-by: beorn7 <beorn@grafana.com>
2021-10-09 06:57:07 -07:00
|
|
|
first := histograms[0]
|
2021-11-17 10:57:31 -08:00
|
|
|
buf.PutBE64(uint64(first.Ref))
|
2021-08-11 05:08:48 -07:00
|
|
|
buf.PutBE64int64(first.T)
|
|
|
|
|
Style cleanup of all the changes in sparsehistogram so far
A lot of this code was hacked together, literally during a
hackathon. This commit intends not to change the code substantially,
but just make the code obey the usual style practices.
A (possibly incomplete) list of areas:
* Generally address linter warnings.
* The `pgk` directory is deprecated as per dev-summit. No new packages should
be added to it. I moved the new `pkg/histogram` package to `model`
anticipating what's proposed in #9478.
* Make the naming of the Sparse Histogram more consistent. Including
abbreviations, there were just too many names for it: SparseHistogram,
Histogram, Histo, hist, his, shs, h. The idea is to call it "Histogram" in
general. Only add "Sparse" if it is needed to avoid confusion with
conventional Histograms (which is rare because the TSDB really has no notion
of conventional Histograms). Use abbreviations only in local scope, and then
really abbreviate (not just removing three out of seven letters like in
"Histo"). This is in the spirit of
https://github.com/golang/go/wiki/CodeReviewComments#variable-names
* Several other minor name changes.
* A lot of formatting of doc comments. For one, following
https://github.com/golang/go/wiki/CodeReviewComments#comment-sentences
, but also layout question, anticipating how things will look like
when rendered by `godoc` (even where `godoc` doesn't render them
right now because they are for unexported types or not a doc comment
at all but just a normal code comment - consistency is queen!).
* Re-enabled `TestQueryLog` and `TestEndopints` (they pass now,
leaving them disabled was presumably an oversight).
* Bucket iterator for histogram.Histogram is now created with a
method.
* HistogramChunk.iterator now allows iterator recycling. (I think
@dieterbe only commented it out because he was confused by the
question in the comment.)
* HistogramAppender.Append panics now because we decided to treat
staleness marker differently.
Signed-off-by: beorn7 <beorn@grafana.com>
2021-10-09 06:57:07 -07:00
|
|
|
for _, h := range histograms {
|
2021-08-11 05:08:48 -07:00
|
|
|
buf.PutVarint64(int64(h.Ref) - int64(first.Ref))
|
|
|
|
buf.PutVarint64(h.T - first.T)
|
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
EncodeHistogram(&buf, h.H)
|
|
|
|
}
|
2023-01-04 01:58:18 -08:00
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
return buf.Get()
|
|
|
|
}
|
2021-08-11 05:08:48 -07:00
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
// EncodeHistogram encodes a Histogram into a byte slice.
|
|
|
|
func EncodeHistogram(buf *encoding.Encbuf, h *histogram.Histogram) {
|
|
|
|
buf.PutByte(byte(h.CounterResetHint))
|
2021-08-11 05:08:48 -07:00
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
buf.PutVarint64(int64(h.Schema))
|
|
|
|
buf.PutBE64(math.Float64bits(h.ZeroThreshold))
|
2021-08-11 05:08:48 -07:00
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
buf.PutUvarint64(h.ZeroCount)
|
|
|
|
buf.PutUvarint64(h.Count)
|
|
|
|
buf.PutBE64(math.Float64bits(h.Sum))
|
2021-08-11 05:08:48 -07:00
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
buf.PutUvarint(len(h.PositiveSpans))
|
|
|
|
for _, s := range h.PositiveSpans {
|
|
|
|
buf.PutVarint64(int64(s.Offset))
|
|
|
|
buf.PutUvarint32(s.Length)
|
|
|
|
}
|
2021-08-11 05:08:48 -07:00
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
buf.PutUvarint(len(h.NegativeSpans))
|
|
|
|
for _, s := range h.NegativeSpans {
|
|
|
|
buf.PutVarint64(int64(s.Offset))
|
|
|
|
buf.PutUvarint32(s.Length)
|
2021-08-11 05:08:48 -07:00
|
|
|
}
|
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
buf.PutUvarint(len(h.PositiveBuckets))
|
|
|
|
for _, b := range h.PositiveBuckets {
|
|
|
|
buf.PutVarint64(b)
|
|
|
|
}
|
|
|
|
|
|
|
|
buf.PutUvarint(len(h.NegativeBuckets))
|
|
|
|
for _, b := range h.NegativeBuckets {
|
|
|
|
buf.PutVarint64(b)
|
|
|
|
}
|
2021-08-11 05:08:48 -07:00
|
|
|
}
|
2022-12-28 00:55:07 -08:00
|
|
|
|
|
|
|
func (e *Encoder) FloatHistogramSamples(histograms []RefFloatHistogramSample, b []byte) []byte {
|
|
|
|
buf := encoding.Encbuf{B: b}
|
|
|
|
buf.PutByte(byte(FloatHistogramSamples))
|
|
|
|
|
|
|
|
if len(histograms) == 0 {
|
|
|
|
return buf.Get()
|
|
|
|
}
|
|
|
|
|
|
|
|
// Store base timestamp and base reference number of first histogram.
|
|
|
|
// All histograms encode their timestamp and ref as delta to those.
|
|
|
|
first := histograms[0]
|
|
|
|
buf.PutBE64(uint64(first.Ref))
|
|
|
|
buf.PutBE64int64(first.T)
|
|
|
|
|
|
|
|
for _, h := range histograms {
|
|
|
|
buf.PutVarint64(int64(h.Ref) - int64(first.Ref))
|
|
|
|
buf.PutVarint64(h.T - first.T)
|
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
EncodeFloatHistogram(&buf, h.FH)
|
|
|
|
}
|
|
|
|
|
|
|
|
return buf.Get()
|
|
|
|
}
|
2023-01-04 01:58:18 -08:00
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
// Encode encodes the Float Histogram into a byte slice.
|
|
|
|
func EncodeFloatHistogram(buf *encoding.Encbuf, h *histogram.FloatHistogram) {
|
|
|
|
buf.PutByte(byte(h.CounterResetHint))
|
2022-12-28 00:55:07 -08:00
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
buf.PutVarint64(int64(h.Schema))
|
|
|
|
buf.PutBEFloat64(h.ZeroThreshold)
|
2022-12-28 00:55:07 -08:00
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
buf.PutBEFloat64(h.ZeroCount)
|
|
|
|
buf.PutBEFloat64(h.Count)
|
|
|
|
buf.PutBEFloat64(h.Sum)
|
2022-12-28 00:55:07 -08:00
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
buf.PutUvarint(len(h.PositiveSpans))
|
|
|
|
for _, s := range h.PositiveSpans {
|
|
|
|
buf.PutVarint64(int64(s.Offset))
|
|
|
|
buf.PutUvarint32(s.Length)
|
|
|
|
}
|
2022-12-28 00:55:07 -08:00
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
buf.PutUvarint(len(h.NegativeSpans))
|
|
|
|
for _, s := range h.NegativeSpans {
|
|
|
|
buf.PutVarint64(int64(s.Offset))
|
|
|
|
buf.PutUvarint32(s.Length)
|
|
|
|
}
|
2022-12-28 00:55:07 -08:00
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
buf.PutUvarint(len(h.PositiveBuckets))
|
|
|
|
for _, b := range h.PositiveBuckets {
|
|
|
|
buf.PutBEFloat64(b)
|
2022-12-28 00:55:07 -08:00
|
|
|
}
|
|
|
|
|
2023-07-05 02:44:13 -07:00
|
|
|
buf.PutUvarint(len(h.NegativeBuckets))
|
|
|
|
for _, b := range h.NegativeBuckets {
|
|
|
|
buf.PutBEFloat64(b)
|
|
|
|
}
|
2022-12-28 00:55:07 -08:00
|
|
|
}
|