mirror of
https://github.com/prometheus/prometheus.git
synced 2024-12-25 05:34:05 -08:00
tsdb: Add support for ingestion of out-of-order native histogram samples (#14546)
Add support for ingesting OOO native histograms * Add flag for enabling and disabling OOO native histogram ingestion * Update OOO querying tests to include native histogram samples * Add OOO head tests * Add test for OOO native histogram counter reset headers Signed-off-by: Carrie Edwards <edwrdscarrie@gmail.com> Signed-off-by: György Krajcsovits <gyorgy.krajcsovits@grafana.com> Co-authored by: Carrie Edwards <edwrdscarrie@gmail.com> Co-authored by: Jeanette Tan <jeanette.tan@grafana.com> Co-authored by: György Krajcsovits <gyorgy.krajcsovits@grafana.com> Co-authored by: Fiona Liao <fiona.liao@grafana.com>
This commit is contained in:
parent
919dc0cbc6
commit
14e3c05ce8
|
@ -230,6 +230,9 @@ func (c *flagConfig) setFeatureListOptions(logger log.Logger) error {
|
||||||
config.DefaultConfig.GlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
|
config.DefaultConfig.GlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
|
||||||
config.DefaultGlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
|
config.DefaultGlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
|
||||||
level.Info(logger).Log("msg", "Experimental native histogram support enabled. Changed default scrape_protocols to prefer PrometheusProto format.", "global.scrape_protocols", fmt.Sprintf("%v", config.DefaultGlobalConfig.ScrapeProtocols))
|
level.Info(logger).Log("msg", "Experimental native histogram support enabled. Changed default scrape_protocols to prefer PrometheusProto format.", "global.scrape_protocols", fmt.Sprintf("%v", config.DefaultGlobalConfig.ScrapeProtocols))
|
||||||
|
case "ooo-native-histograms":
|
||||||
|
c.tsdb.EnableOOONativeHistograms = true
|
||||||
|
level.Info(logger).Log("msg", "Experimental out-of-order native histogram ingestion enabled. This will only take effect if OutOfOrderTimeWindow is > 0 and if EnableNativeHistograms = true")
|
||||||
case "created-timestamp-zero-ingestion":
|
case "created-timestamp-zero-ingestion":
|
||||||
c.scrape.EnableCreatedTimestampZeroIngestion = true
|
c.scrape.EnableCreatedTimestampZeroIngestion = true
|
||||||
// Change relevant global variables. Hacky, but it's hard to pass a new option or default to unmarshallers.
|
// Change relevant global variables. Hacky, but it's hard to pass a new option or default to unmarshallers.
|
||||||
|
@ -1735,6 +1738,7 @@ type tsdbOptions struct {
|
||||||
EnableNativeHistograms bool
|
EnableNativeHistograms bool
|
||||||
EnableDelayedCompaction bool
|
EnableDelayedCompaction bool
|
||||||
EnableOverlappingCompaction bool
|
EnableOverlappingCompaction bool
|
||||||
|
EnableOOONativeHistograms bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func (opts tsdbOptions) ToTSDBOptions() tsdb.Options {
|
func (opts tsdbOptions) ToTSDBOptions() tsdb.Options {
|
||||||
|
@ -1754,6 +1758,7 @@ func (opts tsdbOptions) ToTSDBOptions() tsdb.Options {
|
||||||
MaxExemplars: opts.MaxExemplars,
|
MaxExemplars: opts.MaxExemplars,
|
||||||
EnableMemorySnapshotOnShutdown: opts.EnableMemorySnapshotOnShutdown,
|
EnableMemorySnapshotOnShutdown: opts.EnableMemorySnapshotOnShutdown,
|
||||||
EnableNativeHistograms: opts.EnableNativeHistograms,
|
EnableNativeHistograms: opts.EnableNativeHistograms,
|
||||||
|
EnableOOONativeHistograms: opts.EnableOOONativeHistograms,
|
||||||
OutOfOrderTimeWindow: opts.OutOfOrderTimeWindow,
|
OutOfOrderTimeWindow: opts.OutOfOrderTimeWindow,
|
||||||
EnableDelayedCompaction: opts.EnableDelayedCompaction,
|
EnableDelayedCompaction: opts.EnableDelayedCompaction,
|
||||||
EnableOverlappingCompaction: opts.EnableOverlappingCompaction,
|
EnableOverlappingCompaction: opts.EnableOverlappingCompaction,
|
||||||
|
|
|
@ -43,6 +43,7 @@ var (
|
||||||
ErrExemplarLabelLength = fmt.Errorf("label length for exemplar exceeds maximum of %d UTF-8 characters", exemplar.ExemplarMaxLabelSetLength)
|
ErrExemplarLabelLength = fmt.Errorf("label length for exemplar exceeds maximum of %d UTF-8 characters", exemplar.ExemplarMaxLabelSetLength)
|
||||||
ErrExemplarsDisabled = fmt.Errorf("exemplar storage is disabled or max exemplars is less than or equal to 0")
|
ErrExemplarsDisabled = fmt.Errorf("exemplar storage is disabled or max exemplars is less than or equal to 0")
|
||||||
ErrNativeHistogramsDisabled = fmt.Errorf("native histograms are disabled")
|
ErrNativeHistogramsDisabled = fmt.Errorf("native histograms are disabled")
|
||||||
|
ErrOOONativeHistogramsDisabled = fmt.Errorf("out-of-order native histogram ingestion is disabled")
|
||||||
|
|
||||||
// ErrOutOfOrderCT indicates failed append of CT to the storage
|
// ErrOutOfOrderCT indicates failed append of CT to the storage
|
||||||
// due to CT being older the then newer sample.
|
// due to CT being older the then newer sample.
|
||||||
|
|
17
tsdb/db.go
17
tsdb/db.go
|
@ -173,6 +173,12 @@ type Options struct {
|
||||||
// EnableNativeHistograms enables the ingestion of native histograms.
|
// EnableNativeHistograms enables the ingestion of native histograms.
|
||||||
EnableNativeHistograms bool
|
EnableNativeHistograms bool
|
||||||
|
|
||||||
|
// EnableOOONativeHistograms enables the ingestion of OOO native histograms.
|
||||||
|
// It will only take effect if EnableNativeHistograms is set to true and the
|
||||||
|
// OutOfOrderTimeWindow is > 0. This flag will be removed after testing of
|
||||||
|
// OOO Native Histogram ingestion is complete.
|
||||||
|
EnableOOONativeHistograms bool
|
||||||
|
|
||||||
// OutOfOrderTimeWindow specifies how much out of order is allowed, if any.
|
// OutOfOrderTimeWindow specifies how much out of order is allowed, if any.
|
||||||
// This can change during run-time, so this value from here should only be used
|
// This can change during run-time, so this value from here should only be used
|
||||||
// while initialising.
|
// while initialising.
|
||||||
|
@ -948,6 +954,7 @@ func open(dir string, l log.Logger, r prometheus.Registerer, opts *Options, rngs
|
||||||
headOpts.MaxExemplars.Store(opts.MaxExemplars)
|
headOpts.MaxExemplars.Store(opts.MaxExemplars)
|
||||||
headOpts.EnableMemorySnapshotOnShutdown = opts.EnableMemorySnapshotOnShutdown
|
headOpts.EnableMemorySnapshotOnShutdown = opts.EnableMemorySnapshotOnShutdown
|
||||||
headOpts.EnableNativeHistograms.Store(opts.EnableNativeHistograms)
|
headOpts.EnableNativeHistograms.Store(opts.EnableNativeHistograms)
|
||||||
|
headOpts.EnableOOONativeHistograms.Store(opts.EnableOOONativeHistograms)
|
||||||
headOpts.OutOfOrderTimeWindow.Store(opts.OutOfOrderTimeWindow)
|
headOpts.OutOfOrderTimeWindow.Store(opts.OutOfOrderTimeWindow)
|
||||||
headOpts.OutOfOrderCapMax.Store(opts.OutOfOrderCapMax)
|
headOpts.OutOfOrderCapMax.Store(opts.OutOfOrderCapMax)
|
||||||
headOpts.EnableSharding = opts.EnableSharding
|
headOpts.EnableSharding = opts.EnableSharding
|
||||||
|
@ -1172,6 +1179,16 @@ func (db *DB) DisableNativeHistograms() {
|
||||||
db.head.DisableNativeHistograms()
|
db.head.DisableNativeHistograms()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// EnableOOONativeHistograms enables the ingestion of out-of-order native histograms.
|
||||||
|
func (db *DB) EnableOOONativeHistograms() {
|
||||||
|
db.head.EnableOOONativeHistograms()
|
||||||
|
}
|
||||||
|
|
||||||
|
// DisableOOONativeHistograms disables the ingestion of out-of-order native histograms.
|
||||||
|
func (db *DB) DisableOOONativeHistograms() {
|
||||||
|
db.head.DisableOOONativeHistograms()
|
||||||
|
}
|
||||||
|
|
||||||
// dbAppender wraps the DB's head appender and triggers compactions on commit
|
// dbAppender wraps the DB's head appender and triggers compactions on commit
|
||||||
// if necessary.
|
// if necessary.
|
||||||
type dbAppender struct {
|
type dbAppender struct {
|
||||||
|
|
1589
tsdb/db_test.go
1589
tsdb/db_test.go
File diff suppressed because it is too large
Load diff
15
tsdb/head.go
15
tsdb/head.go
|
@ -150,6 +150,11 @@ type HeadOptions struct {
|
||||||
// EnableNativeHistograms enables the ingestion of native histograms.
|
// EnableNativeHistograms enables the ingestion of native histograms.
|
||||||
EnableNativeHistograms atomic.Bool
|
EnableNativeHistograms atomic.Bool
|
||||||
|
|
||||||
|
// EnableOOONativeHistograms enables the ingestion of OOO native histograms.
|
||||||
|
// It will only take effect if EnableNativeHistograms is set to true and the
|
||||||
|
// OutOfOrderTimeWindow is > 0
|
||||||
|
EnableOOONativeHistograms atomic.Bool
|
||||||
|
|
||||||
// EnableCreatedTimestampZeroIngestion enables the ingestion of the created timestamp as a synthetic zero sample.
|
// EnableCreatedTimestampZeroIngestion enables the ingestion of the created timestamp as a synthetic zero sample.
|
||||||
// See: https://github.com/prometheus/proposals/blob/main/proposals/2023-06-13_created-timestamp.md
|
// See: https://github.com/prometheus/proposals/blob/main/proposals/2023-06-13_created-timestamp.md
|
||||||
EnableCreatedTimestampZeroIngestion bool
|
EnableCreatedTimestampZeroIngestion bool
|
||||||
|
@ -1018,6 +1023,16 @@ func (h *Head) DisableNativeHistograms() {
|
||||||
h.opts.EnableNativeHistograms.Store(false)
|
h.opts.EnableNativeHistograms.Store(false)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// EnableOOONativeHistograms enables the ingestion of out-of-order native histograms.
|
||||||
|
func (h *Head) EnableOOONativeHistograms() {
|
||||||
|
h.opts.EnableOOONativeHistograms.Store(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DisableOOONativeHistograms disables the ingestion of out-of-order native histograms.
|
||||||
|
func (h *Head) DisableOOONativeHistograms() {
|
||||||
|
h.opts.EnableOOONativeHistograms.Store(false)
|
||||||
|
}
|
||||||
|
|
||||||
// PostingsCardinalityStats returns highest cardinality stats by label and value names.
|
// PostingsCardinalityStats returns highest cardinality stats by label and value names.
|
||||||
func (h *Head) PostingsCardinalityStats(statsByLabelName string, limit int) *index.PostingsStats {
|
func (h *Head) PostingsCardinalityStats(statsByLabelName string, limit int) *index.PostingsStats {
|
||||||
cacheKey := statsByLabelName + ";" + strconv.Itoa(limit)
|
cacheKey := statsByLabelName + ";" + strconv.Itoa(limit)
|
||||||
|
|
|
@ -321,8 +321,8 @@ type headAppender struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *headAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64) (storage.SeriesRef, error) {
|
func (a *headAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64) (storage.SeriesRef, error) {
|
||||||
// For OOO inserts, this restriction is irrelevant and will be checked later once we confirm the sample is an in-order append.
|
// Fail fast if OOO is disabled and the sample is out of bounds.
|
||||||
// If OOO inserts are disabled, we may as well as check this as early as we can and avoid more work.
|
// Otherwise a full check will be done later to decide if the sample is in-order or out-of-order.
|
||||||
if a.oooTimeWindow == 0 && t < a.minValidTime {
|
if a.oooTimeWindow == 0 && t < a.minValidTime {
|
||||||
a.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeFloat).Inc()
|
a.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeFloat).Inc()
|
||||||
return 0, storage.ErrOutOfBounds
|
return 0, storage.ErrOutOfBounds
|
||||||
|
@ -493,46 +493,94 @@ func (s *memSeries) appendable(t int64, v float64, headMaxt, minValidTime, oooTi
|
||||||
return false, headMaxt - t, storage.ErrOutOfOrderSample
|
return false, headMaxt - t, storage.ErrOutOfOrderSample
|
||||||
}
|
}
|
||||||
|
|
||||||
// appendableHistogram checks whether the given histogram is valid for appending to the series.
|
// appendableHistogram checks whether the given histogram sample is valid for appending to the series. (if we return false and no error)
|
||||||
func (s *memSeries) appendableHistogram(t int64, h *histogram.Histogram) error {
|
// The sample belongs to the out of order chunk if we return true and no error.
|
||||||
if s.headChunks == nil {
|
// An error signifies the sample cannot be handled.
|
||||||
return nil
|
func (s *memSeries) appendableHistogram(t int64, h *histogram.Histogram, headMaxt, minValidTime, oooTimeWindow int64, oooHistogramsEnabled bool) (isOOO bool, oooDelta int64, err error) {
|
||||||
|
// Check if we can append in the in-order chunk.
|
||||||
|
if t >= minValidTime {
|
||||||
|
if s.headChunks == nil {
|
||||||
|
// The series has no sample and was freshly created.
|
||||||
|
return false, 0, nil
|
||||||
|
}
|
||||||
|
msMaxt := s.maxTime()
|
||||||
|
if t > msMaxt {
|
||||||
|
return false, 0, nil
|
||||||
|
}
|
||||||
|
if t == msMaxt {
|
||||||
|
// We are allowing exact duplicates as we can encounter them in valid cases
|
||||||
|
// like federation and erroring out at that time would be extremely noisy.
|
||||||
|
// This only checks against the latest in-order sample.
|
||||||
|
// The OOO headchunk has its own method to detect these duplicates.
|
||||||
|
if !h.Equals(s.lastHistogramValue) {
|
||||||
|
return false, 0, storage.ErrDuplicateSampleForTimestamp
|
||||||
|
}
|
||||||
|
// Sample is identical (ts + value) with most current (highest ts) sample in sampleBuf.
|
||||||
|
return false, 0, nil
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if t > s.headChunks.maxTime {
|
// The sample cannot go in the in-order chunk. Check if it can go in the out-of-order chunk.
|
||||||
return nil
|
if oooTimeWindow > 0 && t >= headMaxt-oooTimeWindow {
|
||||||
}
|
if !oooHistogramsEnabled {
|
||||||
if t < s.headChunks.maxTime {
|
return true, headMaxt - t, storage.ErrOOONativeHistogramsDisabled
|
||||||
return storage.ErrOutOfOrderSample
|
}
|
||||||
|
return true, headMaxt - t, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// We are allowing exact duplicates as we can encounter them in valid cases
|
// The sample cannot go in both in-order and out-of-order chunk.
|
||||||
// like federation and erroring out at that time would be extremely noisy.
|
if oooTimeWindow > 0 {
|
||||||
if !h.Equals(s.lastHistogramValue) {
|
return true, headMaxt - t, storage.ErrTooOldSample
|
||||||
return storage.ErrDuplicateSampleForTimestamp
|
|
||||||
}
|
}
|
||||||
return nil
|
if t < minValidTime {
|
||||||
|
return false, headMaxt - t, storage.ErrOutOfBounds
|
||||||
|
}
|
||||||
|
return false, headMaxt - t, storage.ErrOutOfOrderSample
|
||||||
}
|
}
|
||||||
|
|
||||||
// appendableFloatHistogram checks whether the given float histogram is valid for appending to the series.
|
// appendableFloatHistogram checks whether the given float histogram sample is valid for appending to the series. (if we return false and no error)
|
||||||
func (s *memSeries) appendableFloatHistogram(t int64, fh *histogram.FloatHistogram) error {
|
// The sample belongs to the out of order chunk if we return true and no error.
|
||||||
if s.headChunks == nil {
|
// An error signifies the sample cannot be handled.
|
||||||
return nil
|
func (s *memSeries) appendableFloatHistogram(t int64, fh *histogram.FloatHistogram, headMaxt, minValidTime, oooTimeWindow int64, oooHistogramsEnabled bool) (isOOO bool, oooDelta int64, err error) {
|
||||||
|
// Check if we can append in the in-order chunk.
|
||||||
|
if t >= minValidTime {
|
||||||
|
if s.headChunks == nil {
|
||||||
|
// The series has no sample and was freshly created.
|
||||||
|
return false, 0, nil
|
||||||
|
}
|
||||||
|
msMaxt := s.maxTime()
|
||||||
|
if t > msMaxt {
|
||||||
|
return false, 0, nil
|
||||||
|
}
|
||||||
|
if t == msMaxt {
|
||||||
|
// We are allowing exact duplicates as we can encounter them in valid cases
|
||||||
|
// like federation and erroring out at that time would be extremely noisy.
|
||||||
|
// This only checks against the latest in-order sample.
|
||||||
|
// The OOO headchunk has its own method to detect these duplicates.
|
||||||
|
if !fh.Equals(s.lastFloatHistogramValue) {
|
||||||
|
return false, 0, storage.ErrDuplicateSampleForTimestamp
|
||||||
|
}
|
||||||
|
// Sample is identical (ts + value) with most current (highest ts) sample in sampleBuf.
|
||||||
|
return false, 0, nil
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if t > s.headChunks.maxTime {
|
// The sample cannot go in the in-order chunk. Check if it can go in the out-of-order chunk.
|
||||||
return nil
|
if oooTimeWindow > 0 && t >= headMaxt-oooTimeWindow {
|
||||||
}
|
if !oooHistogramsEnabled {
|
||||||
if t < s.headChunks.maxTime {
|
return true, headMaxt - t, storage.ErrOOONativeHistogramsDisabled
|
||||||
return storage.ErrOutOfOrderSample
|
}
|
||||||
|
return true, headMaxt - t, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// We are allowing exact duplicates as we can encounter them in valid cases
|
// The sample cannot go in both in-order and out-of-order chunk.
|
||||||
// like federation and erroring out at that time would be extremely noisy.
|
if oooTimeWindow > 0 {
|
||||||
if !fh.Equals(s.lastFloatHistogramValue) {
|
return true, headMaxt - t, storage.ErrTooOldSample
|
||||||
return storage.ErrDuplicateSampleForTimestamp
|
|
||||||
}
|
}
|
||||||
return nil
|
if t < minValidTime {
|
||||||
|
return false, headMaxt - t, storage.ErrOutOfBounds
|
||||||
|
}
|
||||||
|
return false, headMaxt - t, storage.ErrOutOfOrderSample
|
||||||
}
|
}
|
||||||
|
|
||||||
// AppendExemplar for headAppender assumes the series ref already exists, and so it doesn't
|
// AppendExemplar for headAppender assumes the series ref already exists, and so it doesn't
|
||||||
|
@ -577,7 +625,9 @@ func (a *headAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels
|
||||||
return 0, storage.ErrNativeHistogramsDisabled
|
return 0, storage.ErrNativeHistogramsDisabled
|
||||||
}
|
}
|
||||||
|
|
||||||
if t < a.minValidTime {
|
// Fail fast if OOO is disabled and the sample is out of bounds.
|
||||||
|
// Otherwise a full check will be done later to decide if the sample is in-order or out-of-order.
|
||||||
|
if (a.oooTimeWindow == 0 || !a.head.opts.EnableOOONativeHistograms.Load()) && t < a.minValidTime {
|
||||||
a.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeHistogram).Inc()
|
a.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeHistogram).Inc()
|
||||||
return 0, storage.ErrOutOfBounds
|
return 0, storage.ErrOutOfBounds
|
||||||
}
|
}
|
||||||
|
@ -629,15 +679,27 @@ func (a *headAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels
|
||||||
switch {
|
switch {
|
||||||
case h != nil:
|
case h != nil:
|
||||||
s.Lock()
|
s.Lock()
|
||||||
if err := s.appendableHistogram(t, h); err != nil {
|
// TODO(codesome): If we definitely know at this point that the sample is ooo, then optimise
|
||||||
s.Unlock()
|
// to skip that sample from the WAL and write only in the WBL.
|
||||||
if errors.Is(err, storage.ErrOutOfOrderSample) {
|
_, delta, err := s.appendableHistogram(t, h, a.headMaxt, a.minValidTime, a.oooTimeWindow, a.head.opts.EnableOOONativeHistograms.Load())
|
||||||
|
if err != nil {
|
||||||
|
s.pendingCommit = true
|
||||||
|
}
|
||||||
|
s.Unlock()
|
||||||
|
if delta > 0 {
|
||||||
|
a.head.metrics.oooHistogram.Observe(float64(delta) / 1000)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, storage.ErrOutOfOrderSample):
|
||||||
|
fallthrough
|
||||||
|
case errors.Is(err, storage.ErrOOONativeHistogramsDisabled):
|
||||||
a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeHistogram).Inc()
|
a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeHistogram).Inc()
|
||||||
|
case errors.Is(err, storage.ErrTooOldSample):
|
||||||
|
a.head.metrics.tooOldSamples.WithLabelValues(sampleMetricTypeHistogram).Inc()
|
||||||
}
|
}
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
s.pendingCommit = true
|
|
||||||
s.Unlock()
|
|
||||||
a.histograms = append(a.histograms, record.RefHistogramSample{
|
a.histograms = append(a.histograms, record.RefHistogramSample{
|
||||||
Ref: s.ref,
|
Ref: s.ref,
|
||||||
T: t,
|
T: t,
|
||||||
|
@ -646,15 +708,27 @@ func (a *headAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels
|
||||||
a.histogramSeries = append(a.histogramSeries, s)
|
a.histogramSeries = append(a.histogramSeries, s)
|
||||||
case fh != nil:
|
case fh != nil:
|
||||||
s.Lock()
|
s.Lock()
|
||||||
if err := s.appendableFloatHistogram(t, fh); err != nil {
|
// TODO(codesome): If we definitely know at this point that the sample is ooo, then optimise
|
||||||
s.Unlock()
|
// to skip that sample from the WAL and write only in the WBL.
|
||||||
if errors.Is(err, storage.ErrOutOfOrderSample) {
|
_, delta, err := s.appendableFloatHistogram(t, fh, a.headMaxt, a.minValidTime, a.oooTimeWindow, a.head.opts.EnableOOONativeHistograms.Load())
|
||||||
|
if err == nil {
|
||||||
|
s.pendingCommit = true
|
||||||
|
}
|
||||||
|
s.Unlock()
|
||||||
|
if delta > 0 {
|
||||||
|
a.head.metrics.oooHistogram.Observe(float64(delta) / 1000)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, storage.ErrOutOfOrderSample):
|
||||||
|
fallthrough
|
||||||
|
case errors.Is(err, storage.ErrOOONativeHistogramsDisabled):
|
||||||
a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeHistogram).Inc()
|
a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeHistogram).Inc()
|
||||||
|
case errors.Is(err, storage.ErrTooOldSample):
|
||||||
|
a.head.metrics.tooOldSamples.WithLabelValues(sampleMetricTypeHistogram).Inc()
|
||||||
}
|
}
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
s.pendingCommit = true
|
|
||||||
s.Unlock()
|
|
||||||
a.floatHistograms = append(a.floatHistograms, record.RefFloatHistogramSample{
|
a.floatHistograms = append(a.floatHistograms, record.RefFloatHistogramSample{
|
||||||
Ref: s.ref,
|
Ref: s.ref,
|
||||||
T: t,
|
T: t,
|
||||||
|
@ -841,20 +915,24 @@ func (a *headAppender) Commit() (err error) {
|
||||||
floatsAppended = len(a.samples)
|
floatsAppended = len(a.samples)
|
||||||
histogramsAppended = len(a.histograms) + len(a.floatHistograms)
|
histogramsAppended = len(a.histograms) + len(a.floatHistograms)
|
||||||
// number of samples out of order but accepted: with ooo enabled and within time window
|
// number of samples out of order but accepted: with ooo enabled and within time window
|
||||||
oooFloatsAccepted int
|
oooFloatsAccepted int
|
||||||
|
oooHistogramAccepted int
|
||||||
// number of samples rejected due to: out of order but OOO support disabled.
|
// number of samples rejected due to: out of order but OOO support disabled.
|
||||||
floatOOORejected int
|
floatOOORejected int
|
||||||
histoOOORejected int
|
histoOOORejected int
|
||||||
// number of samples rejected due to: that are out of order but too old (OOO support enabled, but outside time window)
|
// number of samples rejected due to: that are out of order but too old (OOO support enabled, but outside time window)
|
||||||
floatTooOldRejected int
|
floatTooOldRejected int
|
||||||
|
histoTooOldRejected int
|
||||||
// number of samples rejected due to: out of bounds: with t < minValidTime (OOO support disabled)
|
// number of samples rejected due to: out of bounds: with t < minValidTime (OOO support disabled)
|
||||||
floatOOBRejected int
|
floatOOBRejected int
|
||||||
|
histoOOBRejected int
|
||||||
inOrderMint int64 = math.MaxInt64
|
inOrderMint int64 = math.MaxInt64
|
||||||
inOrderMaxt int64 = math.MinInt64
|
inOrderMaxt int64 = math.MinInt64
|
||||||
oooMinT int64 = math.MaxInt64
|
oooMinT int64 = math.MaxInt64
|
||||||
oooMaxT int64 = math.MinInt64
|
oooMaxT int64 = math.MinInt64
|
||||||
wblSamples []record.RefSample
|
wblSamples []record.RefSample
|
||||||
|
wblHistograms []record.RefHistogramSample
|
||||||
|
wblFloatHistograms []record.RefFloatHistogramSample
|
||||||
oooMmapMarkers map[chunks.HeadSeriesRef][]chunks.ChunkDiskMapperRef
|
oooMmapMarkers map[chunks.HeadSeriesRef][]chunks.ChunkDiskMapperRef
|
||||||
oooMmapMarkersCount int
|
oooMmapMarkersCount int
|
||||||
oooRecords [][]byte
|
oooRecords [][]byte
|
||||||
|
@ -876,6 +954,8 @@ func (a *headAppender) Commit() (err error) {
|
||||||
if a.head.wbl == nil {
|
if a.head.wbl == nil {
|
||||||
// WBL is not enabled. So no need to collect.
|
// WBL is not enabled. So no need to collect.
|
||||||
wblSamples = nil
|
wblSamples = nil
|
||||||
|
wblHistograms = nil
|
||||||
|
wblFloatHistograms = nil
|
||||||
oooMmapMarkers = nil
|
oooMmapMarkers = nil
|
||||||
oooMmapMarkersCount = 0
|
oooMmapMarkersCount = 0
|
||||||
return
|
return
|
||||||
|
@ -903,8 +983,18 @@ func (a *headAppender) Commit() (err error) {
|
||||||
r := enc.Samples(wblSamples, a.head.getBytesBuffer())
|
r := enc.Samples(wblSamples, a.head.getBytesBuffer())
|
||||||
oooRecords = append(oooRecords, r)
|
oooRecords = append(oooRecords, r)
|
||||||
}
|
}
|
||||||
|
if len(wblHistograms) > 0 {
|
||||||
|
r := enc.HistogramSamples(wblHistograms, a.head.getBytesBuffer())
|
||||||
|
oooRecords = append(oooRecords, r)
|
||||||
|
}
|
||||||
|
if len(wblFloatHistograms) > 0 {
|
||||||
|
r := enc.FloatHistogramSamples(wblFloatHistograms, a.head.getBytesBuffer())
|
||||||
|
oooRecords = append(oooRecords, r)
|
||||||
|
}
|
||||||
|
|
||||||
wblSamples = nil
|
wblSamples = nil
|
||||||
|
wblHistograms = nil
|
||||||
|
wblFloatHistograms = nil
|
||||||
oooMmapMarkers = nil
|
oooMmapMarkers = nil
|
||||||
}
|
}
|
||||||
for i, s := range a.samples {
|
for i, s := range a.samples {
|
||||||
|
@ -1006,51 +1096,193 @@ func (a *headAppender) Commit() (err error) {
|
||||||
for i, s := range a.histograms {
|
for i, s := range a.histograms {
|
||||||
series = a.histogramSeries[i]
|
series = a.histogramSeries[i]
|
||||||
series.Lock()
|
series.Lock()
|
||||||
ok, chunkCreated := series.appendHistogram(s.T, s.H, a.appendID, appendChunkOpts)
|
|
||||||
series.cleanupAppendIDsBelow(a.cleanupAppendIDsBelow)
|
|
||||||
series.pendingCommit = false
|
|
||||||
series.Unlock()
|
|
||||||
|
|
||||||
if ok {
|
oooSample, _, err := series.appendableHistogram(s.T, s.H, a.headMaxt, a.minValidTime, a.oooTimeWindow, a.head.opts.EnableOOONativeHistograms.Load())
|
||||||
if s.T < inOrderMint {
|
switch {
|
||||||
inOrderMint = s.T
|
case err == nil:
|
||||||
}
|
// Do nothing.
|
||||||
if s.T > inOrderMaxt {
|
case errors.Is(err, storage.ErrOutOfOrderSample):
|
||||||
inOrderMaxt = s.T
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
histogramsAppended--
|
histogramsAppended--
|
||||||
histoOOORejected++
|
histoOOORejected++
|
||||||
|
case errors.Is(err, storage.ErrOutOfBounds):
|
||||||
|
histogramsAppended--
|
||||||
|
histoOOBRejected++
|
||||||
|
case errors.Is(err, storage.ErrTooOldSample):
|
||||||
|
histogramsAppended--
|
||||||
|
histoTooOldRejected++
|
||||||
|
default:
|
||||||
|
histogramsAppended--
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var ok, chunkCreated bool
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
// Do nothing here.
|
||||||
|
case oooSample:
|
||||||
|
// Sample is OOO and OOO handling is enabled
|
||||||
|
// and the delta is within the OOO tolerance.
|
||||||
|
var mmapRefs []chunks.ChunkDiskMapperRef
|
||||||
|
ok, chunkCreated, mmapRefs = series.insert(s.T, 0, s.H, nil, a.head.chunkDiskMapper, oooCapMax, a.head.logger)
|
||||||
|
if chunkCreated {
|
||||||
|
r, ok := oooMmapMarkers[series.ref]
|
||||||
|
if !ok || r != nil {
|
||||||
|
// !ok means there are no markers collected for these samples yet. So we first flush the samples
|
||||||
|
// before setting this m-map marker.
|
||||||
|
|
||||||
|
// r != 0 means we have already m-mapped a chunk for this series in the same Commit().
|
||||||
|
// Hence, before we m-map again, we should add the samples and m-map markers
|
||||||
|
// seen till now to the WBL records.
|
||||||
|
collectOOORecords()
|
||||||
|
}
|
||||||
|
|
||||||
|
if oooMmapMarkers == nil {
|
||||||
|
oooMmapMarkers = make(map[chunks.HeadSeriesRef][]chunks.ChunkDiskMapperRef)
|
||||||
|
}
|
||||||
|
if len(mmapRefs) > 0 {
|
||||||
|
oooMmapMarkers[series.ref] = mmapRefs
|
||||||
|
oooMmapMarkersCount += len(mmapRefs)
|
||||||
|
} else {
|
||||||
|
// No chunk was written to disk, so we need to set an initial marker for this series.
|
||||||
|
oooMmapMarkers[series.ref] = []chunks.ChunkDiskMapperRef{0}
|
||||||
|
oooMmapMarkersCount++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ok {
|
||||||
|
wblHistograms = append(wblHistograms, s)
|
||||||
|
if s.T < oooMinT {
|
||||||
|
oooMinT = s.T
|
||||||
|
}
|
||||||
|
if s.T > oooMaxT {
|
||||||
|
oooMaxT = s.T
|
||||||
|
}
|
||||||
|
oooHistogramAccepted++
|
||||||
|
} else {
|
||||||
|
// Sample is an exact duplicate of the last sample.
|
||||||
|
// NOTE: We can only detect updates if they clash with a sample in the OOOHeadChunk,
|
||||||
|
// not with samples in already flushed OOO chunks.
|
||||||
|
// TODO(codesome): Add error reporting? It depends on addressing https://github.com/prometheus/prometheus/discussions/10305.
|
||||||
|
histogramsAppended--
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
ok, chunkCreated = series.appendHistogram(s.T, s.H, a.appendID, appendChunkOpts)
|
||||||
|
if ok {
|
||||||
|
if s.T < inOrderMint {
|
||||||
|
inOrderMint = s.T
|
||||||
|
}
|
||||||
|
if s.T > inOrderMaxt {
|
||||||
|
inOrderMaxt = s.T
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
histogramsAppended--
|
||||||
|
histoOOORejected++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if chunkCreated {
|
if chunkCreated {
|
||||||
a.head.metrics.chunks.Inc()
|
a.head.metrics.chunks.Inc()
|
||||||
a.head.metrics.chunksCreated.Inc()
|
a.head.metrics.chunksCreated.Inc()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
series.cleanupAppendIDsBelow(a.cleanupAppendIDsBelow)
|
||||||
|
series.pendingCommit = false
|
||||||
|
series.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
for i, s := range a.floatHistograms {
|
for i, s := range a.floatHistograms {
|
||||||
series = a.floatHistogramSeries[i]
|
series = a.floatHistogramSeries[i]
|
||||||
series.Lock()
|
series.Lock()
|
||||||
ok, chunkCreated := series.appendFloatHistogram(s.T, s.FH, a.appendID, appendChunkOpts)
|
|
||||||
series.cleanupAppendIDsBelow(a.cleanupAppendIDsBelow)
|
|
||||||
series.pendingCommit = false
|
|
||||||
series.Unlock()
|
|
||||||
|
|
||||||
if ok {
|
oooSample, _, err := series.appendableFloatHistogram(s.T, s.FH, a.headMaxt, a.minValidTime, a.oooTimeWindow, a.head.opts.EnableOOONativeHistograms.Load())
|
||||||
if s.T < inOrderMint {
|
switch {
|
||||||
inOrderMint = s.T
|
case err == nil:
|
||||||
}
|
// Do nothing.
|
||||||
if s.T > inOrderMaxt {
|
case errors.Is(err, storage.ErrOutOfOrderSample):
|
||||||
inOrderMaxt = s.T
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
histogramsAppended--
|
histogramsAppended--
|
||||||
histoOOORejected++
|
histoOOORejected++
|
||||||
|
case errors.Is(err, storage.ErrOutOfBounds):
|
||||||
|
histogramsAppended--
|
||||||
|
histoOOBRejected++
|
||||||
|
case errors.Is(err, storage.ErrTooOldSample):
|
||||||
|
histogramsAppended--
|
||||||
|
histoTooOldRejected++
|
||||||
|
default:
|
||||||
|
histogramsAppended--
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var ok, chunkCreated bool
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
// Do nothing here.
|
||||||
|
case oooSample:
|
||||||
|
// Sample is OOO and OOO handling is enabled
|
||||||
|
// and the delta is within the OOO tolerance.
|
||||||
|
var mmapRefs []chunks.ChunkDiskMapperRef
|
||||||
|
ok, chunkCreated, mmapRefs = series.insert(s.T, 0, nil, s.FH, a.head.chunkDiskMapper, oooCapMax, a.head.logger)
|
||||||
|
if chunkCreated {
|
||||||
|
r, ok := oooMmapMarkers[series.ref]
|
||||||
|
if !ok || r != nil {
|
||||||
|
// !ok means there are no markers collected for these samples yet. So we first flush the samples
|
||||||
|
// before setting this m-map marker.
|
||||||
|
|
||||||
|
// r != 0 means we have already m-mapped a chunk for this series in the same Commit().
|
||||||
|
// Hence, before we m-map again, we should add the samples and m-map markers
|
||||||
|
// seen till now to the WBL records.
|
||||||
|
collectOOORecords()
|
||||||
|
}
|
||||||
|
|
||||||
|
if oooMmapMarkers == nil {
|
||||||
|
oooMmapMarkers = make(map[chunks.HeadSeriesRef][]chunks.ChunkDiskMapperRef)
|
||||||
|
}
|
||||||
|
if len(mmapRefs) > 0 {
|
||||||
|
oooMmapMarkers[series.ref] = mmapRefs
|
||||||
|
oooMmapMarkersCount += len(mmapRefs)
|
||||||
|
} else {
|
||||||
|
// No chunk was written to disk, so we need to set an initial marker for this series.
|
||||||
|
oooMmapMarkers[series.ref] = []chunks.ChunkDiskMapperRef{0}
|
||||||
|
oooMmapMarkersCount++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ok {
|
||||||
|
wblFloatHistograms = append(wblFloatHistograms, s)
|
||||||
|
if s.T < oooMinT {
|
||||||
|
oooMinT = s.T
|
||||||
|
}
|
||||||
|
if s.T > oooMaxT {
|
||||||
|
oooMaxT = s.T
|
||||||
|
}
|
||||||
|
oooHistogramAccepted++
|
||||||
|
} else {
|
||||||
|
// Sample is an exact duplicate of the last sample.
|
||||||
|
// NOTE: We can only detect updates if they clash with a sample in the OOOHeadChunk,
|
||||||
|
// not with samples in already flushed OOO chunks.
|
||||||
|
// TODO(codesome): Add error reporting? It depends on addressing https://github.com/prometheus/prometheus/discussions/10305.
|
||||||
|
histogramsAppended--
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
ok, chunkCreated = series.appendFloatHistogram(s.T, s.FH, a.appendID, appendChunkOpts)
|
||||||
|
if ok {
|
||||||
|
if s.T < inOrderMint {
|
||||||
|
inOrderMint = s.T
|
||||||
|
}
|
||||||
|
if s.T > inOrderMaxt {
|
||||||
|
inOrderMaxt = s.T
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
histogramsAppended--
|
||||||
|
histoOOORejected++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if chunkCreated {
|
if chunkCreated {
|
||||||
a.head.metrics.chunks.Inc()
|
a.head.metrics.chunks.Inc()
|
||||||
a.head.metrics.chunksCreated.Inc()
|
a.head.metrics.chunksCreated.Inc()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
series.cleanupAppendIDsBelow(a.cleanupAppendIDsBelow)
|
||||||
|
series.pendingCommit = false
|
||||||
|
series.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
for i, m := range a.metadata {
|
for i, m := range a.metadata {
|
||||||
|
@ -1067,6 +1299,7 @@ func (a *headAppender) Commit() (err error) {
|
||||||
a.head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeFloat).Add(float64(floatsAppended))
|
a.head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeFloat).Add(float64(floatsAppended))
|
||||||
a.head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeHistogram).Add(float64(histogramsAppended))
|
a.head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeHistogram).Add(float64(histogramsAppended))
|
||||||
a.head.metrics.outOfOrderSamplesAppended.WithLabelValues(sampleMetricTypeFloat).Add(float64(oooFloatsAccepted))
|
a.head.metrics.outOfOrderSamplesAppended.WithLabelValues(sampleMetricTypeFloat).Add(float64(oooFloatsAccepted))
|
||||||
|
a.head.metrics.outOfOrderSamplesAppended.WithLabelValues(sampleMetricTypeHistogram).Add(float64(oooHistogramAccepted))
|
||||||
a.head.updateMinMaxTime(inOrderMint, inOrderMaxt)
|
a.head.updateMinMaxTime(inOrderMint, inOrderMaxt)
|
||||||
a.head.updateMinOOOMaxOOOTime(oooMinT, oooMaxT)
|
a.head.updateMinOOOMaxOOOTime(oooMinT, oooMaxT)
|
||||||
|
|
||||||
|
|
|
@ -2723,15 +2723,32 @@ func TestIsolationWithoutAdd(t *testing.T) {
|
||||||
func TestOutOfOrderSamplesMetric(t *testing.T) {
|
func TestOutOfOrderSamplesMetric(t *testing.T) {
|
||||||
for name, scenario := range sampleTypeScenarios {
|
for name, scenario := range sampleTypeScenarios {
|
||||||
t.Run(name, func(t *testing.T) {
|
t.Run(name, func(t *testing.T) {
|
||||||
testOutOfOrderSamplesMetric(t, scenario)
|
options := DefaultOptions()
|
||||||
|
options.EnableNativeHistograms = true
|
||||||
|
options.EnableOOONativeHistograms = true
|
||||||
|
testOutOfOrderSamplesMetric(t, scenario, options, storage.ErrOutOfOrderSample)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func testOutOfOrderSamplesMetric(t *testing.T, scenario sampleTypeScenario) {
|
func TestOutOfOrderSamplesMetricNativeHistogramOOODisabled(t *testing.T) {
|
||||||
dir := t.TempDir()
|
for name, scenario := range sampleTypeScenarios {
|
||||||
|
if scenario.sampleType != "histogram" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
options := DefaultOptions()
|
||||||
|
options.OutOfOrderTimeWindow = (1000 * time.Minute).Milliseconds()
|
||||||
|
options.EnableNativeHistograms = true
|
||||||
|
options.EnableOOONativeHistograms = false
|
||||||
|
testOutOfOrderSamplesMetric(t, scenario, options, storage.ErrOOONativeHistogramsDisabled)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
db, err := Open(dir, nil, nil, DefaultOptions(), nil)
|
func testOutOfOrderSamplesMetric(t *testing.T, scenario sampleTypeScenario, options *Options, expectOutOfOrderError error) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
db, err := Open(dir, nil, nil, options, nil)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
defer func() {
|
defer func() {
|
||||||
require.NoError(t, db.Close())
|
require.NoError(t, db.Close())
|
||||||
|
@ -2755,15 +2772,15 @@ func testOutOfOrderSamplesMetric(t *testing.T, scenario sampleTypeScenario) {
|
||||||
require.Equal(t, 0.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType)))
|
require.Equal(t, 0.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType)))
|
||||||
app = db.Appender(ctx)
|
app = db.Appender(ctx)
|
||||||
_, err = appendSample(app, 2)
|
_, err = appendSample(app, 2)
|
||||||
require.Equal(t, storage.ErrOutOfOrderSample, err)
|
require.Equal(t, expectOutOfOrderError, err)
|
||||||
require.Equal(t, 1.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType)))
|
require.Equal(t, 1.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType)))
|
||||||
|
|
||||||
_, err = appendSample(app, 3)
|
_, err = appendSample(app, 3)
|
||||||
require.Equal(t, storage.ErrOutOfOrderSample, err)
|
require.Equal(t, expectOutOfOrderError, err)
|
||||||
require.Equal(t, 2.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType)))
|
require.Equal(t, 2.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType)))
|
||||||
|
|
||||||
_, err = appendSample(app, 4)
|
_, err = appendSample(app, 4)
|
||||||
require.Equal(t, storage.ErrOutOfOrderSample, err)
|
require.Equal(t, expectOutOfOrderError, err)
|
||||||
require.Equal(t, 3.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType)))
|
require.Equal(t, 3.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType)))
|
||||||
require.NoError(t, app.Commit())
|
require.NoError(t, app.Commit())
|
||||||
|
|
||||||
|
@ -2798,15 +2815,15 @@ func testOutOfOrderSamplesMetric(t *testing.T, scenario sampleTypeScenario) {
|
||||||
// Test out of order metric.
|
// Test out of order metric.
|
||||||
app = db.Appender(ctx)
|
app = db.Appender(ctx)
|
||||||
_, err = appendSample(app, db.head.minValidTime.Load()+DefaultBlockDuration+2)
|
_, err = appendSample(app, db.head.minValidTime.Load()+DefaultBlockDuration+2)
|
||||||
require.Equal(t, storage.ErrOutOfOrderSample, err)
|
require.Equal(t, expectOutOfOrderError, err)
|
||||||
require.Equal(t, 4.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType)))
|
require.Equal(t, 4.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType)))
|
||||||
|
|
||||||
_, err = appendSample(app, db.head.minValidTime.Load()+DefaultBlockDuration+3)
|
_, err = appendSample(app, db.head.minValidTime.Load()+DefaultBlockDuration+3)
|
||||||
require.Equal(t, storage.ErrOutOfOrderSample, err)
|
require.Equal(t, expectOutOfOrderError, err)
|
||||||
require.Equal(t, 5.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType)))
|
require.Equal(t, 5.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType)))
|
||||||
|
|
||||||
_, err = appendSample(app, db.head.minValidTime.Load()+DefaultBlockDuration+4)
|
_, err = appendSample(app, db.head.minValidTime.Load()+DefaultBlockDuration+4)
|
||||||
require.Equal(t, storage.ErrOutOfOrderSample, err)
|
require.Equal(t, expectOutOfOrderError, err)
|
||||||
require.Equal(t, 6.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType)))
|
require.Equal(t, 6.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(scenario.sampleType)))
|
||||||
require.NoError(t, app.Commit())
|
require.NoError(t, app.Commit())
|
||||||
}
|
}
|
||||||
|
@ -4657,10 +4674,172 @@ func TestHistogramCounterResetHeader(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestOOOHistogramCounterResetHeaders(t *testing.T) {
|
||||||
|
for _, floatHisto := range []bool{true, false} {
|
||||||
|
t.Run(fmt.Sprintf("floatHistogram=%t", floatHisto), func(t *testing.T) {
|
||||||
|
l := labels.FromStrings("a", "b")
|
||||||
|
head, _ := newTestHead(t, 1000, wlog.CompressionNone, true)
|
||||||
|
head.opts.OutOfOrderCapMax.Store(5)
|
||||||
|
head.opts.EnableOOONativeHistograms.Store(true)
|
||||||
|
|
||||||
|
t.Cleanup(func() {
|
||||||
|
require.NoError(t, head.Close())
|
||||||
|
})
|
||||||
|
require.NoError(t, head.Init(0))
|
||||||
|
|
||||||
|
appendHistogram := func(ts int64, h *histogram.Histogram) {
|
||||||
|
app := head.Appender(context.Background())
|
||||||
|
var err error
|
||||||
|
if floatHisto {
|
||||||
|
_, err = app.AppendHistogram(0, l, ts, nil, h.ToFloat(nil))
|
||||||
|
} else {
|
||||||
|
_, err = app.AppendHistogram(0, l, ts, h.Copy(), nil)
|
||||||
|
}
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, app.Commit())
|
||||||
|
}
|
||||||
|
|
||||||
|
type expOOOMmappedChunks struct {
|
||||||
|
header chunkenc.CounterResetHeader
|
||||||
|
mint, maxt int64
|
||||||
|
numSamples uint16
|
||||||
|
}
|
||||||
|
|
||||||
|
var expChunks []expOOOMmappedChunks
|
||||||
|
checkOOOExpCounterResetHeader := func(newChunks ...expOOOMmappedChunks) {
|
||||||
|
expChunks = append(expChunks, newChunks...)
|
||||||
|
|
||||||
|
ms, _, err := head.getOrCreate(l.Hash(), l)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
require.Len(t, ms.ooo.oooMmappedChunks, len(expChunks))
|
||||||
|
|
||||||
|
for i, mmapChunk := range ms.ooo.oooMmappedChunks {
|
||||||
|
chk, err := head.chunkDiskMapper.Chunk(mmapChunk.ref)
|
||||||
|
require.NoError(t, err)
|
||||||
|
if floatHisto {
|
||||||
|
require.Equal(t, expChunks[i].header, chk.(*chunkenc.FloatHistogramChunk).GetCounterResetHeader())
|
||||||
|
} else {
|
||||||
|
require.Equal(t, expChunks[i].header, chk.(*chunkenc.HistogramChunk).GetCounterResetHeader())
|
||||||
|
}
|
||||||
|
require.Equal(t, expChunks[i].mint, mmapChunk.minTime)
|
||||||
|
require.Equal(t, expChunks[i].maxt, mmapChunk.maxTime)
|
||||||
|
require.Equal(t, expChunks[i].numSamples, mmapChunk.numSamples)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Append an in-order histogram, so the rest of the samples can be detected as OOO.
|
||||||
|
appendHistogram(1000, tsdbutil.GenerateTestHistogram(1000))
|
||||||
|
|
||||||
|
// OOO histogram
|
||||||
|
for i := 1; i <= 5; i++ {
|
||||||
|
appendHistogram(100+int64(i), tsdbutil.GenerateTestHistogram(1000+i))
|
||||||
|
}
|
||||||
|
// Nothing mmapped yet.
|
||||||
|
checkOOOExpCounterResetHeader()
|
||||||
|
|
||||||
|
// 6th observation (which triggers a head chunk mmapping).
|
||||||
|
appendHistogram(int64(112), tsdbutil.GenerateTestHistogram(1002))
|
||||||
|
|
||||||
|
// One mmapped chunk with (ts, val) [(101, 1001), (102, 1002), (103, 1003), (104, 1004), (105, 1005)].
|
||||||
|
checkOOOExpCounterResetHeader(expOOOMmappedChunks{
|
||||||
|
header: chunkenc.UnknownCounterReset,
|
||||||
|
mint: 101,
|
||||||
|
maxt: 105,
|
||||||
|
numSamples: 5,
|
||||||
|
})
|
||||||
|
|
||||||
|
// Add more samples, there's a counter reset at ts 122.
|
||||||
|
appendHistogram(int64(110), tsdbutil.GenerateTestHistogram(1001))
|
||||||
|
appendHistogram(int64(124), tsdbutil.GenerateTestHistogram(904))
|
||||||
|
appendHistogram(int64(123), tsdbutil.GenerateTestHistogram(903))
|
||||||
|
appendHistogram(int64(122), tsdbutil.GenerateTestHistogram(902))
|
||||||
|
|
||||||
|
// New samples not mmapped yet.
|
||||||
|
checkOOOExpCounterResetHeader()
|
||||||
|
|
||||||
|
// 11th observation (which triggers another head chunk mmapping).
|
||||||
|
appendHistogram(int64(200), tsdbutil.GenerateTestHistogram(2000))
|
||||||
|
|
||||||
|
// Two new mmapped chunks [(110, 1001), (112, 1002)], [(122, 902), (123, 903), (124, 904)].
|
||||||
|
checkOOOExpCounterResetHeader(
|
||||||
|
expOOOMmappedChunks{
|
||||||
|
header: chunkenc.UnknownCounterReset,
|
||||||
|
mint: 110,
|
||||||
|
maxt: 112,
|
||||||
|
numSamples: 2,
|
||||||
|
},
|
||||||
|
expOOOMmappedChunks{
|
||||||
|
header: chunkenc.CounterReset,
|
||||||
|
mint: 122,
|
||||||
|
maxt: 124,
|
||||||
|
numSamples: 3,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
// Count is lower than previous sample at ts 200, and NotCounterReset is always ignored on append.
|
||||||
|
appendHistogram(int64(205), tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(1000)))
|
||||||
|
|
||||||
|
appendHistogram(int64(210), tsdbutil.SetHistogramCounterReset(tsdbutil.GenerateTestHistogram(2010)))
|
||||||
|
|
||||||
|
appendHistogram(int64(220), tsdbutil.GenerateTestHistogram(2020))
|
||||||
|
|
||||||
|
appendHistogram(int64(215), tsdbutil.GenerateTestHistogram(2005))
|
||||||
|
|
||||||
|
// 16th observation (which triggers another head chunk mmapping).
|
||||||
|
appendHistogram(int64(350), tsdbutil.GenerateTestHistogram(4000))
|
||||||
|
|
||||||
|
// Four new mmapped chunks: [(200, 2000)] [(205, 1000)], [(210, 2010)], [(215, 2015), (220, 2020)]
|
||||||
|
checkOOOExpCounterResetHeader(
|
||||||
|
expOOOMmappedChunks{
|
||||||
|
header: chunkenc.UnknownCounterReset,
|
||||||
|
mint: 200,
|
||||||
|
maxt: 200,
|
||||||
|
numSamples: 1,
|
||||||
|
},
|
||||||
|
expOOOMmappedChunks{
|
||||||
|
header: chunkenc.CounterReset,
|
||||||
|
mint: 205,
|
||||||
|
maxt: 205,
|
||||||
|
numSamples: 1,
|
||||||
|
},
|
||||||
|
expOOOMmappedChunks{
|
||||||
|
header: chunkenc.CounterReset,
|
||||||
|
mint: 210,
|
||||||
|
maxt: 210,
|
||||||
|
numSamples: 1,
|
||||||
|
},
|
||||||
|
expOOOMmappedChunks{
|
||||||
|
header: chunkenc.CounterReset,
|
||||||
|
mint: 215,
|
||||||
|
maxt: 220,
|
||||||
|
numSamples: 2,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
// Adding five more samples (21 in total), so another mmapped chunk is created.
|
||||||
|
appendHistogram(300, tsdbutil.SetHistogramCounterReset(tsdbutil.GenerateTestHistogram(3000)))
|
||||||
|
|
||||||
|
for i := 1; i <= 4; i++ {
|
||||||
|
appendHistogram(300+int64(i), tsdbutil.GenerateTestHistogram(3000+i))
|
||||||
|
}
|
||||||
|
|
||||||
|
// One mmapped chunk with (ts, val) [(300, 3000), (301, 3001), (302, 3002), (303, 3003), (350, 4000)].
|
||||||
|
checkOOOExpCounterResetHeader(expOOOMmappedChunks{
|
||||||
|
header: chunkenc.CounterReset,
|
||||||
|
mint: 300,
|
||||||
|
maxt: 350,
|
||||||
|
numSamples: 5,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestAppendingDifferentEncodingToSameSeries(t *testing.T) {
|
func TestAppendingDifferentEncodingToSameSeries(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
opts := DefaultOptions()
|
opts := DefaultOptions()
|
||||||
opts.EnableNativeHistograms = true
|
opts.EnableNativeHistograms = true
|
||||||
|
opts.EnableOOONativeHistograms = true
|
||||||
db, err := Open(dir, nil, nil, opts, nil)
|
db, err := Open(dir, nil, nil, opts, nil)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
t.Cleanup(func() {
|
t.Cleanup(func() {
|
||||||
|
@ -4931,6 +5110,8 @@ func testWBLReplay(t *testing.T, scenario sampleTypeScenario) {
|
||||||
opts.ChunkRange = 1000
|
opts.ChunkRange = 1000
|
||||||
opts.ChunkDirRoot = dir
|
opts.ChunkDirRoot = dir
|
||||||
opts.OutOfOrderTimeWindow.Store(30 * time.Minute.Milliseconds())
|
opts.OutOfOrderTimeWindow.Store(30 * time.Minute.Milliseconds())
|
||||||
|
opts.EnableNativeHistograms.Store(true)
|
||||||
|
opts.EnableOOONativeHistograms.Store(true)
|
||||||
|
|
||||||
h, err := NewHead(nil, nil, wal, oooWlog, opts, nil)
|
h, err := NewHead(nil, nil, wal, oooWlog, opts, nil)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
@ -4940,13 +5121,12 @@ func testWBLReplay(t *testing.T, scenario sampleTypeScenario) {
|
||||||
l := labels.FromStrings("foo", "bar")
|
l := labels.FromStrings("foo", "bar")
|
||||||
appendSample := func(mins int64, val float64, isOOO bool) {
|
appendSample := func(mins int64, val float64, isOOO bool) {
|
||||||
app := h.Appender(context.Background())
|
app := h.Appender(context.Background())
|
||||||
ts, v := mins*time.Minute.Milliseconds(), val
|
_, s, err := scenario.appendFunc(app, l, mins*time.Minute.Milliseconds(), mins)
|
||||||
_, err := app.Append(0, l, ts, v)
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.NoError(t, app.Commit())
|
require.NoError(t, app.Commit())
|
||||||
|
|
||||||
if isOOO {
|
if isOOO {
|
||||||
expOOOSamples = append(expOOOSamples, sample{t: ts, f: v})
|
expOOOSamples = append(expOOOSamples, s)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -5025,6 +5205,8 @@ func testOOOMmapReplay(t *testing.T, scenario sampleTypeScenario) {
|
||||||
opts.ChunkDirRoot = dir
|
opts.ChunkDirRoot = dir
|
||||||
opts.OutOfOrderCapMax.Store(30)
|
opts.OutOfOrderCapMax.Store(30)
|
||||||
opts.OutOfOrderTimeWindow.Store(1000 * time.Minute.Milliseconds())
|
opts.OutOfOrderTimeWindow.Store(1000 * time.Minute.Milliseconds())
|
||||||
|
opts.EnableNativeHistograms.Store(true)
|
||||||
|
opts.EnableOOONativeHistograms.Store(true)
|
||||||
|
|
||||||
h, err := NewHead(nil, nil, wal, oooWlog, opts, nil)
|
h, err := NewHead(nil, nil, wal, oooWlog, opts, nil)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
@ -5326,6 +5508,8 @@ func testOOOAppendWithNoSeries(t *testing.T, appendFunc func(appender storage.Ap
|
||||||
opts.ChunkDirRoot = dir
|
opts.ChunkDirRoot = dir
|
||||||
opts.OutOfOrderCapMax.Store(30)
|
opts.OutOfOrderCapMax.Store(30)
|
||||||
opts.OutOfOrderTimeWindow.Store(120 * time.Minute.Milliseconds())
|
opts.OutOfOrderTimeWindow.Store(120 * time.Minute.Milliseconds())
|
||||||
|
opts.EnableNativeHistograms.Store(true)
|
||||||
|
opts.EnableOOONativeHistograms.Store(true)
|
||||||
|
|
||||||
h, err := NewHead(nil, nil, wal, oooWlog, opts, nil)
|
h, err := NewHead(nil, nil, wal, oooWlog, opts, nil)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
@ -5399,7 +5583,9 @@ func testOOOAppendWithNoSeries(t *testing.T, appendFunc func(appender storage.Ap
|
||||||
func TestHeadMinOOOTimeUpdate(t *testing.T) {
|
func TestHeadMinOOOTimeUpdate(t *testing.T) {
|
||||||
for name, scenario := range sampleTypeScenarios {
|
for name, scenario := range sampleTypeScenarios {
|
||||||
t.Run(name, func(t *testing.T) {
|
t.Run(name, func(t *testing.T) {
|
||||||
testHeadMinOOOTimeUpdate(t, scenario)
|
if scenario.sampleType == sampleMetricTypeFloat {
|
||||||
|
testHeadMinOOOTimeUpdate(t, scenario)
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -5414,6 +5600,8 @@ func testHeadMinOOOTimeUpdate(t *testing.T, scenario sampleTypeScenario) {
|
||||||
opts := DefaultHeadOptions()
|
opts := DefaultHeadOptions()
|
||||||
opts.ChunkDirRoot = dir
|
opts.ChunkDirRoot = dir
|
||||||
opts.OutOfOrderTimeWindow.Store(10 * time.Minute.Milliseconds())
|
opts.OutOfOrderTimeWindow.Store(10 * time.Minute.Milliseconds())
|
||||||
|
opts.EnableNativeHistograms.Store(true)
|
||||||
|
opts.EnableOOONativeHistograms.Store(true)
|
||||||
|
|
||||||
h, err := NewHead(nil, nil, wal, oooWlog, opts, nil)
|
h, err := NewHead(nil, nil, wal, oooWlog, opts, nil)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
168
tsdb/head_wal.go
168
tsdb/head_wal.go
|
@ -646,9 +646,9 @@ func (wp *walSubsetProcessor) processWALSamples(h *Head, mmappedChunks, oooMmapp
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *Head) loadWBL(r *wlog.Reader, syms *labels.SymbolTable, multiRef map[chunks.HeadSeriesRef]chunks.HeadSeriesRef, lastMmapRef chunks.ChunkDiskMapperRef) (err error) {
|
func (h *Head) loadWBL(r *wlog.Reader, syms *labels.SymbolTable, multiRef map[chunks.HeadSeriesRef]chunks.HeadSeriesRef, lastMmapRef chunks.ChunkDiskMapperRef) (err error) {
|
||||||
// Track number of samples, m-map markers, that referenced a series we don't know about
|
// Track number of samples, histogram samples, m-map markers, that referenced a series we don't know about
|
||||||
// for error reporting.
|
// for error reporting.
|
||||||
var unknownRefs, mmapMarkerUnknownRefs atomic.Uint64
|
var unknownRefs, unknownHistogramRefs, mmapMarkerUnknownRefs atomic.Uint64
|
||||||
|
|
||||||
lastSeq, lastOff := lastMmapRef.Unpack()
|
lastSeq, lastOff := lastMmapRef.Unpack()
|
||||||
// Start workers that each process samples for a partition of the series ID space.
|
// Start workers that each process samples for a partition of the series ID space.
|
||||||
|
@ -657,8 +657,9 @@ func (h *Head) loadWBL(r *wlog.Reader, syms *labels.SymbolTable, multiRef map[ch
|
||||||
concurrency = h.opts.WALReplayConcurrency
|
concurrency = h.opts.WALReplayConcurrency
|
||||||
processors = make([]wblSubsetProcessor, concurrency)
|
processors = make([]wblSubsetProcessor, concurrency)
|
||||||
|
|
||||||
dec = record.NewDecoder(syms)
|
dec record.Decoder
|
||||||
shards = make([][]record.RefSample, concurrency)
|
shards = make([][]record.RefSample, concurrency)
|
||||||
|
histogramShards = make([][]histogramRecord, concurrency)
|
||||||
|
|
||||||
decodedCh = make(chan interface{}, 10)
|
decodedCh = make(chan interface{}, 10)
|
||||||
decodeErr error
|
decodeErr error
|
||||||
|
@ -672,6 +673,16 @@ func (h *Head) loadWBL(r *wlog.Reader, syms *labels.SymbolTable, multiRef map[ch
|
||||||
return []record.RefMmapMarker{}
|
return []record.RefMmapMarker{}
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
histogramSamplesPool = sync.Pool{
|
||||||
|
New: func() interface{} {
|
||||||
|
return []record.RefHistogramSample{}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
floatHistogramSamplesPool = sync.Pool{
|
||||||
|
New: func() interface{} {
|
||||||
|
return []record.RefFloatHistogramSample{}
|
||||||
|
},
|
||||||
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
defer func() {
|
defer func() {
|
||||||
|
@ -692,8 +703,9 @@ func (h *Head) loadWBL(r *wlog.Reader, syms *labels.SymbolTable, multiRef map[ch
|
||||||
processors[i].setup()
|
processors[i].setup()
|
||||||
|
|
||||||
go func(wp *wblSubsetProcessor) {
|
go func(wp *wblSubsetProcessor) {
|
||||||
unknown := wp.processWBLSamples(h)
|
unknown, unknownHistograms := wp.processWBLSamples(h)
|
||||||
unknownRefs.Add(unknown)
|
unknownRefs.Add(unknown)
|
||||||
|
unknownHistogramRefs.Add(unknownHistograms)
|
||||||
wg.Done()
|
wg.Done()
|
||||||
}(&processors[i])
|
}(&processors[i])
|
||||||
}
|
}
|
||||||
|
@ -727,6 +739,30 @@ func (h *Head) loadWBL(r *wlog.Reader, syms *labels.SymbolTable, multiRef map[ch
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
decodedCh <- markers
|
decodedCh <- markers
|
||||||
|
case record.HistogramSamples:
|
||||||
|
hists := histogramSamplesPool.Get().([]record.RefHistogramSample)[:0]
|
||||||
|
hists, err = dec.HistogramSamples(rec, hists)
|
||||||
|
if err != nil {
|
||||||
|
decodeErr = &wlog.CorruptionErr{
|
||||||
|
Err: fmt.Errorf("decode histograms: %w", err),
|
||||||
|
Segment: r.Segment(),
|
||||||
|
Offset: r.Offset(),
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
decodedCh <- hists
|
||||||
|
case record.FloatHistogramSamples:
|
||||||
|
hists := floatHistogramSamplesPool.Get().([]record.RefFloatHistogramSample)[:0]
|
||||||
|
hists, err = dec.FloatHistogramSamples(rec, hists)
|
||||||
|
if err != nil {
|
||||||
|
decodeErr = &wlog.CorruptionErr{
|
||||||
|
Err: fmt.Errorf("decode float histograms: %w", err),
|
||||||
|
Segment: r.Segment(),
|
||||||
|
Offset: r.Offset(),
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
decodedCh <- hists
|
||||||
default:
|
default:
|
||||||
// Noop.
|
// Noop.
|
||||||
}
|
}
|
||||||
|
@ -791,6 +827,70 @@ func (h *Head) loadWBL(r *wlog.Reader, syms *labels.SymbolTable, multiRef map[ch
|
||||||
idx := uint64(ms.ref) % uint64(concurrency)
|
idx := uint64(ms.ref) % uint64(concurrency)
|
||||||
processors[idx].input <- wblSubsetProcessorInputItem{mmappedSeries: ms}
|
processors[idx].input <- wblSubsetProcessorInputItem{mmappedSeries: ms}
|
||||||
}
|
}
|
||||||
|
case []record.RefHistogramSample:
|
||||||
|
samples := v
|
||||||
|
// We split up the samples into chunks of 5000 samples or less.
|
||||||
|
// With O(300 * #cores) in-flight sample batches, large scrapes could otherwise
|
||||||
|
// cause thousands of very large in flight buffers occupying large amounts
|
||||||
|
// of unused memory.
|
||||||
|
for len(samples) > 0 {
|
||||||
|
m := 5000
|
||||||
|
if len(samples) < m {
|
||||||
|
m = len(samples)
|
||||||
|
}
|
||||||
|
for i := 0; i < concurrency; i++ {
|
||||||
|
if histogramShards[i] == nil {
|
||||||
|
histogramShards[i] = processors[i].reuseHistogramBuf()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, sam := range samples[:m] {
|
||||||
|
if r, ok := multiRef[sam.Ref]; ok {
|
||||||
|
sam.Ref = r
|
||||||
|
}
|
||||||
|
mod := uint64(sam.Ref) % uint64(concurrency)
|
||||||
|
histogramShards[mod] = append(histogramShards[mod], histogramRecord{ref: sam.Ref, t: sam.T, h: sam.H})
|
||||||
|
}
|
||||||
|
for i := 0; i < concurrency; i++ {
|
||||||
|
if len(histogramShards[i]) > 0 {
|
||||||
|
processors[i].input <- wblSubsetProcessorInputItem{histogramSamples: histogramShards[i]}
|
||||||
|
histogramShards[i] = nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
samples = samples[m:]
|
||||||
|
}
|
||||||
|
histogramSamplesPool.Put(v) //nolint:staticcheck
|
||||||
|
case []record.RefFloatHistogramSample:
|
||||||
|
samples := v
|
||||||
|
// We split up the samples into chunks of 5000 samples or less.
|
||||||
|
// With O(300 * #cores) in-flight sample batches, large scrapes could otherwise
|
||||||
|
// cause thousands of very large in flight buffers occupying large amounts
|
||||||
|
// of unused memory.
|
||||||
|
for len(samples) > 0 {
|
||||||
|
m := 5000
|
||||||
|
if len(samples) < m {
|
||||||
|
m = len(samples)
|
||||||
|
}
|
||||||
|
for i := 0; i < concurrency; i++ {
|
||||||
|
if histogramShards[i] == nil {
|
||||||
|
histogramShards[i] = processors[i].reuseHistogramBuf()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, sam := range samples[:m] {
|
||||||
|
if r, ok := multiRef[sam.Ref]; ok {
|
||||||
|
sam.Ref = r
|
||||||
|
}
|
||||||
|
mod := uint64(sam.Ref) % uint64(concurrency)
|
||||||
|
histogramShards[mod] = append(histogramShards[mod], histogramRecord{ref: sam.Ref, t: sam.T, fh: sam.FH})
|
||||||
|
}
|
||||||
|
for i := 0; i < concurrency; i++ {
|
||||||
|
if len(histogramShards[i]) > 0 {
|
||||||
|
processors[i].input <- wblSubsetProcessorInputItem{histogramSamples: histogramShards[i]}
|
||||||
|
histogramShards[i] = nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
samples = samples[m:]
|
||||||
|
}
|
||||||
|
floatHistogramSamplesPool.Put(v) //nolint:staticcheck
|
||||||
default:
|
default:
|
||||||
panic(fmt.Errorf("unexpected decodedCh type: %T", d))
|
panic(fmt.Errorf("unexpected decodedCh type: %T", d))
|
||||||
}
|
}
|
||||||
|
@ -833,17 +933,20 @@ func (e errLoadWbl) Unwrap() error {
|
||||||
}
|
}
|
||||||
|
|
||||||
type wblSubsetProcessor struct {
|
type wblSubsetProcessor struct {
|
||||||
input chan wblSubsetProcessorInputItem
|
input chan wblSubsetProcessorInputItem
|
||||||
output chan []record.RefSample
|
output chan []record.RefSample
|
||||||
|
histogramsOutput chan []histogramRecord
|
||||||
}
|
}
|
||||||
|
|
||||||
type wblSubsetProcessorInputItem struct {
|
type wblSubsetProcessorInputItem struct {
|
||||||
mmappedSeries *memSeries
|
mmappedSeries *memSeries
|
||||||
samples []record.RefSample
|
samples []record.RefSample
|
||||||
|
histogramSamples []histogramRecord
|
||||||
}
|
}
|
||||||
|
|
||||||
func (wp *wblSubsetProcessor) setup() {
|
func (wp *wblSubsetProcessor) setup() {
|
||||||
wp.output = make(chan []record.RefSample, 300)
|
wp.output = make(chan []record.RefSample, 300)
|
||||||
|
wp.histogramsOutput = make(chan []histogramRecord, 300)
|
||||||
wp.input = make(chan wblSubsetProcessorInputItem, 300)
|
wp.input = make(chan wblSubsetProcessorInputItem, 300)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -851,6 +954,8 @@ func (wp *wblSubsetProcessor) closeAndDrain() {
|
||||||
close(wp.input)
|
close(wp.input)
|
||||||
for range wp.output {
|
for range wp.output {
|
||||||
}
|
}
|
||||||
|
for range wp.histogramsOutput {
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// If there is a buffer in the output chan, return it for reuse, otherwise return nil.
|
// If there is a buffer in the output chan, return it for reuse, otherwise return nil.
|
||||||
|
@ -863,10 +968,21 @@ func (wp *wblSubsetProcessor) reuseBuf() []record.RefSample {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// If there is a buffer in the output chan, return it for reuse, otherwise return nil.
|
||||||
|
func (wp *wblSubsetProcessor) reuseHistogramBuf() []histogramRecord {
|
||||||
|
select {
|
||||||
|
case buf := <-wp.histogramsOutput:
|
||||||
|
return buf[:0]
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// processWBLSamples adds the samples it receives to the head and passes
|
// processWBLSamples adds the samples it receives to the head and passes
|
||||||
// the buffer received to an output channel for reuse.
|
// the buffer received to an output channel for reuse.
|
||||||
func (wp *wblSubsetProcessor) processWBLSamples(h *Head) (unknownRefs uint64) {
|
func (wp *wblSubsetProcessor) processWBLSamples(h *Head) (unknownRefs, unknownHistogramRefs uint64) {
|
||||||
defer close(wp.output)
|
defer close(wp.output)
|
||||||
|
defer close(wp.histogramsOutput)
|
||||||
|
|
||||||
oooCapMax := h.opts.OutOfOrderCapMax.Load()
|
oooCapMax := h.opts.OutOfOrderCapMax.Load()
|
||||||
// We don't check for minValidTime for ooo samples.
|
// We don't check for minValidTime for ooo samples.
|
||||||
|
@ -905,11 +1021,41 @@ func (wp *wblSubsetProcessor) processWBLSamples(h *Head) (unknownRefs uint64) {
|
||||||
case wp.output <- in.samples:
|
case wp.output <- in.samples:
|
||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
for _, s := range in.histogramSamples {
|
||||||
|
ms := h.series.getByID(s.ref)
|
||||||
|
if ms == nil {
|
||||||
|
unknownHistogramRefs++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var chunkCreated bool
|
||||||
|
var ok bool
|
||||||
|
if s.h != nil {
|
||||||
|
ok, chunkCreated, _ = ms.insert(s.t, 0, s.h, nil, h.chunkDiskMapper, oooCapMax, h.logger)
|
||||||
|
} else {
|
||||||
|
ok, chunkCreated, _ = ms.insert(s.t, 0, nil, s.fh, h.chunkDiskMapper, oooCapMax, h.logger)
|
||||||
|
}
|
||||||
|
if chunkCreated {
|
||||||
|
h.metrics.chunksCreated.Inc()
|
||||||
|
h.metrics.chunks.Inc()
|
||||||
|
}
|
||||||
|
if ok {
|
||||||
|
if s.t > maxt {
|
||||||
|
maxt = s.t
|
||||||
|
}
|
||||||
|
if s.t < mint {
|
||||||
|
mint = s.t
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case wp.histogramsOutput <- in.histogramSamples:
|
||||||
|
default:
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
h.updateMinOOOMaxOOOTime(mint, maxt)
|
h.updateMinOOOMaxOOOTime(mint, maxt)
|
||||||
|
|
||||||
return unknownRefs
|
return unknownRefs, unknownHistogramRefs
|
||||||
}
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
|
|
@ -389,6 +389,7 @@ func TestOOOHeadChunkReader_LabelValues(t *testing.T) {
|
||||||
func testOOOHeadChunkReader_LabelValues(t *testing.T, scenario sampleTypeScenario) {
|
func testOOOHeadChunkReader_LabelValues(t *testing.T, scenario sampleTypeScenario) {
|
||||||
chunkRange := int64(2000)
|
chunkRange := int64(2000)
|
||||||
head, _ := newTestHead(t, chunkRange, wlog.CompressionNone, true)
|
head, _ := newTestHead(t, chunkRange, wlog.CompressionNone, true)
|
||||||
|
head.opts.EnableOOONativeHistograms.Store(true)
|
||||||
t.Cleanup(func() { require.NoError(t, head.Close()) })
|
t.Cleanup(func() { require.NoError(t, head.Close()) })
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
@ -493,6 +494,8 @@ func testOOOHeadChunkReader_Chunk(t *testing.T, scenario sampleTypeScenario) {
|
||||||
opts := DefaultOptions()
|
opts := DefaultOptions()
|
||||||
opts.OutOfOrderCapMax = 5
|
opts.OutOfOrderCapMax = 5
|
||||||
opts.OutOfOrderTimeWindow = 120 * time.Minute.Milliseconds()
|
opts.OutOfOrderTimeWindow = 120 * time.Minute.Milliseconds()
|
||||||
|
opts.EnableNativeHistograms = true
|
||||||
|
opts.EnableOOONativeHistograms = true
|
||||||
|
|
||||||
s1 := labels.FromStrings("l", "v1")
|
s1 := labels.FromStrings("l", "v1")
|
||||||
minutes := func(m int64) int64 { return m * time.Minute.Milliseconds() }
|
minutes := func(m int64) int64 { return m * time.Minute.Milliseconds() }
|
||||||
|
@ -902,6 +905,8 @@ func testOOOHeadChunkReader_Chunk_ConsistentQueryResponseDespiteOfHeadExpanding(
|
||||||
opts := DefaultOptions()
|
opts := DefaultOptions()
|
||||||
opts.OutOfOrderCapMax = 5
|
opts.OutOfOrderCapMax = 5
|
||||||
opts.OutOfOrderTimeWindow = 120 * time.Minute.Milliseconds()
|
opts.OutOfOrderTimeWindow = 120 * time.Minute.Milliseconds()
|
||||||
|
opts.EnableNativeHistograms = true
|
||||||
|
opts.EnableOOONativeHistograms = true
|
||||||
|
|
||||||
s1 := labels.FromStrings("l", "v1")
|
s1 := labels.FromStrings("l", "v1")
|
||||||
minutes := func(m int64) int64 { return m * time.Minute.Milliseconds() }
|
minutes := func(m int64) int64 { return m * time.Minute.Milliseconds() }
|
||||||
|
|
|
@ -28,15 +28,14 @@ import (
|
||||||
const testMaxSize int = 32
|
const testMaxSize int = 32
|
||||||
|
|
||||||
// Formulas chosen to make testing easy.
|
// Formulas chosen to make testing easy.
|
||||||
func valEven(pos int) int { return pos*2 + 2 } // s[0]=2, s[1]=4, s[2]=6, ..., s[31]=64 - Predictable pre-existing values
|
// Formulas chosen to make testing easy.
|
||||||
func valOdd(pos int) int { return pos*2 + 1 } // s[0]=1, s[1]=3, s[2]=5, ..., s[31]=63 - New values will interject at chosen position because they sort before the pre-existing vals.
|
func valEven(pos int) int64 { return int64(pos*2 + 2) } // s[0]=2, s[1]=4, s[2]=6, ..., s[31]=64 - Predictable pre-existing values
|
||||||
|
func valOdd(pos int) int64 { return int64(pos*2 + 1) } // s[0]=1, s[1]=3, s[2]=5, ..., s[31]=63 - New values will interject at chosen position because they sort before the pre-existing vals.
|
||||||
|
|
||||||
func samplify(v int) sample { return sample{int64(v), float64(v), nil, nil} }
|
func makeEvenSampleSlice(n int, sampleFunc func(ts int64) sample) []sample {
|
||||||
|
|
||||||
func makeEvenSampleSlice(n int) []sample {
|
|
||||||
s := make([]sample, n)
|
s := make([]sample, n)
|
||||||
for i := 0; i < n; i++ {
|
for i := 0; i < n; i++ {
|
||||||
s[i] = samplify(valEven(i))
|
s[i] = sampleFunc(valEven(i))
|
||||||
}
|
}
|
||||||
return s
|
return s
|
||||||
}
|
}
|
||||||
|
@ -45,8 +44,36 @@ func makeEvenSampleSlice(n int) []sample {
|
||||||
// - Number of pre-existing samples anywhere from 0 to testMaxSize-1.
|
// - Number of pre-existing samples anywhere from 0 to testMaxSize-1.
|
||||||
// - Insert new sample before first pre-existing samples, after the last, and anywhere in between.
|
// - Insert new sample before first pre-existing samples, after the last, and anywhere in between.
|
||||||
// - With a chunk initial capacity of testMaxSize/8 and testMaxSize, which lets us test non-full and full chunks, and chunks that need to expand themselves.
|
// - With a chunk initial capacity of testMaxSize/8 and testMaxSize, which lets us test non-full and full chunks, and chunks that need to expand themselves.
|
||||||
// Note: In all samples used, t always equals v in numeric value. when we talk about 'value' we just refer to a value that will be used for both sample.t and sample.v.
|
|
||||||
func TestOOOInsert(t *testing.T) {
|
func TestOOOInsert(t *testing.T) {
|
||||||
|
scenarios := map[string]struct {
|
||||||
|
sampleFunc func(ts int64) sample
|
||||||
|
}{
|
||||||
|
"float": {
|
||||||
|
sampleFunc: func(ts int64) sample {
|
||||||
|
return sample{t: ts, f: float64(ts)}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"integer histogram": {
|
||||||
|
sampleFunc: func(ts int64) sample {
|
||||||
|
return sample{t: ts, h: tsdbutil.GenerateTestHistogram(int(ts))}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"float histogram": {
|
||||||
|
sampleFunc: func(ts int64) sample {
|
||||||
|
return sample{t: ts, fh: tsdbutil.GenerateTestFloatHistogram(int(ts))}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for name, scenario := range scenarios {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
testOOOInsert(t, scenario.sampleFunc)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testOOOInsert(t *testing.T,
|
||||||
|
sampleFunc func(ts int64) sample,
|
||||||
|
) {
|
||||||
for numPreExisting := 0; numPreExisting <= testMaxSize; numPreExisting++ {
|
for numPreExisting := 0; numPreExisting <= testMaxSize; numPreExisting++ {
|
||||||
// For example, if we have numPreExisting 2, then:
|
// For example, if we have numPreExisting 2, then:
|
||||||
// chunk.samples indexes filled 0 1
|
// chunk.samples indexes filled 0 1
|
||||||
|
@ -56,20 +83,21 @@ func TestOOOInsert(t *testing.T) {
|
||||||
|
|
||||||
for insertPos := 0; insertPos <= numPreExisting; insertPos++ {
|
for insertPos := 0; insertPos <= numPreExisting; insertPos++ {
|
||||||
chunk := NewOOOChunk()
|
chunk := NewOOOChunk()
|
||||||
chunk.samples = makeEvenSampleSlice(numPreExisting)
|
chunk.samples = make([]sample, numPreExisting)
|
||||||
newSample := samplify(valOdd(insertPos))
|
chunk.samples = makeEvenSampleSlice(numPreExisting, sampleFunc)
|
||||||
chunk.Insert(newSample.t, newSample.f, nil, nil)
|
newSample := sampleFunc(valOdd(insertPos))
|
||||||
|
chunk.Insert(newSample.t, newSample.f, newSample.h, newSample.fh)
|
||||||
|
|
||||||
var expSamples []sample
|
var expSamples []sample
|
||||||
// Our expected new samples slice, will be first the original samples.
|
// Our expected new samples slice, will be first the original samples.
|
||||||
for i := 0; i < insertPos; i++ {
|
for i := 0; i < insertPos; i++ {
|
||||||
expSamples = append(expSamples, samplify(valEven(i)))
|
expSamples = append(expSamples, sampleFunc(valEven(i)))
|
||||||
}
|
}
|
||||||
// Then the new sample.
|
// Then the new sample.
|
||||||
expSamples = append(expSamples, newSample)
|
expSamples = append(expSamples, newSample)
|
||||||
// Followed by any original samples that were pushed back by the new one.
|
// Followed by any original samples that were pushed back by the new one.
|
||||||
for i := insertPos; i < numPreExisting; i++ {
|
for i := insertPos; i < numPreExisting; i++ {
|
||||||
expSamples = append(expSamples, samplify(valEven(i)))
|
expSamples = append(expSamples, sampleFunc(valEven(i)))
|
||||||
}
|
}
|
||||||
|
|
||||||
require.Equal(t, expSamples, chunk.samples, "numPreExisting %d, insertPos %d", numPreExisting, insertPos)
|
require.Equal(t, expSamples, chunk.samples, "numPreExisting %d, insertPos %d", numPreExisting, insertPos)
|
||||||
|
@ -81,17 +109,46 @@ func TestOOOInsert(t *testing.T) {
|
||||||
// pre-existing samples, with between 1 and testMaxSize pre-existing samples and
|
// pre-existing samples, with between 1 and testMaxSize pre-existing samples and
|
||||||
// with a chunk initial capacity of testMaxSize/8 and testMaxSize, which lets us test non-full and full chunks, and chunks that need to expand themselves.
|
// with a chunk initial capacity of testMaxSize/8 and testMaxSize, which lets us test non-full and full chunks, and chunks that need to expand themselves.
|
||||||
func TestOOOInsertDuplicate(t *testing.T) {
|
func TestOOOInsertDuplicate(t *testing.T) {
|
||||||
|
scenarios := map[string]struct {
|
||||||
|
sampleFunc func(ts int64) sample
|
||||||
|
}{
|
||||||
|
"float": {
|
||||||
|
sampleFunc: func(ts int64) sample {
|
||||||
|
return sample{t: ts, f: float64(ts)}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"integer histogram": {
|
||||||
|
sampleFunc: func(ts int64) sample {
|
||||||
|
return sample{t: ts, h: tsdbutil.GenerateTestHistogram(int(ts))}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"float histogram": {
|
||||||
|
sampleFunc: func(ts int64) sample {
|
||||||
|
return sample{t: ts, fh: tsdbutil.GenerateTestFloatHistogram(int(ts))}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for name, scenario := range scenarios {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
testOOOInsertDuplicate(t, scenario.sampleFunc)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testOOOInsertDuplicate(t *testing.T,
|
||||||
|
sampleFunc func(ts int64) sample,
|
||||||
|
) {
|
||||||
for num := 1; num <= testMaxSize; num++ {
|
for num := 1; num <= testMaxSize; num++ {
|
||||||
for dupPos := 0; dupPos < num; dupPos++ {
|
for dupPos := 0; dupPos < num; dupPos++ {
|
||||||
chunk := NewOOOChunk()
|
chunk := NewOOOChunk()
|
||||||
chunk.samples = makeEvenSampleSlice(num)
|
chunk.samples = makeEvenSampleSlice(num, sampleFunc)
|
||||||
|
|
||||||
dupSample := chunk.samples[dupPos]
|
dupSample := chunk.samples[dupPos]
|
||||||
dupSample.f = 0.123
|
dupSample.f = 0.123
|
||||||
|
|
||||||
ok := chunk.Insert(dupSample.t, dupSample.f, nil, nil)
|
ok := chunk.Insert(dupSample.t, dupSample.f, dupSample.h, dupSample.fh)
|
||||||
|
|
||||||
expSamples := makeEvenSampleSlice(num) // We expect no change.
|
expSamples := makeEvenSampleSlice(num, sampleFunc) // We expect no change.
|
||||||
require.False(t, ok)
|
require.False(t, ok)
|
||||||
require.Equal(t, expSamples, chunk.samples, "num %d, dupPos %d", num, dupPos)
|
require.Equal(t, expSamples, chunk.samples, "num %d, dupPos %d", num, dupPos)
|
||||||
}
|
}
|
||||||
|
|
|
@ -16,6 +16,8 @@ package tsdb
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/prometheus/prometheus/tsdb/tsdbutil"
|
||||||
|
|
||||||
prom_testutil "github.com/prometheus/client_golang/prometheus/testutil"
|
prom_testutil "github.com/prometheus/client_golang/prometheus/testutil"
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
@ -27,7 +29,11 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
float = "float"
|
float = "float"
|
||||||
|
intHistogram = "integer histogram"
|
||||||
|
floatHistogram = "float histogram"
|
||||||
|
gaugeIntHistogram = "gauge int histogram"
|
||||||
|
gaugeFloatHistogram = "gauge float histogram"
|
||||||
)
|
)
|
||||||
|
|
||||||
type testValue struct {
|
type testValue struct {
|
||||||
|
@ -42,7 +48,6 @@ type sampleTypeScenario struct {
|
||||||
sampleFunc func(ts, value int64) sample
|
sampleFunc func(ts, value int64) sample
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO: native histogram sample types will be added as part of out-of-order native histogram support; see #11220.
|
|
||||||
var sampleTypeScenarios = map[string]sampleTypeScenario{
|
var sampleTypeScenarios = map[string]sampleTypeScenario{
|
||||||
float: {
|
float: {
|
||||||
sampleType: sampleMetricTypeFloat,
|
sampleType: sampleMetricTypeFloat,
|
||||||
|
@ -55,50 +60,50 @@ var sampleTypeScenarios = map[string]sampleTypeScenario{
|
||||||
return sample{t: ts, f: float64(value)}
|
return sample{t: ts, f: float64(value)}
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
// intHistogram: {
|
intHistogram: {
|
||||||
// sampleType: sampleMetricTypeHistogram,
|
sampleType: sampleMetricTypeHistogram,
|
||||||
// appendFunc: func(appender storage.Appender, lbls labels.Labels, ts, value int64) (storage.SeriesRef, sample, error) {
|
appendFunc: func(appender storage.Appender, lbls labels.Labels, ts, value int64) (storage.SeriesRef, sample, error) {
|
||||||
// s := sample{t: ts, h: tsdbutil.GenerateTestHistogram(int(value))}
|
s := sample{t: ts, h: tsdbutil.GenerateTestHistogram(int(value))}
|
||||||
// ref, err := appender.AppendHistogram(0, lbls, ts, s.h, nil)
|
ref, err := appender.AppendHistogram(0, lbls, ts, s.h, nil)
|
||||||
// return ref, s, err
|
return ref, s, err
|
||||||
// },
|
},
|
||||||
// sampleFunc: func(ts, value int64) sample {
|
sampleFunc: func(ts, value int64) sample {
|
||||||
// return sample{t: ts, h: tsdbutil.GenerateTestHistogram(int(value))}
|
return sample{t: ts, h: tsdbutil.GenerateTestHistogram(int(value))}
|
||||||
// },
|
},
|
||||||
// },
|
},
|
||||||
// floatHistogram: {
|
floatHistogram: {
|
||||||
// sampleType: sampleMetricTypeHistogram,
|
sampleType: sampleMetricTypeHistogram,
|
||||||
// appendFunc: func(appender storage.Appender, lbls labels.Labels, ts, value int64) (storage.SeriesRef, sample, error) {
|
appendFunc: func(appender storage.Appender, lbls labels.Labels, ts, value int64) (storage.SeriesRef, sample, error) {
|
||||||
// s := sample{t: ts, fh: tsdbutil.GenerateTestFloatHistogram(int(value))}
|
s := sample{t: ts, fh: tsdbutil.GenerateTestFloatHistogram(int(value))}
|
||||||
// ref, err := appender.AppendHistogram(0, lbls, ts, nil, s.fh)
|
ref, err := appender.AppendHistogram(0, lbls, ts, nil, s.fh)
|
||||||
// return ref, s, err
|
return ref, s, err
|
||||||
// },
|
},
|
||||||
// sampleFunc: func(ts, value int64) sample {
|
sampleFunc: func(ts, value int64) sample {
|
||||||
// return sample{t: ts, fh: tsdbutil.GenerateTestFloatHistogram(int(value))}
|
return sample{t: ts, fh: tsdbutil.GenerateTestFloatHistogram(int(value))}
|
||||||
// },
|
},
|
||||||
// },
|
},
|
||||||
// gaugeIntHistogram: {
|
gaugeIntHistogram: {
|
||||||
// sampleType: sampleMetricTypeHistogram,
|
sampleType: sampleMetricTypeHistogram,
|
||||||
// appendFunc: func(appender storage.Appender, lbls labels.Labels, ts, value int64) (storage.SeriesRef, sample, error) {
|
appendFunc: func(appender storage.Appender, lbls labels.Labels, ts, value int64) (storage.SeriesRef, sample, error) {
|
||||||
// s := sample{t: ts, h: tsdbutil.GenerateTestGaugeHistogram(int(value))}
|
s := sample{t: ts, h: tsdbutil.GenerateTestGaugeHistogram(int(value))}
|
||||||
// ref, err := appender.AppendHistogram(0, lbls, ts, s.h, nil)
|
ref, err := appender.AppendHistogram(0, lbls, ts, s.h, nil)
|
||||||
// return ref, s, err
|
return ref, s, err
|
||||||
// },
|
},
|
||||||
// sampleFunc: func(ts, value int64) sample {
|
sampleFunc: func(ts, value int64) sample {
|
||||||
// return sample{t: ts, h: tsdbutil.GenerateTestGaugeHistogram(int(value))}
|
return sample{t: ts, h: tsdbutil.GenerateTestGaugeHistogram(int(value))}
|
||||||
// },
|
},
|
||||||
// },
|
},
|
||||||
// gaugeFloatHistogram: {
|
gaugeFloatHistogram: {
|
||||||
// sampleType: sampleMetricTypeHistogram,
|
sampleType: sampleMetricTypeHistogram,
|
||||||
// appendFunc: func(appender storage.Appender, lbls labels.Labels, ts, value int64) (storage.SeriesRef, sample, error) {
|
appendFunc: func(appender storage.Appender, lbls labels.Labels, ts, value int64) (storage.SeriesRef, sample, error) {
|
||||||
// s := sample{t: ts, fh: tsdbutil.GenerateTestGaugeFloatHistogram(int(value))}
|
s := sample{t: ts, fh: tsdbutil.GenerateTestGaugeFloatHistogram(int(value))}
|
||||||
// ref, err := appender.AppendHistogram(0, lbls, ts, nil, s.fh)
|
ref, err := appender.AppendHistogram(0, lbls, ts, nil, s.fh)
|
||||||
// return ref, s, err
|
return ref, s, err
|
||||||
// },
|
},
|
||||||
// sampleFunc: func(ts, value int64) sample {
|
sampleFunc: func(ts, value int64) sample {
|
||||||
// return sample{t: ts, fh: tsdbutil.GenerateTestGaugeFloatHistogram(int(value))}
|
return sample{t: ts, fh: tsdbutil.GenerateTestGaugeFloatHistogram(int(value))}
|
||||||
// },
|
},
|
||||||
// },
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
// requireEqualSeries checks that the actual series are equal to the expected ones. It ignores the counter reset hints for histograms.
|
// requireEqualSeries checks that the actual series are equal to the expected ones. It ignores the counter reset hints for histograms.
|
||||||
|
|
Loading…
Reference in a new issue