From fed6f813770fb0cada529f8c84c4086c949b9ff5 Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Tue, 31 Oct 2023 16:58:42 -0400 Subject: [PATCH 01/10] scrape: Added trackTimestampsStaleness configuration option Add the ability to track staleness when an explicit timestamp is set. Useful for cAdvisor. Signed-off-by: Julien Pivotto --- config/config.go | 2 + docs/configuration/configuration.md | 8 ++ scrape/scrape.go | 165 +++++++++++++++------------- scrape/scrape_test.go | 98 ++++++++++++++++- 4 files changed, 194 insertions(+), 79 deletions(-) diff --git a/config/config.go b/config/config.go index 7824780c34..f0b359d8e1 100644 --- a/config/config.go +++ b/config/config.go @@ -484,6 +484,8 @@ type ScrapeConfig struct { HonorLabels bool `yaml:"honor_labels,omitempty"` // Indicator whether the scraped timestamps should be respected. HonorTimestamps bool `yaml:"honor_timestamps"` + // Indicator whether to track the staleness of the scraped timestamps. + TrackTimestampsStaleness bool `yaml:"track_timestamps_staleness"` // A set of query parameters with which the target is scraped. Params url.Values `yaml:"params,omitempty"` // How frequently to scrape the targets of this scrape config. diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index b55bb7d721..31d1b526fe 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -210,6 +210,14 @@ job_name: # by the target will be ignored. [ honor_timestamps: | default = true ] +# track_timestamps_staleness controls whether Prometheus tracks staleness of +# the metrics that have an explicit timestamps present in scraped data. +# +# If track_timestamps_staleness is set to "true", a staleness marker will be +# inserted in the TSDB when a metric is no longer present or the target +# is down. +[ track_timestamps_staleness: | default = false ] + # Configures the protocol scheme used for requests. [ scheme: | default = http ] diff --git a/scrape/scrape.go b/scrape/scrape.go index 299ffdc98b..5db309008c 100644 --- a/scrape/scrape.go +++ b/scrape/scrape.go @@ -261,18 +261,19 @@ type labelLimits struct { } type scrapeLoopOptions struct { - target *Target - scraper scraper - sampleLimit int - bucketLimit int - labelLimits *labelLimits - honorLabels bool - honorTimestamps bool - interval time.Duration - timeout time.Duration - scrapeClassicHistograms bool - mrc []*relabel.Config - cache *scrapeCache + target *Target + scraper scraper + sampleLimit int + bucketLimit int + labelLimits *labelLimits + honorLabels bool + honorTimestamps bool + trackTimestampsStaleness bool + interval time.Duration + timeout time.Duration + scrapeClassicHistograms bool + mrc []*relabel.Config + cache *scrapeCache } const maxAheadTime = 10 * time.Minute @@ -328,6 +329,7 @@ func newScrapePool(cfg *config.ScrapeConfig, app storage.Appendable, offsetSeed cache, offsetSeed, opts.honorTimestamps, + opts.trackTimestampsStaleness, opts.sampleLimit, opts.bucketLimit, opts.labelLimits, @@ -437,9 +439,10 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) error { labelNameLengthLimit: int(sp.config.LabelNameLengthLimit), labelValueLengthLimit: int(sp.config.LabelValueLengthLimit), } - honorLabels = sp.config.HonorLabels - honorTimestamps = sp.config.HonorTimestamps - mrc = sp.config.MetricRelabelConfigs + honorLabels = sp.config.HonorLabels + honorTimestamps = sp.config.HonorTimestamps + trackTimestampsStaleness = sp.config.TrackTimestampsStaleness + mrc = sp.config.MetricRelabelConfigs ) sp.targetMtx.Lock() @@ -463,17 +466,18 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) error { var ( s = &targetScraper{Target: t, client: sp.client, timeout: timeout, bodySizeLimit: bodySizeLimit, acceptHeader: acceptHeader} newLoop = sp.newLoop(scrapeLoopOptions{ - target: t, - scraper: s, - sampleLimit: sampleLimit, - bucketLimit: bucketLimit, - labelLimits: labelLimits, - honorLabels: honorLabels, - honorTimestamps: honorTimestamps, - mrc: mrc, - cache: cache, - interval: interval, - timeout: timeout, + target: t, + scraper: s, + sampleLimit: sampleLimit, + bucketLimit: bucketLimit, + labelLimits: labelLimits, + honorLabels: honorLabels, + honorTimestamps: honorTimestamps, + trackTimestampsStaleness: trackTimestampsStaleness, + mrc: mrc, + cache: cache, + interval: interval, + timeout: timeout, }) ) if err != nil { @@ -561,10 +565,11 @@ func (sp *scrapePool) sync(targets []*Target) { labelNameLengthLimit: int(sp.config.LabelNameLengthLimit), labelValueLengthLimit: int(sp.config.LabelValueLengthLimit), } - honorLabels = sp.config.HonorLabels - honorTimestamps = sp.config.HonorTimestamps - mrc = sp.config.MetricRelabelConfigs - scrapeClassicHistograms = sp.config.ScrapeClassicHistograms + honorLabels = sp.config.HonorLabels + honorTimestamps = sp.config.HonorTimestamps + trackTimestampsStaleness = sp.config.TrackTimestampsStaleness + mrc = sp.config.MetricRelabelConfigs + scrapeClassicHistograms = sp.config.ScrapeClassicHistograms ) sp.targetMtx.Lock() @@ -583,17 +588,18 @@ func (sp *scrapePool) sync(targets []*Target) { } s := &targetScraper{Target: t, client: sp.client, timeout: timeout, bodySizeLimit: bodySizeLimit, acceptHeader: acceptHeader} l := sp.newLoop(scrapeLoopOptions{ - target: t, - scraper: s, - sampleLimit: sampleLimit, - bucketLimit: bucketLimit, - labelLimits: labelLimits, - honorLabels: honorLabels, - honorTimestamps: honorTimestamps, - mrc: mrc, - interval: interval, - timeout: timeout, - scrapeClassicHistograms: scrapeClassicHistograms, + target: t, + scraper: s, + sampleLimit: sampleLimit, + bucketLimit: bucketLimit, + labelLimits: labelLimits, + honorLabels: honorLabels, + honorTimestamps: honorTimestamps, + trackTimestampsStaleness: trackTimestampsStaleness, + mrc: mrc, + interval: interval, + timeout: timeout, + scrapeClassicHistograms: scrapeClassicHistograms, }) if err != nil { l.setForcedError(err) @@ -900,21 +906,22 @@ type cacheEntry struct { } type scrapeLoop struct { - scraper scraper - l log.Logger - cache *scrapeCache - lastScrapeSize int - buffers *pool.Pool - offsetSeed uint64 - honorTimestamps bool - forcedErr error - forcedErrMtx sync.Mutex - sampleLimit int - bucketLimit int - labelLimits *labelLimits - interval time.Duration - timeout time.Duration - scrapeClassicHistograms bool + scraper scraper + l log.Logger + cache *scrapeCache + lastScrapeSize int + buffers *pool.Pool + offsetSeed uint64 + honorTimestamps bool + trackTimestampsStaleness bool + forcedErr error + forcedErrMtx sync.Mutex + sampleLimit int + bucketLimit int + labelLimits *labelLimits + interval time.Duration + timeout time.Duration + scrapeClassicHistograms bool appender func(ctx context.Context) storage.Appender sampleMutator labelsMutator @@ -1191,6 +1198,7 @@ func newScrapeLoop(ctx context.Context, cache *scrapeCache, offsetSeed uint64, honorTimestamps bool, + trackTimestampsStaleness bool, sampleLimit int, bucketLimit int, labelLimits *labelLimits, @@ -1224,26 +1232,27 @@ func newScrapeLoop(ctx context.Context, } sl := &scrapeLoop{ - scraper: sc, - buffers: buffers, - cache: cache, - appender: appender, - sampleMutator: sampleMutator, - reportSampleMutator: reportSampleMutator, - stopped: make(chan struct{}), - offsetSeed: offsetSeed, - l: l, - parentCtx: ctx, - appenderCtx: appenderCtx, - honorTimestamps: honorTimestamps, - sampleLimit: sampleLimit, - bucketLimit: bucketLimit, - labelLimits: labelLimits, - interval: interval, - timeout: timeout, - scrapeClassicHistograms: scrapeClassicHistograms, - reportExtraMetrics: reportExtraMetrics, - appendMetadataToWAL: appendMetadataToWAL, + scraper: sc, + buffers: buffers, + cache: cache, + appender: appender, + sampleMutator: sampleMutator, + reportSampleMutator: reportSampleMutator, + stopped: make(chan struct{}), + offsetSeed: offsetSeed, + l: l, + parentCtx: ctx, + appenderCtx: appenderCtx, + honorTimestamps: honorTimestamps, + trackTimestampsStaleness: trackTimestampsStaleness, + sampleLimit: sampleLimit, + bucketLimit: bucketLimit, + labelLimits: labelLimits, + interval: interval, + timeout: timeout, + scrapeClassicHistograms: scrapeClassicHistograms, + reportExtraMetrics: reportExtraMetrics, + appendMetadataToWAL: appendMetadataToWAL, } sl.ctx, sl.cancel = context.WithCancel(ctx) @@ -1690,7 +1699,7 @@ loop: } if !ok { - if parsedTimestamp == nil { + if parsedTimestamp == nil || sl.trackTimestampsStaleness { // Bypass staleness logic if there is an explicit timestamp. sl.cache.trackStaleness(hash, lset) } @@ -1771,7 +1780,7 @@ loop: func (sl *scrapeLoop) checkAddError(ce *cacheEntry, met []byte, tp *int64, err error, sampleLimitErr, bucketLimitErr *error, appErrs *appendErrors) (bool, error) { switch errors.Cause(err) { case nil: - if tp == nil && ce != nil { + if (tp == nil || sl.trackTimestampsStaleness) && ce != nil { sl.cache.trackStaleness(ce.hash, ce.lset) } return true, nil diff --git a/scrape/scrape_test.go b/scrape/scrape_test.go index 6c09b95e5a..c88d61ac1a 100644 --- a/scrape/scrape_test.go +++ b/scrape/scrape_test.go @@ -634,6 +634,7 @@ func TestScrapeLoopStopBeforeRun(t *testing.T) { nopMutator, nil, nil, 0, true, + false, 0, 0, nil, 1, @@ -707,6 +708,7 @@ func TestScrapeLoopStop(t *testing.T) { nil, 0, true, + false, 0, 0, nil, 10*time.Millisecond, @@ -784,6 +786,7 @@ func TestScrapeLoopRun(t *testing.T) { nil, 0, true, + false, 0, 0, nil, time.Second, @@ -840,6 +843,7 @@ func TestScrapeLoopRun(t *testing.T) { nil, 0, true, + false, 0, 0, nil, time.Second, @@ -900,6 +904,7 @@ func TestScrapeLoopForcedErr(t *testing.T) { nil, 0, true, + false, 0, 0, nil, time.Second, @@ -959,6 +964,7 @@ func TestScrapeLoopMetadata(t *testing.T) { cache, 0, true, + false, 0, 0, nil, 0, @@ -1017,6 +1023,7 @@ func simpleTestScrapeLoop(t testing.TB) (context.Context, *scrapeLoop) { nil, 0, true, + false, 0, 0, nil, 0, @@ -1078,6 +1085,7 @@ func TestScrapeLoopFailWithInvalidLabelsAfterRelabel(t *testing.T) { nil, 0, true, + false, 0, 0, nil, 0, @@ -1157,6 +1165,7 @@ func TestScrapeLoopRunCreatesStaleMarkersOnFailedScrape(t *testing.T) { nil, 0, true, + false, 0, 0, nil, 10*time.Millisecond, @@ -1221,6 +1230,7 @@ func TestScrapeLoopRunCreatesStaleMarkersOnParseFailure(t *testing.T) { nil, 0, true, + false, 0, 0, nil, 10*time.Millisecond, @@ -1288,6 +1298,7 @@ func TestScrapeLoopCache(t *testing.T) { nil, 0, true, + false, 0, 0, nil, 10*time.Millisecond, @@ -1372,6 +1383,7 @@ func TestScrapeLoopCacheMemoryExhaustionProtection(t *testing.T) { nil, 0, true, + false, 0, 0, nil, 10*time.Millisecond, @@ -1487,6 +1499,7 @@ func TestScrapeLoopAppend(t *testing.T) { nil, 0, true, + false, 0, 0, nil, 0, @@ -1583,7 +1596,7 @@ func TestScrapeLoopAppendForConflictingPrefixedLabels(t *testing.T) { return mutateSampleLabels(l, &Target{labels: labels.FromStrings(tc.targetLabels...)}, false, nil) }, nil, - func(ctx context.Context) storage.Appender { return app }, nil, 0, true, 0, 0, nil, 0, 0, false, false, false, nil, false, + func(ctx context.Context) storage.Appender { return app }, nil, 0, true, false, 0, 0, nil, 0, 0, false, false, false, nil, false, ) slApp := sl.appender(context.Background()) _, _, _, err := sl.append(slApp, []byte(tc.exposedLabels), "", time.Date(2000, 1, 1, 1, 0, 0, 0, time.UTC)) @@ -1614,6 +1627,7 @@ func TestScrapeLoopAppendCacheEntryButErrNotFound(t *testing.T) { nil, 0, true, + false, 0, 0, nil, 0, @@ -1673,6 +1687,7 @@ func TestScrapeLoopAppendSampleLimit(t *testing.T) { nil, 0, true, + false, app.limit, 0, nil, 0, @@ -1751,6 +1766,7 @@ func TestScrapeLoop_HistogramBucketLimit(t *testing.T) { nil, 0, true, + false, app.limit, 0, nil, 0, @@ -1850,6 +1866,7 @@ func TestScrapeLoop_ChangingMetricString(t *testing.T) { nil, 0, true, + false, 0, 0, nil, 0, @@ -1899,6 +1916,7 @@ func TestScrapeLoopAppendStaleness(t *testing.T) { nil, 0, true, + false, 0, 0, nil, 0, @@ -1951,6 +1969,7 @@ func TestScrapeLoopAppendNoStalenessIfTimestamp(t *testing.T) { nil, 0, true, + false, 0, 0, nil, 0, @@ -2277,6 +2296,7 @@ metric: < nil, 0, true, + false, 0, 0, nil, 0, @@ -2365,6 +2385,7 @@ func TestScrapeLoopAppendExemplarSeries(t *testing.T) { nil, 0, true, + false, 0, 0, nil, 0, @@ -2418,6 +2439,7 @@ func TestScrapeLoopRunReportsTargetDownOnScrapeError(t *testing.T) { nil, 0, true, + false, 0, 0, nil, 10*time.Millisecond, @@ -2455,6 +2477,7 @@ func TestScrapeLoopRunReportsTargetDownOnInvalidUTF8(t *testing.T) { nil, 0, true, + false, 0, 0, nil, 10*time.Millisecond, @@ -2505,6 +2528,7 @@ func TestScrapeLoopAppendGracefullyIfAmendOrOutOfOrderOrOutOfBounds(t *testing.T nil, 0, true, + false, 0, 0, nil, 0, @@ -2551,6 +2575,7 @@ func TestScrapeLoopOutOfBoundsTimeError(t *testing.T) { nil, 0, true, + false, 0, 0, nil, 0, @@ -2840,6 +2865,7 @@ func TestScrapeLoop_RespectTimestamps(t *testing.T) { func(ctx context.Context) storage.Appender { return capp }, nil, 0, true, + false, 0, 0, nil, 0, @@ -2882,6 +2908,7 @@ func TestScrapeLoop_DiscardTimestamps(t *testing.T) { func(ctx context.Context) storage.Appender { return capp }, nil, 0, false, + false, 0, 0, nil, 0, @@ -2923,6 +2950,7 @@ func TestScrapeLoopDiscardDuplicateLabels(t *testing.T) { nil, 0, true, + false, 0, 0, nil, 0, @@ -2982,6 +3010,7 @@ func TestScrapeLoopDiscardUnnamedMetrics(t *testing.T) { nil, 0, true, + false, 0, 0, nil, 0, @@ -3246,6 +3275,7 @@ func TestScrapeAddFast(t *testing.T) { nil, 0, true, + false, 0, 0, nil, 0, @@ -3333,6 +3363,7 @@ func TestScrapeReportSingleAppender(t *testing.T) { nil, 0, true, + false, 0, 0, nil, 10*time.Millisecond, @@ -3536,6 +3567,7 @@ func TestScrapeLoopLabelLimit(t *testing.T) { nil, 0, true, + false, 0, 0, &test.labelLimits, 0, @@ -3596,3 +3628,67 @@ func TestTargetScrapeIntervalAndTimeoutRelabel(t *testing.T) { require.Equal(t, "3s", sp.ActiveTargets()[0].labels.Get(model.ScrapeIntervalLabel)) require.Equal(t, "750ms", sp.ActiveTargets()[0].labels.Get(model.ScrapeTimeoutLabel)) } + +func TestScrapeLoopRunCreatesStaleMarkersOnFailedScrapeForTimestampedMetrics(t *testing.T) { + appender := &collectResultAppender{} + var ( + signal = make(chan struct{}, 1) + scraper = &testScraper{} + app = func(ctx context.Context) storage.Appender { return appender } + ) + + ctx, cancel := context.WithCancel(context.Background()) + sl := newScrapeLoop(ctx, + scraper, + nil, nil, + nopMutator, + nopMutator, + app, + nil, + 0, + true, + true, + 0, 0, + nil, + 10*time.Millisecond, + time.Hour, + false, + false, + false, + nil, + false, + ) + // Succeed once, several failures, then stop. + numScrapes := 0 + + scraper.scrapeFunc = func(ctx context.Context, w io.Writer) error { + numScrapes++ + + switch numScrapes { + case 1: + w.Write([]byte(fmt.Sprintf("metric_a 42 %d\n", time.Now().UnixNano()/int64(time.Millisecond)))) + return nil + case 5: + cancel() + } + return errors.New("scrape failed") + } + + go func() { + sl.run(nil) + signal <- struct{}{} + }() + + select { + case <-signal: + case <-time.After(5 * time.Second): + t.Fatalf("Scrape wasn't stopped.") + } + + // 1 successfully scraped sample, 1 stale marker after first fail, 5 report samples for + // each scrape successful or not. + require.Equal(t, 27, len(appender.resultFloats), "Appended samples not as expected:\n%s", appender) + require.Equal(t, 42.0, appender.resultFloats[0].f, "Appended first sample not as expected") + require.True(t, value.IsStaleNaN(appender.resultFloats[6].f), + "Appended second sample not as expected. Wanted: stale NaN Got: %x", math.Float64bits(appender.resultFloats[6].f)) +} From 8b87f3bbf1f9f957fcdc7618cd19ea707a136f21 Mon Sep 17 00:00:00 2001 From: Levi Harrison Date: Wed, 1 Nov 2023 21:19:24 -0400 Subject: [PATCH 02/10] Release 2.48.0-rc.2 Signed-off-by: Levi Harrison --- CHANGELOG.md | 6 ++++++ VERSION | 2 +- web/ui/module/codemirror-promql/package.json | 4 ++-- web/ui/module/lezer-promql/package.json | 2 +- web/ui/package-lock.json | 18 +++++++++--------- web/ui/package.json | 2 +- web/ui/react-app/package.json | 4 ++-- 7 files changed, 22 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc14091cde..35f9f77682 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 2.48.0-rc.2 / 2023-11-02 + +* [ENHANCEMENT] Scraping: Add configuration option for tracking staleness of scraped timestamps. #13060 +* [BUGFIX] Storage: Fix crash caused by incorrect mixed samples handling. #13055 +* [BUGFIX] TSDB: Fix compactor failures by adding min time to histogram chunks. #13062 + ## 2.48.0-rc.1 / 2023-10-24 * [BUGFIX] PromQL: Reduce inefficiency introduced by warnings/annotations and temporarily remove possible non-counter warnings. #13012 diff --git a/VERSION b/VERSION index f2bfe03dba..10429c7ad0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.48.0-rc.1 +2.48.0-rc.2 diff --git a/web/ui/module/codemirror-promql/package.json b/web/ui/module/codemirror-promql/package.json index 0e62d640ab..408891028c 100644 --- a/web/ui/module/codemirror-promql/package.json +++ b/web/ui/module/codemirror-promql/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/codemirror-promql", - "version": "0.48.0-rc.1", + "version": "0.48.0-rc.2", "description": "a CodeMirror mode for the PromQL language", "types": "dist/esm/index.d.ts", "module": "dist/esm/index.js", @@ -29,7 +29,7 @@ }, "homepage": "https://github.com/prometheus/prometheus/blob/main/web/ui/module/codemirror-promql/README.md", "dependencies": { - "@prometheus-io/lezer-promql": "0.48.0-rc.1", + "@prometheus-io/lezer-promql": "0.48.0-rc.2", "lru-cache": "^7.18.3" }, "devDependencies": { diff --git a/web/ui/module/lezer-promql/package.json b/web/ui/module/lezer-promql/package.json index d38b3288de..9f4a19d007 100644 --- a/web/ui/module/lezer-promql/package.json +++ b/web/ui/module/lezer-promql/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/lezer-promql", - "version": "0.48.0-rc.1", + "version": "0.48.0-rc.2", "description": "lezer-based PromQL grammar", "main": "dist/index.cjs", "type": "module", diff --git a/web/ui/package-lock.json b/web/ui/package-lock.json index f951aba10d..43cda8b139 100644 --- a/web/ui/package-lock.json +++ b/web/ui/package-lock.json @@ -1,12 +1,12 @@ { "name": "prometheus-io", - "version": "0.48.0-rc.1", + "version": "0.48.0-rc.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "prometheus-io", - "version": "0.48.0-rc.1", + "version": "0.48.0-rc.2", "workspaces": [ "react-app", "module/*" @@ -30,10 +30,10 @@ }, "module/codemirror-promql": { "name": "@prometheus-io/codemirror-promql", - "version": "0.48.0-rc.1", + "version": "0.48.0-rc.2", "license": "Apache-2.0", "dependencies": { - "@prometheus-io/lezer-promql": "0.48.0-rc.1", + "@prometheus-io/lezer-promql": "0.48.0-rc.2", "lru-cache": "^7.18.3" }, "devDependencies": { @@ -70,7 +70,7 @@ }, "module/lezer-promql": { "name": "@prometheus-io/lezer-promql", - "version": "0.48.0-rc.1", + "version": "0.48.0-rc.2", "license": "Apache-2.0", "devDependencies": { "@lezer/generator": "^1.2.3", @@ -20764,7 +20764,7 @@ }, "react-app": { "name": "@prometheus-io/app", - "version": "0.48.0-rc.1", + "version": "0.48.0-rc.2", "dependencies": { "@codemirror/autocomplete": "^6.7.1", "@codemirror/commands": "^6.2.4", @@ -20782,7 +20782,7 @@ "@lezer/lr": "^1.3.6", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "0.48.0-rc.1", + "@prometheus-io/codemirror-promql": "0.48.0-rc.2", "bootstrap": "^4.6.2", "css.escape": "^1.5.1", "downshift": "^7.6.0", @@ -23422,7 +23422,7 @@ "@lezer/lr": "^1.3.6", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "0.48.0-rc.1", + "@prometheus-io/codemirror-promql": "0.48.0-rc.2", "@testing-library/react-hooks": "^7.0.2", "@types/enzyme": "^3.10.13", "@types/flot": "0.0.32", @@ -23486,7 +23486,7 @@ "@lezer/common": "^1.0.3", "@lezer/highlight": "^1.1.6", "@lezer/lr": "^1.3.6", - "@prometheus-io/lezer-promql": "0.48.0-rc.1", + "@prometheus-io/lezer-promql": "0.48.0-rc.2", "isomorphic-fetch": "^3.0.0", "lru-cache": "^7.18.3", "nock": "^13.3.1" diff --git a/web/ui/package.json b/web/ui/package.json index 76b5ad2a89..9b4dfbc628 100644 --- a/web/ui/package.json +++ b/web/ui/package.json @@ -28,5 +28,5 @@ "ts-jest": "^29.1.0", "typescript": "^4.9.5" }, - "version": "0.48.0-rc.1" + "version": "0.48.0-rc.2" } diff --git a/web/ui/react-app/package.json b/web/ui/react-app/package.json index 3d1de883a5..de0d0e3150 100644 --- a/web/ui/react-app/package.json +++ b/web/ui/react-app/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/app", - "version": "0.48.0-rc.1", + "version": "0.48.0-rc.2", "private": true, "dependencies": { "@codemirror/autocomplete": "^6.7.1", @@ -19,7 +19,7 @@ "@lezer/lr": "^1.3.6", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "0.48.0-rc.1", + "@prometheus-io/codemirror-promql": "0.48.0-rc.2", "bootstrap": "^4.6.2", "css.escape": "^1.5.1", "downshift": "^7.6.0", From 46be85f2dc054920bcf0fa7718de099436ee8491 Mon Sep 17 00:00:00 2001 From: Jeanette Tan Date: Wed, 1 Nov 2023 19:53:41 +0800 Subject: [PATCH 03/10] Make TestPopulateWithDelSeriesIterator tests cover histogram types and check MinTime Signed-off-by: Jeanette Tan --- tsdb/querier_test.go | 237 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 195 insertions(+), 42 deletions(-) diff --git a/tsdb/querier_test.go b/tsdb/querier_test.go index fc6c688010..096e0ff729 100644 --- a/tsdb/querier_test.go +++ b/tsdb/querier_test.go @@ -1160,65 +1160,218 @@ func rmChunkRefs(chks []chunks.Meta) { // Regression for: https://github.com/prometheus/tsdb/pull/97 func TestPopulateWithDelSeriesIterator_DoubleSeek(t *testing.T) { - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{}, - []chunks.Sample{sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}}, - []chunks.Sample{sample{4, 4, nil, nil}, sample{5, 5, nil, nil}}, - ) - - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, chunkenc.ValFloat, it.Seek(1)) - require.Equal(t, chunkenc.ValFloat, it.Seek(2)) - require.Equal(t, chunkenc.ValFloat, it.Seek(2)) - ts, v := it.At() - require.Equal(t, int64(2), ts) - require.Equal(t, float64(2), v) + t.Run("float", func(t *testing.T) { + valType := chunkenc.ValFloat + f, chkMetas := createFakeReaderAndNotPopulatedChunks( + []chunks.Sample{}, + []chunks.Sample{sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}}, + []chunks.Sample{sample{4, 4, nil, nil}, sample{5, 5, nil, nil}}, + ) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, valType, it.Seek(1)) + require.Equal(t, valType, it.Seek(2)) + require.Equal(t, valType, it.Seek(2)) + ts, v := it.At() + require.Equal(t, int64(2), ts) + require.Equal(t, float64(2), v) + require.Equal(t, int64(0), chkMetas[0].MinTime) + require.Equal(t, int64(1), chkMetas[1].MinTime) + require.Equal(t, int64(4), chkMetas[2].MinTime) + }) + t.Run("histogram", func(t *testing.T) { + valType := chunkenc.ValHistogram + f, chkMetas := createFakeReaderAndNotPopulatedChunks( + []chunks.Sample{}, + []chunks.Sample{sample{1, 0, tsdbutil.GenerateTestHistogram(1), nil}, sample{2, 0, tsdbutil.GenerateTestHistogram(2), nil}, sample{3, 0, tsdbutil.GenerateTestHistogram(3), nil}}, + []chunks.Sample{sample{4, 0, tsdbutil.GenerateTestHistogram(4), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(5), nil}}, + ) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, valType, it.Seek(1)) + require.Equal(t, valType, it.Seek(2)) + require.Equal(t, valType, it.Seek(2)) + ts, h := it.AtHistogram() + require.Equal(t, int64(2), ts) + h.CounterResetHint = histogram.UnknownCounterReset + require.Equal(t, tsdbutil.GenerateTestHistogram(2), h) + require.Equal(t, int64(0), chkMetas[0].MinTime) + require.Equal(t, int64(1), chkMetas[1].MinTime) + require.Equal(t, int64(4), chkMetas[2].MinTime) + }) + t.Run("float histogram", func(t *testing.T) { + valType := chunkenc.ValFloatHistogram + f, chkMetas := createFakeReaderAndNotPopulatedChunks( + []chunks.Sample{}, + []chunks.Sample{sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(1)}, sample{2, 0, nil, tsdbutil.GenerateTestFloatHistogram(2)}, sample{3, 0, nil, tsdbutil.GenerateTestFloatHistogram(3)}}, + []chunks.Sample{sample{4, 0, nil, tsdbutil.GenerateTestFloatHistogram(4)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(5)}}, + ) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, valType, it.Seek(1)) + require.Equal(t, valType, it.Seek(2)) + require.Equal(t, valType, it.Seek(2)) + ts, h := it.AtFloatHistogram() + require.Equal(t, int64(2), ts) + h.CounterResetHint = histogram.UnknownCounterReset + require.Equal(t, tsdbutil.GenerateTestFloatHistogram(2), h) + require.Equal(t, int64(0), chkMetas[0].MinTime) + require.Equal(t, int64(1), chkMetas[1].MinTime) + require.Equal(t, int64(4), chkMetas[2].MinTime) + }) } // Regression when seeked chunks were still found via binary search and we always // skipped to the end when seeking a value in the current chunk. func TestPopulateWithDelSeriesIterator_SeekInCurrentChunk(t *testing.T) { - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{}, - []chunks.Sample{sample{1, 2, nil, nil}, sample{3, 4, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, - []chunks.Sample{}, - ) + t.Run("float", func(t *testing.T) { + valType := chunkenc.ValFloat + f, chkMetas := createFakeReaderAndNotPopulatedChunks( + []chunks.Sample{}, + []chunks.Sample{sample{1, 2, nil, nil}, sample{3, 4, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, + []chunks.Sample{}, + ) - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, chunkenc.ValFloat, it.Next()) - ts, v := it.At() - require.Equal(t, int64(1), ts) - require.Equal(t, float64(2), v) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, valType, it.Next()) + ts, v := it.At() + require.Equal(t, int64(1), ts) + require.Equal(t, float64(2), v) - require.Equal(t, chunkenc.ValFloat, it.Seek(4)) - ts, v = it.At() - require.Equal(t, int64(5), ts) - require.Equal(t, float64(6), v) + require.Equal(t, valType, it.Seek(4)) + ts, v = it.At() + require.Equal(t, int64(5), ts) + require.Equal(t, float64(6), v) + + require.Equal(t, int64(0), chkMetas[0].MinTime) + require.Equal(t, int64(1), chkMetas[1].MinTime) + require.Equal(t, int64(0), chkMetas[2].MinTime) + }) + t.Run("histogram", func(t *testing.T) { + valType := chunkenc.ValHistogram + f, chkMetas := createFakeReaderAndNotPopulatedChunks( + []chunks.Sample{}, + []chunks.Sample{sample{1, 0, tsdbutil.GenerateTestHistogram(2), nil}, sample{3, 0, tsdbutil.GenerateTestHistogram(4), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{7, 0, tsdbutil.GenerateTestHistogram(8), nil}}, + []chunks.Sample{}, + ) + + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, valType, it.Next()) + ts, h := it.AtHistogram() + require.Equal(t, int64(1), ts) + require.Equal(t, tsdbutil.GenerateTestHistogram(2), h) + + require.Equal(t, valType, it.Seek(4)) + ts, h = it.AtHistogram() + require.Equal(t, int64(5), ts) + h.CounterResetHint = histogram.UnknownCounterReset + require.Equal(t, tsdbutil.GenerateTestHistogram(6), h) + + require.Equal(t, int64(0), chkMetas[0].MinTime) + require.Equal(t, int64(1), chkMetas[1].MinTime) + require.Equal(t, int64(0), chkMetas[2].MinTime) + }) + t.Run("float histogram", func(t *testing.T) { + valType := chunkenc.ValFloatHistogram + f, chkMetas := createFakeReaderAndNotPopulatedChunks( + []chunks.Sample{}, + []chunks.Sample{sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(2)}, sample{3, 0, nil, tsdbutil.GenerateTestFloatHistogram(4)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{7, 0, nil, tsdbutil.GenerateTestFloatHistogram(8)}}, + []chunks.Sample{}, + ) + + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, valType, it.Next()) + ts, h := it.AtFloatHistogram() + require.Equal(t, int64(1), ts) + require.Equal(t, tsdbutil.GenerateTestFloatHistogram(2), h) + + require.Equal(t, valType, it.Seek(4)) + ts, h = it.AtFloatHistogram() + require.Equal(t, int64(5), ts) + h.CounterResetHint = histogram.UnknownCounterReset + require.Equal(t, tsdbutil.GenerateTestFloatHistogram(6), h) + + require.Equal(t, int64(0), chkMetas[0].MinTime) + require.Equal(t, int64(1), chkMetas[1].MinTime) + require.Equal(t, int64(0), chkMetas[2].MinTime) + }) } func TestPopulateWithDelSeriesIterator_SeekWithMinTime(t *testing.T) { - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{6, 8, nil, nil}}, - ) + t.Run("float", func(t *testing.T) { + valType := chunkenc.ValFloat + f, chkMetas := createFakeReaderAndNotPopulatedChunks( + []chunks.Sample{sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{6, 8, nil, nil}}, + ) - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, chunkenc.ValNone, it.Seek(7)) - require.Equal(t, chunkenc.ValFloat, it.Seek(3)) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, chunkenc.ValNone, it.Seek(7)) + require.Equal(t, valType, it.Seek(3)) + require.Equal(t, int64(1), chkMetas[0].MinTime) + }) + t.Run("histogram", func(t *testing.T) { + valType := chunkenc.ValHistogram + f, chkMetas := createFakeReaderAndNotPopulatedChunks( + []chunks.Sample{sample{1, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{6, 0, tsdbutil.GenerateTestHistogram(8), nil}}, + ) + + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, chunkenc.ValNone, it.Seek(7)) + require.Equal(t, valType, it.Seek(3)) + require.Equal(t, int64(1), chkMetas[0].MinTime) + }) + t.Run("float histogram", func(t *testing.T) { + valType := chunkenc.ValFloatHistogram + f, chkMetas := createFakeReaderAndNotPopulatedChunks( + []chunks.Sample{sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{6, 0, nil, tsdbutil.GenerateTestFloatHistogram(8)}}, + ) + + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, chunkenc.ValNone, it.Seek(7)) + require.Equal(t, valType, it.Seek(3)) + require.Equal(t, int64(1), chkMetas[0].MinTime) + }) } // Regression when calling Next() with a time bounded to fit within two samples. // Seek gets called and advances beyond the max time, which was just accepted as a valid sample. func TestPopulateWithDelSeriesIterator_NextWithMinTime(t *testing.T) { - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, - ) + t.Run("float", func(t *testing.T) { + f, chkMetas := createFakeReaderAndNotPopulatedChunks( + []chunks.Sample{sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, + ) - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, tombstones.Intervals{{Mint: math.MinInt64, Maxt: 2}}.Add(tombstones.Interval{Mint: 4, Maxt: math.MaxInt64})) - require.Equal(t, chunkenc.ValNone, it.Next()) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, tombstones.Intervals{{Mint: math.MinInt64, Maxt: 2}}.Add(tombstones.Interval{Mint: 4, Maxt: math.MaxInt64})) + require.Equal(t, chunkenc.ValNone, it.Next()) + require.Equal(t, int64(1), chkMetas[0].MinTime) + }) + t.Run("histogram", func(t *testing.T) { + f, chkMetas := createFakeReaderAndNotPopulatedChunks( + []chunks.Sample{sample{1, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{7, 0, tsdbutil.GenerateTestHistogram(8), nil}}, + ) + + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, tombstones.Intervals{{Mint: math.MinInt64, Maxt: 2}}.Add(tombstones.Interval{Mint: 4, Maxt: math.MaxInt64})) + require.Equal(t, chunkenc.ValNone, it.Next()) + require.Equal(t, int64(1), chkMetas[0].MinTime) + }) + t.Run("float histogram", func(t *testing.T) { + f, chkMetas := createFakeReaderAndNotPopulatedChunks( + []chunks.Sample{sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{7, 0, nil, tsdbutil.GenerateTestFloatHistogram(8)}}, + ) + + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, tombstones.Intervals{{Mint: math.MinInt64, Maxt: 2}}.Add(tombstones.Interval{Mint: 4, Maxt: math.MaxInt64})) + require.Equal(t, chunkenc.ValNone, it.Next()) + require.Equal(t, int64(1), chkMetas[0].MinTime) + }) } // Test the cost of merging series sets for different number of merged sets and their size. From 04aabdd7cc86fedba3dd2e614fa5289b746bda28 Mon Sep 17 00:00:00 2001 From: Jeanette Tan Date: Wed, 1 Nov 2023 19:53:41 +0800 Subject: [PATCH 04/10] Refactor TestPopulateWithDelSeriesIterator unit tests to reuse more code Signed-off-by: Jeanette Tan --- tsdb/querier_test.go | 385 +++++++++++++++++++++---------------------- 1 file changed, 189 insertions(+), 196 deletions(-) diff --git a/tsdb/querier_test.go b/tsdb/querier_test.go index 096e0ff729..1c50724d7c 100644 --- a/tsdb/querier_test.go +++ b/tsdb/querier_test.go @@ -1158,220 +1158,213 @@ func rmChunkRefs(chks []chunks.Meta) { } } +func checkCurrVal(t *testing.T, valType chunkenc.ValueType, it *populateWithDelSeriesIterator, expectedTs, expectedValue int) { + switch valType { + case chunkenc.ValFloat: + ts, v := it.At() + require.Equal(t, int64(expectedTs), ts) + require.Equal(t, float64(expectedValue), v) + case chunkenc.ValHistogram: + ts, h := it.AtHistogram() + require.Equal(t, int64(expectedTs), ts) + h.CounterResetHint = histogram.UnknownCounterReset + require.Equal(t, tsdbutil.GenerateTestHistogram(expectedValue), h) + case chunkenc.ValFloatHistogram: + ts, h := it.AtFloatHistogram() + require.Equal(t, int64(expectedTs), ts) + h.CounterResetHint = histogram.UnknownCounterReset + require.Equal(t, tsdbutil.GenerateTestFloatHistogram(expectedValue), h) + default: + panic("unexpected value type") + } +} + // Regression for: https://github.com/prometheus/tsdb/pull/97 func TestPopulateWithDelSeriesIterator_DoubleSeek(t *testing.T) { - t.Run("float", func(t *testing.T) { - valType := chunkenc.ValFloat - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{}, - []chunks.Sample{sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}}, - []chunks.Sample{sample{4, 4, nil, nil}, sample{5, 5, nil, nil}}, - ) - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, valType, it.Seek(1)) - require.Equal(t, valType, it.Seek(2)) - require.Equal(t, valType, it.Seek(2)) - ts, v := it.At() - require.Equal(t, int64(2), ts) - require.Equal(t, float64(2), v) - require.Equal(t, int64(0), chkMetas[0].MinTime) - require.Equal(t, int64(1), chkMetas[1].MinTime) - require.Equal(t, int64(4), chkMetas[2].MinTime) - }) - t.Run("histogram", func(t *testing.T) { - valType := chunkenc.ValHistogram - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{}, - []chunks.Sample{sample{1, 0, tsdbutil.GenerateTestHistogram(1), nil}, sample{2, 0, tsdbutil.GenerateTestHistogram(2), nil}, sample{3, 0, tsdbutil.GenerateTestHistogram(3), nil}}, - []chunks.Sample{sample{4, 0, tsdbutil.GenerateTestHistogram(4), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(5), nil}}, - ) - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, valType, it.Seek(1)) - require.Equal(t, valType, it.Seek(2)) - require.Equal(t, valType, it.Seek(2)) - ts, h := it.AtHistogram() - require.Equal(t, int64(2), ts) - h.CounterResetHint = histogram.UnknownCounterReset - require.Equal(t, tsdbutil.GenerateTestHistogram(2), h) - require.Equal(t, int64(0), chkMetas[0].MinTime) - require.Equal(t, int64(1), chkMetas[1].MinTime) - require.Equal(t, int64(4), chkMetas[2].MinTime) - }) - t.Run("float histogram", func(t *testing.T) { - valType := chunkenc.ValFloatHistogram - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{}, - []chunks.Sample{sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(1)}, sample{2, 0, nil, tsdbutil.GenerateTestFloatHistogram(2)}, sample{3, 0, nil, tsdbutil.GenerateTestFloatHistogram(3)}}, - []chunks.Sample{sample{4, 0, nil, tsdbutil.GenerateTestFloatHistogram(4)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(5)}}, - ) - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, valType, it.Seek(1)) - require.Equal(t, valType, it.Seek(2)) - require.Equal(t, valType, it.Seek(2)) - ts, h := it.AtFloatHistogram() - require.Equal(t, int64(2), ts) - h.CounterResetHint = histogram.UnknownCounterReset - require.Equal(t, tsdbutil.GenerateTestFloatHistogram(2), h) - require.Equal(t, int64(0), chkMetas[0].MinTime) - require.Equal(t, int64(1), chkMetas[1].MinTime) - require.Equal(t, int64(4), chkMetas[2].MinTime) - }) + cases := []struct { + name string + valType chunkenc.ValueType + chks [][]chunks.Sample + }{ + { + name: "float", + valType: chunkenc.ValFloat, + chks: [][]chunks.Sample{ + {}, + {sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}}, + {sample{4, 4, nil, nil}, sample{5, 5, nil, nil}}, + }, + }, + { + name: "histogram", + valType: chunkenc.ValHistogram, + chks: [][]chunks.Sample{ + {}, + {sample{1, 0, tsdbutil.GenerateTestHistogram(1), nil}, sample{2, 0, tsdbutil.GenerateTestHistogram(2), nil}, sample{3, 0, tsdbutil.GenerateTestHistogram(3), nil}}, + {sample{4, 0, tsdbutil.GenerateTestHistogram(4), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(5), nil}}, + }, + }, + { + name: "float histogram", + valType: chunkenc.ValFloatHistogram, + chks: [][]chunks.Sample{ + {}, + {sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(1)}, sample{2, 0, nil, tsdbutil.GenerateTestFloatHistogram(2)}, sample{3, 0, nil, tsdbutil.GenerateTestFloatHistogram(3)}}, + {sample{4, 0, nil, tsdbutil.GenerateTestFloatHistogram(4)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(5)}}, + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f, chkMetas := createFakeReaderAndNotPopulatedChunks(tc.chks...) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, tc.valType, it.Seek(1)) + require.Equal(t, tc.valType, it.Seek(2)) + require.Equal(t, tc.valType, it.Seek(2)) + checkCurrVal(t, tc.valType, it, 2, 2) + require.Equal(t, int64(0), chkMetas[0].MinTime) + require.Equal(t, int64(1), chkMetas[1].MinTime) + require.Equal(t, int64(4), chkMetas[2].MinTime) + }) + } } // Regression when seeked chunks were still found via binary search and we always // skipped to the end when seeking a value in the current chunk. func TestPopulateWithDelSeriesIterator_SeekInCurrentChunk(t *testing.T) { - t.Run("float", func(t *testing.T) { - valType := chunkenc.ValFloat - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{}, - []chunks.Sample{sample{1, 2, nil, nil}, sample{3, 4, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, - []chunks.Sample{}, - ) + cases := []struct { + name string + valType chunkenc.ValueType + chks [][]chunks.Sample + }{ + { + name: "float", + valType: chunkenc.ValFloat, + chks: [][]chunks.Sample{ + {}, + {sample{1, 2, nil, nil}, sample{3, 4, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, + {}, + }, + }, + { + name: "histogram", + valType: chunkenc.ValHistogram, + chks: [][]chunks.Sample{ + {}, + {sample{1, 0, tsdbutil.GenerateTestHistogram(2), nil}, sample{3, 0, tsdbutil.GenerateTestHistogram(4), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{7, 0, tsdbutil.GenerateTestHistogram(8), nil}}, + {}, + }, + }, + { + name: "float histogram", + valType: chunkenc.ValFloatHistogram, + chks: [][]chunks.Sample{ + {}, + {sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(2)}, sample{3, 0, nil, tsdbutil.GenerateTestFloatHistogram(4)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{7, 0, nil, tsdbutil.GenerateTestFloatHistogram(8)}}, + {}, + }, + }, + } - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, valType, it.Next()) - ts, v := it.At() - require.Equal(t, int64(1), ts) - require.Equal(t, float64(2), v) - - require.Equal(t, valType, it.Seek(4)) - ts, v = it.At() - require.Equal(t, int64(5), ts) - require.Equal(t, float64(6), v) - - require.Equal(t, int64(0), chkMetas[0].MinTime) - require.Equal(t, int64(1), chkMetas[1].MinTime) - require.Equal(t, int64(0), chkMetas[2].MinTime) - }) - t.Run("histogram", func(t *testing.T) { - valType := chunkenc.ValHistogram - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{}, - []chunks.Sample{sample{1, 0, tsdbutil.GenerateTestHistogram(2), nil}, sample{3, 0, tsdbutil.GenerateTestHistogram(4), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{7, 0, tsdbutil.GenerateTestHistogram(8), nil}}, - []chunks.Sample{}, - ) - - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, valType, it.Next()) - ts, h := it.AtHistogram() - require.Equal(t, int64(1), ts) - require.Equal(t, tsdbutil.GenerateTestHistogram(2), h) - - require.Equal(t, valType, it.Seek(4)) - ts, h = it.AtHistogram() - require.Equal(t, int64(5), ts) - h.CounterResetHint = histogram.UnknownCounterReset - require.Equal(t, tsdbutil.GenerateTestHistogram(6), h) - - require.Equal(t, int64(0), chkMetas[0].MinTime) - require.Equal(t, int64(1), chkMetas[1].MinTime) - require.Equal(t, int64(0), chkMetas[2].MinTime) - }) - t.Run("float histogram", func(t *testing.T) { - valType := chunkenc.ValFloatHistogram - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{}, - []chunks.Sample{sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(2)}, sample{3, 0, nil, tsdbutil.GenerateTestFloatHistogram(4)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{7, 0, nil, tsdbutil.GenerateTestFloatHistogram(8)}}, - []chunks.Sample{}, - ) - - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, valType, it.Next()) - ts, h := it.AtFloatHistogram() - require.Equal(t, int64(1), ts) - require.Equal(t, tsdbutil.GenerateTestFloatHistogram(2), h) - - require.Equal(t, valType, it.Seek(4)) - ts, h = it.AtFloatHistogram() - require.Equal(t, int64(5), ts) - h.CounterResetHint = histogram.UnknownCounterReset - require.Equal(t, tsdbutil.GenerateTestFloatHistogram(6), h) - - require.Equal(t, int64(0), chkMetas[0].MinTime) - require.Equal(t, int64(1), chkMetas[1].MinTime) - require.Equal(t, int64(0), chkMetas[2].MinTime) - }) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f, chkMetas := createFakeReaderAndNotPopulatedChunks(tc.chks...) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, tc.valType, it.Next()) + checkCurrVal(t, tc.valType, it, 1, 2) + require.Equal(t, tc.valType, it.Seek(4)) + checkCurrVal(t, tc.valType, it, 5, 6) + require.Equal(t, int64(0), chkMetas[0].MinTime) + require.Equal(t, int64(1), chkMetas[1].MinTime) + require.Equal(t, int64(0), chkMetas[2].MinTime) + }) + } } func TestPopulateWithDelSeriesIterator_SeekWithMinTime(t *testing.T) { - t.Run("float", func(t *testing.T) { - valType := chunkenc.ValFloat - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{6, 8, nil, nil}}, - ) + cases := []struct { + name string + valType chunkenc.ValueType + chks [][]chunks.Sample + }{ + { + name: "float", + valType: chunkenc.ValFloat, + chks: [][]chunks.Sample{ + {sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{6, 8, nil, nil}}, + }, + }, + { + name: "histogram", + valType: chunkenc.ValHistogram, + chks: [][]chunks.Sample{ + {sample{1, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{6, 0, tsdbutil.GenerateTestHistogram(8), nil}}, + }, + }, + { + name: "float histogram", + valType: chunkenc.ValFloatHistogram, + chks: [][]chunks.Sample{ + {sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{6, 0, nil, tsdbutil.GenerateTestFloatHistogram(8)}}, + }, + }, + } - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, chunkenc.ValNone, it.Seek(7)) - require.Equal(t, valType, it.Seek(3)) - require.Equal(t, int64(1), chkMetas[0].MinTime) - }) - t.Run("histogram", func(t *testing.T) { - valType := chunkenc.ValHistogram - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{sample{1, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{6, 0, tsdbutil.GenerateTestHistogram(8), nil}}, - ) - - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, chunkenc.ValNone, it.Seek(7)) - require.Equal(t, valType, it.Seek(3)) - require.Equal(t, int64(1), chkMetas[0].MinTime) - }) - t.Run("float histogram", func(t *testing.T) { - valType := chunkenc.ValFloatHistogram - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{6, 0, nil, tsdbutil.GenerateTestFloatHistogram(8)}}, - ) - - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, chunkenc.ValNone, it.Seek(7)) - require.Equal(t, valType, it.Seek(3)) - require.Equal(t, int64(1), chkMetas[0].MinTime) - }) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f, chkMetas := createFakeReaderAndNotPopulatedChunks(tc.chks...) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, chunkenc.ValNone, it.Seek(7)) + require.Equal(t, tc.valType, it.Seek(3)) + require.Equal(t, int64(1), chkMetas[0].MinTime) + }) + } } // Regression when calling Next() with a time bounded to fit within two samples. // Seek gets called and advances beyond the max time, which was just accepted as a valid sample. func TestPopulateWithDelSeriesIterator_NextWithMinTime(t *testing.T) { - t.Run("float", func(t *testing.T) { - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, - ) + cases := []struct { + name string + valType chunkenc.ValueType + chks [][]chunks.Sample + }{ + { + name: "float", + valType: chunkenc.ValFloat, + chks: [][]chunks.Sample{ + {sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, + }, + }, + { + name: "histogram", + valType: chunkenc.ValHistogram, + chks: [][]chunks.Sample{ + {sample{1, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{7, 0, tsdbutil.GenerateTestHistogram(8), nil}}, + }, + }, + { + name: "float histogram", + valType: chunkenc.ValFloatHistogram, + chks: [][]chunks.Sample{ + {sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{7, 0, nil, tsdbutil.GenerateTestFloatHistogram(8)}}, + }, + }, + } - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, tombstones.Intervals{{Mint: math.MinInt64, Maxt: 2}}.Add(tombstones.Interval{Mint: 4, Maxt: math.MaxInt64})) - require.Equal(t, chunkenc.ValNone, it.Next()) - require.Equal(t, int64(1), chkMetas[0].MinTime) - }) - t.Run("histogram", func(t *testing.T) { - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{sample{1, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{7, 0, tsdbutil.GenerateTestHistogram(8), nil}}, - ) - - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, tombstones.Intervals{{Mint: math.MinInt64, Maxt: 2}}.Add(tombstones.Interval{Mint: 4, Maxt: math.MaxInt64})) - require.Equal(t, chunkenc.ValNone, it.Next()) - require.Equal(t, int64(1), chkMetas[0].MinTime) - }) - t.Run("float histogram", func(t *testing.T) { - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{7, 0, nil, tsdbutil.GenerateTestFloatHistogram(8)}}, - ) - - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, tombstones.Intervals{{Mint: math.MinInt64, Maxt: 2}}.Add(tombstones.Interval{Mint: 4, Maxt: math.MaxInt64})) - require.Equal(t, chunkenc.ValNone, it.Next()) - require.Equal(t, int64(1), chkMetas[0].MinTime) - }) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f, chkMetas := createFakeReaderAndNotPopulatedChunks(tc.chks...) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, tombstones.Intervals{{Mint: math.MinInt64, Maxt: 2}}.Add(tombstones.Interval{Mint: 4, Maxt: math.MaxInt64})) + require.Equal(t, chunkenc.ValNone, it.Next()) + require.Equal(t, int64(1), chkMetas[0].MinTime) + }) + } } // Test the cost of merging series sets for different number of merged sets and their size. From 7a4a1127b70d91ee3805238c61957ada1874b13a Mon Sep 17 00:00:00 2001 From: Jeanette Tan Date: Wed, 1 Nov 2023 19:53:41 +0800 Subject: [PATCH 05/10] Expand TestPopulateWithTombSeriesIterators to test min max times of chunks, including mixed chunks Signed-off-by: Jeanette Tan --- tsdb/querier_test.go | 124 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 122 insertions(+), 2 deletions(-) diff --git a/tsdb/querier_test.go b/tsdb/querier_test.go index 1c50724d7c..5ad5003f1f 100644 --- a/tsdb/querier_test.go +++ b/tsdb/querier_test.go @@ -694,12 +694,16 @@ func (r *fakeChunksReader) Chunk(meta chunks.Meta) (chunkenc.Chunk, error) { } func TestPopulateWithTombSeriesIterators(t *testing.T) { + type minMaxTimes struct { + minTime, maxTime int64 + } cases := []struct { name string chks [][]chunks.Sample - expected []chunks.Sample - expectedChks []chunks.Meta + expected []chunks.Sample + expectedChks []chunks.Meta + expectedMinMaxTimes []minMaxTimes intervals tombstones.Intervals @@ -718,6 +722,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { expectedChks: []chunks.Meta{ assureChunkFromSamples(t, []chunks.Sample{}), }, + expectedMinMaxTimes: []minMaxTimes{{0, 0}}, }, { name: "three empty chunks", // This should never happen. @@ -728,6 +733,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { assureChunkFromSamples(t, []chunks.Sample{}), assureChunkFromSamples(t, []chunks.Sample{}), }, + expectedMinMaxTimes: []minMaxTimes{{0, 0}, {0, 0}, {0, 0}}, }, { name: "one chunk", @@ -743,6 +749,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 6}}, }, { name: "two full chunks", @@ -762,6 +769,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{7, 89, nil, nil}, sample{9, 8, nil, nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 6}, {7, 9}}, }, { name: "three full chunks", @@ -785,6 +793,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{10, 22, nil, nil}, sample{203, 3493, nil, nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 6}, {7, 9}, {10, 203}}, }, // Seek cases. { @@ -855,6 +864,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{7, 89, nil, nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{3, 6}, {7, 7}}, }, { name: "two chunks with trimmed middle sample of first chunk", @@ -875,6 +885,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{7, 89, nil, nil}, sample{9, 8, nil, nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 6}, {7, 9}}, }, { name: "two chunks with deletion across two chunks", @@ -895,6 +906,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{9, 8, nil, nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 3}, {9, 9}}, }, // Deletion with seek. { @@ -935,6 +947,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{6, 0, tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(6)), nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 6}}, }, { name: "one histogram chunk intersect with deletion interval", @@ -959,6 +972,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{3, 0, tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(3)), nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 3}}, }, { name: "one float histogram chunk", @@ -984,6 +998,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{6, 0, nil, tsdbutil.SetFloatHistogramNotCounterReset(tsdbutil.GenerateTestFloatHistogram(6))}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 6}}, }, { name: "one float histogram chunk intersect with deletion interval", @@ -1008,6 +1023,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{3, 0, nil, tsdbutil.SetFloatHistogramNotCounterReset(tsdbutil.GenerateTestFloatHistogram(3))}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 3}}, }, { name: "one gauge histogram chunk", @@ -1033,6 +1049,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{6, 0, tsdbutil.GenerateTestGaugeHistogram(6), nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 6}}, }, { name: "one gauge histogram chunk intersect with deletion interval", @@ -1057,6 +1074,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{3, 0, tsdbutil.GenerateTestGaugeHistogram(3), nil}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 3}}, }, { name: "one gauge float histogram", @@ -1082,6 +1100,7 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{6, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(6)}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 6}}, }, { name: "one gauge float histogram chunk intersect with deletion interval", @@ -1106,6 +1125,102 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { sample{3, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3)}, }), }, + expectedMinMaxTimes: []minMaxTimes{{1, 3}}, + }, + { + name: "three full mixed chunks", + chks: [][]chunks.Sample{ + {sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}}, + { + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, + sample{9, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, + }, + { + sample{10, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, + sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }, + }, + + expected: []chunks.Sample{ + sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}, sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, sample{9, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, sample{10, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }, + expectedChks: []chunks.Meta{ + assureChunkFromSamples(t, []chunks.Sample{ + sample{1, 2, nil, nil}, sample{2, 3, nil, nil}, sample{3, 5, nil, nil}, sample{6, 1, nil, nil}, + }), + assureChunkFromSamples(t, []chunks.Sample{ + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, + sample{9, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, + }), + assureChunkFromSamples(t, []chunks.Sample{ + sample{10, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, + sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }), + }, + expectedMinMaxTimes: []minMaxTimes{{1, 6}, {7, 9}, {10, 203}}, + }, + { + name: "three full mixed chunks in different order", + chks: [][]chunks.Sample{ + { + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, + sample{9, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, + }, + {sample{11, 2, nil, nil}, sample{12, 3, nil, nil}, sample{13, 5, nil, nil}, sample{16, 1, nil, nil}}, + { + sample{100, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, + sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }, + }, + + expected: []chunks.Sample{ + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, sample{9, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, sample{11, 2, nil, nil}, sample{12, 3, nil, nil}, sample{13, 5, nil, nil}, sample{16, 1, nil, nil}, sample{100, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }, + expectedChks: []chunks.Meta{ + assureChunkFromSamples(t, []chunks.Sample{ + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, + sample{9, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, + }), + assureChunkFromSamples(t, []chunks.Sample{ + sample{11, 2, nil, nil}, sample{12, 3, nil, nil}, sample{13, 5, nil, nil}, sample{16, 1, nil, nil}, + }), + assureChunkFromSamples(t, []chunks.Sample{ + sample{100, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, + sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }), + }, + expectedMinMaxTimes: []minMaxTimes{{7, 9}, {11, 16}, {100, 203}}, + }, + { + name: "three full mixed chunks in different order intersect with deletion interval", + chks: [][]chunks.Sample{ + { + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, + sample{9, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, + }, + {sample{11, 2, nil, nil}, sample{12, 3, nil, nil}, sample{13, 5, nil, nil}, sample{16, 1, nil, nil}}, + { + sample{100, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, + sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }, + }, + intervals: tombstones.Intervals{{Mint: 8, Maxt: 11}, {Mint: 15, Maxt: 150}}, + + expected: []chunks.Sample{ + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, sample{12, 3, nil, nil}, sample{13, 5, nil, nil}, sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }, + expectedChks: []chunks.Meta{ + assureChunkFromSamples(t, []chunks.Sample{ + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, + }), + assureChunkFromSamples(t, []chunks.Sample{ + sample{12, 3, nil, nil}, sample{13, 5, nil, nil}, + }), + assureChunkFromSamples(t, []chunks.Sample{ + sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }), + }, + expectedMinMaxTimes: []minMaxTimes{{7, 7}, {12, 13}, {203, 203}}, }, } for _, tc := range cases { @@ -1147,6 +1262,11 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { rmChunkRefs(expandedResult) rmChunkRefs(tc.expectedChks) require.Equal(t, tc.expectedChks, expandedResult) + + for i, meta := range expandedResult { + require.Equal(t, tc.expectedMinMaxTimes[i].minTime, meta.MinTime) + require.Equal(t, tc.expectedMinMaxTimes[i].maxTime, meta.MaxTime) + } }) }) } From 2f7060bd5afa40d50e7251cacd08a69bb8c4684f Mon Sep 17 00:00:00 2001 From: Jeanette Tan Date: Wed, 1 Nov 2023 20:04:23 +0800 Subject: [PATCH 06/10] Expand TestPopulateWithTombSeriesIterators to test earlier deletion intervals for histogram chunks as well as time-overlapping chunks Signed-off-by: Jeanette Tan --- tsdb/querier_test.go | 132 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 128 insertions(+), 4 deletions(-) diff --git a/tsdb/querier_test.go b/tsdb/querier_test.go index 5ad5003f1f..720d5c6997 100644 --- a/tsdb/querier_test.go +++ b/tsdb/querier_test.go @@ -950,7 +950,30 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { expectedMinMaxTimes: []minMaxTimes{{1, 6}}, }, { - name: "one histogram chunk intersect with deletion interval", + name: "one histogram chunk intersect with earlier deletion interval", + chks: [][]chunks.Sample{ + { + sample{1, 0, tsdbutil.GenerateTestHistogram(1), nil}, + sample{2, 0, tsdbutil.GenerateTestHistogram(2), nil}, + sample{3, 0, tsdbutil.GenerateTestHistogram(3), nil}, + sample{6, 0, tsdbutil.GenerateTestHistogram(6), nil}, + }, + }, + intervals: tombstones.Intervals{{Mint: 1, Maxt: 2}}, + expected: []chunks.Sample{ + sample{3, 0, tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(3)), nil}, + sample{6, 0, tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(6)), nil}, + }, + expectedChks: []chunks.Meta{ + assureChunkFromSamples(t, []chunks.Sample{ + sample{3, 0, tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(3)), nil}, + sample{6, 0, tsdbutil.SetHistogramNotCounterReset(tsdbutil.GenerateTestHistogram(6)), nil}, + }), + }, + expectedMinMaxTimes: []minMaxTimes{{3, 6}}, + }, + { + name: "one histogram chunk intersect with later deletion interval", chks: [][]chunks.Sample{ { sample{1, 0, tsdbutil.GenerateTestHistogram(1), nil}, @@ -1001,7 +1024,30 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { expectedMinMaxTimes: []minMaxTimes{{1, 6}}, }, { - name: "one float histogram chunk intersect with deletion interval", + name: "one float histogram chunk intersect with earlier deletion interval", + chks: [][]chunks.Sample{ + { + sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(1)}, + sample{2, 0, nil, tsdbutil.GenerateTestFloatHistogram(2)}, + sample{3, 0, nil, tsdbutil.GenerateTestFloatHistogram(3)}, + sample{6, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, + }, + }, + intervals: tombstones.Intervals{{Mint: 1, Maxt: 2}}, + expected: []chunks.Sample{ + sample{3, 0, nil, tsdbutil.SetFloatHistogramNotCounterReset(tsdbutil.GenerateTestFloatHistogram(3))}, + sample{6, 0, nil, tsdbutil.SetFloatHistogramNotCounterReset(tsdbutil.GenerateTestFloatHistogram(6))}, + }, + expectedChks: []chunks.Meta{ + assureChunkFromSamples(t, []chunks.Sample{ + sample{3, 0, nil, tsdbutil.SetFloatHistogramNotCounterReset(tsdbutil.GenerateTestFloatHistogram(3))}, + sample{6, 0, nil, tsdbutil.SetFloatHistogramNotCounterReset(tsdbutil.GenerateTestFloatHistogram(6))}, + }), + }, + expectedMinMaxTimes: []minMaxTimes{{3, 6}}, + }, + { + name: "one float histogram chunk intersect with later deletion interval", chks: [][]chunks.Sample{ { sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(1)}, @@ -1052,7 +1098,30 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { expectedMinMaxTimes: []minMaxTimes{{1, 6}}, }, { - name: "one gauge histogram chunk intersect with deletion interval", + name: "one gauge histogram chunk intersect with earlier deletion interval", + chks: [][]chunks.Sample{ + { + sample{1, 0, tsdbutil.GenerateTestGaugeHistogram(1), nil}, + sample{2, 0, tsdbutil.GenerateTestGaugeHistogram(2), nil}, + sample{3, 0, tsdbutil.GenerateTestGaugeHistogram(3), nil}, + sample{6, 0, tsdbutil.GenerateTestGaugeHistogram(6), nil}, + }, + }, + intervals: tombstones.Intervals{{Mint: 1, Maxt: 2}}, + expected: []chunks.Sample{ + sample{3, 0, tsdbutil.GenerateTestGaugeHistogram(3), nil}, + sample{6, 0, tsdbutil.GenerateTestGaugeHistogram(6), nil}, + }, + expectedChks: []chunks.Meta{ + assureChunkFromSamples(t, []chunks.Sample{ + sample{3, 0, tsdbutil.GenerateTestGaugeHistogram(3), nil}, + sample{6, 0, tsdbutil.GenerateTestGaugeHistogram(6), nil}, + }), + }, + expectedMinMaxTimes: []minMaxTimes{{3, 6}}, + }, + { + name: "one gauge histogram chunk intersect with later deletion interval", chks: [][]chunks.Sample{ { sample{1, 0, tsdbutil.GenerateTestGaugeHistogram(1), nil}, @@ -1103,7 +1172,30 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { expectedMinMaxTimes: []minMaxTimes{{1, 6}}, }, { - name: "one gauge float histogram chunk intersect with deletion interval", + name: "one gauge float histogram chunk intersect with earlier deletion interval", + chks: [][]chunks.Sample{ + { + sample{1, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(1)}, + sample{2, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(2)}, + sample{3, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3)}, + sample{6, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(6)}, + }, + }, + intervals: tombstones.Intervals{{Mint: 1, Maxt: 2}}, + expected: []chunks.Sample{ + sample{3, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3)}, + sample{6, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(6)}, + }, + expectedChks: []chunks.Meta{ + assureChunkFromSamples(t, []chunks.Sample{ + sample{3, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3)}, + sample{6, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(6)}, + }), + }, + expectedMinMaxTimes: []minMaxTimes{{3, 6}}, + }, + { + name: "one gauge float histogram chunk intersect with later deletion interval", chks: [][]chunks.Sample{ { sample{1, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(1)}, @@ -1222,6 +1314,38 @@ func TestPopulateWithTombSeriesIterators(t *testing.T) { }, expectedMinMaxTimes: []minMaxTimes{{7, 7}, {12, 13}, {203, 203}}, }, + { + name: "three full mixed chunks overlapping", + chks: [][]chunks.Sample{ + { + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, + sample{12, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, + }, + {sample{11, 2, nil, nil}, sample{12, 3, nil, nil}, sample{13, 5, nil, nil}, sample{16, 1, nil, nil}}, + { + sample{10, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, + sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }, + }, + + expected: []chunks.Sample{ + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, sample{12, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, sample{11, 2, nil, nil}, sample{12, 3, nil, nil}, sample{13, 5, nil, nil}, sample{16, 1, nil, nil}, sample{10, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }, + expectedChks: []chunks.Meta{ + assureChunkFromSamples(t, []chunks.Sample{ + sample{7, 0, tsdbutil.GenerateTestGaugeHistogram(89), nil}, + sample{12, 0, tsdbutil.GenerateTestGaugeHistogram(8), nil}, + }), + assureChunkFromSamples(t, []chunks.Sample{ + sample{11, 2, nil, nil}, sample{12, 3, nil, nil}, sample{13, 5, nil, nil}, sample{16, 1, nil, nil}, + }), + assureChunkFromSamples(t, []chunks.Sample{ + sample{10, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(22)}, + sample{203, 0, nil, tsdbutil.GenerateTestGaugeFloatHistogram(3493)}, + }), + }, + expectedMinMaxTimes: []minMaxTimes{{7, 12}, {11, 16}, {10, 203}}, + }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { From 27abf09e7fe80237d4925874de4d2e524460e71a Mon Sep 17 00:00:00 2001 From: Jeanette Tan Date: Wed, 1 Nov 2023 19:53:41 +0800 Subject: [PATCH 07/10] Fix missing MinTime in histogram chunks Signed-off-by: Jeanette Tan --- tsdb/querier.go | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/tsdb/querier.go b/tsdb/querier.go index 96406f1bdf..9a01e8c49b 100644 --- a/tsdb/querier.go +++ b/tsdb/querier.go @@ -855,13 +855,18 @@ func (p *populateWithDelChunkSeriesIterator) Next() bool { if app, err = newChunk.Appender(); err != nil { break } - - for vt := valueType; vt != chunkenc.ValNone; vt = p.currDelIter.Next() { + var h *histogram.Histogram + t, h = p.currDelIter.AtHistogram() + p.curr.MinTime = t + _, _, app, err = app.AppendHistogram(nil, t, h, true) + if err != nil { + break + } + for vt := p.currDelIter.Next(); vt != chunkenc.ValNone; vt = p.currDelIter.Next() { if vt != chunkenc.ValHistogram { err = fmt.Errorf("found value type %v in histogram chunk", vt) break } - var h *histogram.Histogram t, h = p.currDelIter.AtHistogram() _, _, app, err = app.AppendHistogram(nil, t, h, true) if err != nil { @@ -890,13 +895,18 @@ func (p *populateWithDelChunkSeriesIterator) Next() bool { if app, err = newChunk.Appender(); err != nil { break } - - for vt := valueType; vt != chunkenc.ValNone; vt = p.currDelIter.Next() { + var h *histogram.FloatHistogram + t, h = p.currDelIter.AtFloatHistogram() + p.curr.MinTime = t + _, _, app, err = app.AppendFloatHistogram(nil, t, h, true) + if err != nil { + break + } + for vt := p.currDelIter.Next(); vt != chunkenc.ValNone; vt = p.currDelIter.Next() { if vt != chunkenc.ValFloatHistogram { err = fmt.Errorf("found value type %v in histogram chunk", vt) break } - var h *histogram.FloatHistogram t, h = p.currDelIter.AtFloatHistogram() _, _, app, err = app.AppendFloatHistogram(nil, t, h, true) if err != nil { From 4296ecbd14dd2639eb5768ab66e4c61ea86c7b33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gy=C3=B6rgy=20Krajcsovits?= Date: Wed, 1 Nov 2023 15:52:04 +0100 Subject: [PATCH 08/10] tsdb/compact_test.go: test mixed typed series with PopulateBlock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add testcase and update test so that it can test native histograms as well. Signed-off-by: György Krajcsovits --- tsdb/compact_test.go | 43 ++++++++++++++++++++++++++++++++++++++++--- tsdb/querier_test.go | 33 ++++++++++++++++++++++++++++----- 2 files changed, 68 insertions(+), 8 deletions(-) diff --git a/tsdb/compact_test.go b/tsdb/compact_test.go index 098be8bfa0..2d98b30502 100644 --- a/tsdb/compact_test.go +++ b/tsdb/compact_test.go @@ -38,6 +38,7 @@ import ( "github.com/prometheus/prometheus/tsdb/chunks" "github.com/prometheus/prometheus/tsdb/fileutil" "github.com/prometheus/prometheus/tsdb/tombstones" + "github.com/prometheus/prometheus/tsdb/tsdbutil" "github.com/prometheus/prometheus/tsdb/wlog" ) @@ -933,6 +934,31 @@ func TestCompaction_populateBlock(t *testing.T) { }, }, }, + { + title: "Populate from mixed type series and expect sample inside the interval only", // regression test for populateWithDelChunkSeriesIterator failing to set minTime on chunks + compactMinTime: 1, + compactMaxTime: 11, + inputSeriesSamples: [][]seriesSamples{ + { + { + lset: map[string]string{"a": "1"}, + chunks: [][]sample{ + {{t: 0, h: tsdbutil.GenerateTestHistogram(0)}, {t: 1, h: tsdbutil.GenerateTestHistogram(1)}}, + {{t: 10, f: 1}, {t: 11, f: 2}}, + }, + }, + }, + }, + expSeriesSamples: []seriesSamples{ + { + lset: map[string]string{"a": "1"}, + chunks: [][]sample{ + {{t: 1, h: tsdbutil.GenerateTestHistogram(1)}}, + {{t: 10, f: 1}}, + }, + }, + }, + }, } { t.Run(tc.title, func(t *testing.T) { blocks := make([]BlockReader, 0, len(tc.inputSeriesSamples)) @@ -974,12 +1000,23 @@ func TestCompaction_populateBlock(t *testing.T) { firstTs int64 = math.MaxInt64 s sample ) - for iter.Next() == chunkenc.ValFloat { - s.t, s.f = iter.At() + for vt := iter.Next(); vt != chunkenc.ValNone; vt = iter.Next() { + switch vt { + case chunkenc.ValFloat: + s.t, s.f = iter.At() + samples = append(samples, s) + case chunkenc.ValHistogram: + s.t, s.h = iter.AtHistogram() + samples = append(samples, s) + case chunkenc.ValFloatHistogram: + s.t, s.fh = iter.AtFloatHistogram() + samples = append(samples, s) + default: + require.Fail(t, "unexpected value type") + } if firstTs == math.MaxInt64 { firstTs = s.t } - samples = append(samples, s) } // Check if chunk has correct min, max times. diff --git a/tsdb/querier_test.go b/tsdb/querier_test.go index 720d5c6997..17a99c6e38 100644 --- a/tsdb/querier_test.go +++ b/tsdb/querier_test.go @@ -133,12 +133,35 @@ func createIdxChkReaders(t *testing.T, tc []seriesSamples) (IndexReader, ChunkRe Ref: chunkRef, }) - chunk := chunkenc.NewXORChunk() - app, _ := chunk.Appender() - for _, smpl := range chk { - app.Append(smpl.t, smpl.f) + switch { + case chk[0].fh != nil: + chunk := chunkenc.NewFloatHistogramChunk() + app, _ := chunk.Appender() + for _, smpl := range chk { + require.NotNil(t, smpl.fh, "chunk can only contain one type of sample") + _, _, _, err := app.AppendFloatHistogram(nil, smpl.t, smpl.fh, true) + require.NoError(t, err, "chunk should be appendable") + } + chkReader[chunkRef] = chunk + case chk[0].h != nil: + chunk := chunkenc.NewHistogramChunk() + app, _ := chunk.Appender() + for _, smpl := range chk { + require.NotNil(t, smpl.h, "chunk can only contain one type of sample") + _, _, _, err := app.AppendHistogram(nil, smpl.t, smpl.h, true) + require.NoError(t, err, "chunk should be appendable") + } + chkReader[chunkRef] = chunk + default: + chunk := chunkenc.NewXORChunk() + app, _ := chunk.Appender() + for _, smpl := range chk { + require.Nil(t, smpl.h, "chunk can only contain one type of sample") + require.Nil(t, smpl.fh, "chunk can only contain one type of sample") + app.Append(smpl.t, smpl.f) + } + chkReader[chunkRef] = chunk } - chkReader[chunkRef] = chunk chunkRef++ } ls := labels.FromMap(s.lset) From 3ccaaa40ba088b74e69a471e95fce9958bd291a5 Mon Sep 17 00:00:00 2001 From: Jeanette Tan Date: Thu, 2 Nov 2023 13:37:07 +0800 Subject: [PATCH 09/10] Fix according to code review Signed-off-by: Jeanette Tan --- tsdb/compact_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tsdb/compact_test.go b/tsdb/compact_test.go index 2d98b30502..cf8c2439ac 100644 --- a/tsdb/compact_test.go +++ b/tsdb/compact_test.go @@ -935,7 +935,8 @@ func TestCompaction_populateBlock(t *testing.T) { }, }, { - title: "Populate from mixed type series and expect sample inside the interval only", // regression test for populateWithDelChunkSeriesIterator failing to set minTime on chunks + // Regression test for populateWithDelChunkSeriesIterator failing to set minTime on chunks. + title: "Populate from mixed type series and expect sample inside the interval only.", compactMinTime: 1, compactMaxTime: 11, inputSeriesSamples: [][]seriesSamples{ From 52eb303031450f06e4d2da6c3f08681c0a5b8bb3 Mon Sep 17 00:00:00 2001 From: Jeanette Tan Date: Thu, 2 Nov 2023 21:23:05 +0800 Subject: [PATCH 10/10] Refactor assigning MinTime in histogram chunks Signed-off-by: Jeanette Tan --- tsdb/querier.go | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/tsdb/querier.go b/tsdb/querier.go index 9a01e8c49b..a832c0d1ba 100644 --- a/tsdb/querier.go +++ b/tsdb/querier.go @@ -840,6 +840,7 @@ func (p *populateWithDelChunkSeriesIterator) Next() bool { } return false } + p.curr.MinTime = p.currDelIter.AtT() // Re-encode the chunk if iterator is provider. This means that it has // some samples to be deleted or chunk is opened. @@ -855,18 +856,12 @@ func (p *populateWithDelChunkSeriesIterator) Next() bool { if app, err = newChunk.Appender(); err != nil { break } - var h *histogram.Histogram - t, h = p.currDelIter.AtHistogram() - p.curr.MinTime = t - _, _, app, err = app.AppendHistogram(nil, t, h, true) - if err != nil { - break - } - for vt := p.currDelIter.Next(); vt != chunkenc.ValNone; vt = p.currDelIter.Next() { + for vt := valueType; vt != chunkenc.ValNone; vt = p.currDelIter.Next() { if vt != chunkenc.ValHistogram { err = fmt.Errorf("found value type %v in histogram chunk", vt) break } + var h *histogram.Histogram t, h = p.currDelIter.AtHistogram() _, _, app, err = app.AppendHistogram(nil, t, h, true) if err != nil { @@ -878,15 +873,12 @@ func (p *populateWithDelChunkSeriesIterator) Next() bool { if app, err = newChunk.Appender(); err != nil { break } - var v float64 - t, v = p.currDelIter.At() - p.curr.MinTime = t - app.Append(t, v) - for vt := p.currDelIter.Next(); vt != chunkenc.ValNone; vt = p.currDelIter.Next() { + for vt := valueType; vt != chunkenc.ValNone; vt = p.currDelIter.Next() { if vt != chunkenc.ValFloat { err = fmt.Errorf("found value type %v in float chunk", vt) break } + var v float64 t, v = p.currDelIter.At() app.Append(t, v) } @@ -895,18 +887,12 @@ func (p *populateWithDelChunkSeriesIterator) Next() bool { if app, err = newChunk.Appender(); err != nil { break } - var h *histogram.FloatHistogram - t, h = p.currDelIter.AtFloatHistogram() - p.curr.MinTime = t - _, _, app, err = app.AppendFloatHistogram(nil, t, h, true) - if err != nil { - break - } - for vt := p.currDelIter.Next(); vt != chunkenc.ValNone; vt = p.currDelIter.Next() { + for vt := valueType; vt != chunkenc.ValNone; vt = p.currDelIter.Next() { if vt != chunkenc.ValFloatHistogram { err = fmt.Errorf("found value type %v in histogram chunk", vt) break } + var h *histogram.FloatHistogram t, h = p.currDelIter.AtFloatHistogram() _, _, app, err = app.AppendFloatHistogram(nil, t, h, true) if err != nil {