From 17e2c307541dae957dae7fe1e444190f6179d41a Mon Sep 17 00:00:00 2001 From: tdakkota Date: Wed, 20 Mar 2024 22:01:05 +0300 Subject: [PATCH 1/4] promql: validate `label_join` destination label Signed-off-by: tdakkota --- promql/functions.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/promql/functions.go b/promql/functions.go index da66af2f02..d5840374e7 100644 --- a/promql/functions.go +++ b/promql/functions.go @@ -1386,6 +1386,9 @@ func (ev *evaluator) evalLabelJoin(args parser.Expressions) (parser.Value, annot } srcLabels[i-3] = src } + if !model.LabelName(dst).IsValid() { + panic(fmt.Errorf("invalid destination label name in label_join(): %s", dst)) + } val, ws := ev.eval(args[0]) matrix := val.(Matrix) From 3929d6500a89c36b31264b50fd60de28426eaa43 Mon Sep 17 00:00:00 2001 From: Domantas Date: Wed, 27 Mar 2024 12:35:17 +0200 Subject: [PATCH 2/4] [BUGFIX] labels: don't modify original labels in DropMetricName (#13845) Restrict the capacity of first argument to `append()` to force an allocation. This is for the slice implementation only. Signed-off-by: Domantas Jadenkus Signed-off-by: Bryan Boreham --- model/labels/labels.go | 4 +++- model/labels/labels_test.go | 6 +++++- promql/engine_test.go | 18 ++++++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/model/labels/labels.go b/model/labels/labels.go index e998248269..01514abf38 100644 --- a/model/labels/labels.go +++ b/model/labels/labels.go @@ -349,7 +349,9 @@ func (ls Labels) DropMetricName() Labels { if i == 0 { // Make common case fast with no allocations. return ls[1:] } - return append(ls[:i], ls[i+1:]...) + // Avoid modifying original Labels - use [:i:i] so that left slice would not + // have any spare capacity and append would have to allocate a new slice for the result. + return append(ls[:i:i], ls[i+1:]...) } } return ls diff --git a/model/labels/labels_test.go b/model/labels/labels_test.go index c2ac6d63a0..359e6de3b8 100644 --- a/model/labels/labels_test.go +++ b/model/labels/labels_test.go @@ -450,7 +450,11 @@ func TestLabels_Get(t *testing.T) { func TestLabels_DropMetricName(t *testing.T) { require.True(t, Equal(FromStrings("aaa", "111", "bbb", "222"), FromStrings("aaa", "111", "bbb", "222").DropMetricName())) require.True(t, Equal(FromStrings("aaa", "111"), FromStrings(MetricName, "myname", "aaa", "111").DropMetricName())) - require.True(t, Equal(FromStrings("__aaa__", "111", "bbb", "222"), FromStrings("__aaa__", "111", MetricName, "myname", "bbb", "222").DropMetricName())) + + original := FromStrings("__aaa__", "111", MetricName, "myname", "bbb", "222") + check := FromStrings("__aaa__", "111", MetricName, "myname", "bbb", "222") + require.True(t, Equal(FromStrings("__aaa__", "111", "bbb", "222"), check.DropMetricName())) + require.True(t, Equal(original, check)) } // BenchmarkLabels_Get was written to check whether a binary search can improve the performance vs the linear search implementation diff --git a/promql/engine_test.go b/promql/engine_test.go index 105108d5b2..ea65e01a63 100644 --- a/promql/engine_test.go +++ b/promql/engine_test.go @@ -3136,6 +3136,24 @@ func TestRangeQuery(t *testing.T) { End: time.Unix(120, 0), Interval: 1 * time.Minute, }, + { + Name: "drop-metric-name", + Load: `load 30s + requests{job="1", __address__="bar"} 100`, + Query: `requests * 2`, + Result: Matrix{ + Series{ + Floats: []FPoint{{F: 200, T: 0}, {F: 200, T: 60000}, {F: 200, T: 120000}}, + Metric: labels.FromStrings( + "__address__", "bar", + "job", "1", + ), + }, + }, + Start: time.Unix(0, 0), + End: time.Unix(120, 0), + Interval: 1 * time.Minute, + }, } for _, c := range cases { t.Run(c.Name, func(t *testing.T) { From d64c6fe34faf5ea0d64ea1555b7b0a66cbc09020 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gy=C3=B6rgy=20Krajcsovits?= Date: Wed, 27 Mar 2024 16:41:23 +0100 Subject: [PATCH 3/4] fix the bug of setting native histogram min bucket factor (#13846) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix the bug of setting native histogram min bucket factor Signed-off-by: Ziqi Zhao * Add unit test for checking that min_bucket_factor is correctly applied Signed-off-by: György Krajcsovits --------- Signed-off-by: Ziqi Zhao Signed-off-by: György Krajcsovits Co-authored-by: György Krajcsovits --- scrape/scrape.go | 2 + scrape/scrape_test.go | 135 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 137 insertions(+) diff --git a/scrape/scrape.go b/scrape/scrape.go index f6f3367736..219eba4d99 100644 --- a/scrape/scrape.go +++ b/scrape/scrape.go @@ -427,6 +427,7 @@ func (sp *scrapePool) sync(targets []*Target) { bodySizeLimit = int64(sp.config.BodySizeLimit) sampleLimit = int(sp.config.SampleLimit) bucketLimit = int(sp.config.NativeHistogramBucketLimit) + maxSchema = pickSchema(sp.config.NativeHistogramMinBucketFactor) labelLimits = &labelLimits{ labelLimit: int(sp.config.LabelLimit), labelNameLengthLimit: int(sp.config.LabelNameLengthLimit), @@ -464,6 +465,7 @@ func (sp *scrapePool) sync(targets []*Target) { scraper: s, sampleLimit: sampleLimit, bucketLimit: bucketLimit, + maxSchema: maxSchema, labelLimits: labelLimits, honorLabels: honorLabels, honorTimestamps: honorTimestamps, diff --git a/scrape/scrape_test.go b/scrape/scrape_test.go index bcaeb460e2..4dfa4f684e 100644 --- a/scrape/scrape_test.go +++ b/scrape/scrape_test.go @@ -41,6 +41,7 @@ import ( "github.com/stretchr/testify/require" "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/model/exemplar" "github.com/prometheus/prometheus/model/histogram" @@ -3599,3 +3600,137 @@ func BenchmarkTargetScraperGzip(b *testing.B) { }) } } + +// This tests running a full scrape loop and checking that the scrape option +// `native_histogram_min_bucket_factor` is used correctly. +func TestNativeHistogramMaxSchemaSet(t *testing.T) { + testcases := map[string]struct { + minBucketFactor string + expectedSchema int32 + }{ + "min factor not specified": { + minBucketFactor: "", + expectedSchema: 3, // Factor 1.09. + }, + "min factor 1": { + minBucketFactor: "native_histogram_min_bucket_factor: 1", + expectedSchema: 3, // Factor 1.09. + }, + "min factor 2": { + minBucketFactor: "native_histogram_min_bucket_factor: 2", + expectedSchema: 0, // Factor 2.00. + }, + } + for name, tc := range testcases { + t.Run(name, func(t *testing.T) { + testNativeHistogramMaxSchemaSet(t, tc.minBucketFactor, tc.expectedSchema) + }) + } +} + +func testNativeHistogramMaxSchemaSet(t *testing.T, minBucketFactor string, expectedSchema int32) { + // Create a ProtoBuf message to serve as a Prometheus metric. + nativeHistogram := prometheus.NewHistogram( + prometheus.HistogramOpts{ + Namespace: "testing", + Name: "example_native_histogram", + Help: "This is used for testing", + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + }, + ) + registry := prometheus.NewRegistry() + registry.Register(nativeHistogram) + nativeHistogram.Observe(1.0) + nativeHistogram.Observe(1.0) + nativeHistogram.Observe(1.0) + nativeHistogram.Observe(10.0) // in different bucket since > 1*1.1. + nativeHistogram.Observe(10.0) + + gathered, err := registry.Gather() + require.NoError(t, err) + require.NotEmpty(t, gathered) + + histogramMetricFamily := gathered[0] + buffer := protoMarshalDelimited(t, histogramMetricFamily) + + // Create a HTTP server to serve /metrics via ProtoBuf + metricsServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", `application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; encoding=delimited`) + w.Write(buffer) + })) + defer metricsServer.Close() + + // Create a scrape loop with the HTTP server as the target. + configStr := fmt.Sprintf(` +global: + scrape_interval: 1s + scrape_timeout: 1s +scrape_configs: + - job_name: test + %s + static_configs: + - targets: [%s] +`, minBucketFactor, strings.ReplaceAll(metricsServer.URL, "http://", "")) + + s := teststorage.New(t) + defer s.Close() + s.DB.EnableNativeHistograms() + reg := prometheus.NewRegistry() + + mng, err := NewManager(nil, nil, s, reg) + require.NoError(t, err) + cfg, err := config.Load(configStr, false, log.NewNopLogger()) + require.NoError(t, err) + mng.ApplyConfig(cfg) + tsets := make(chan map[string][]*targetgroup.Group) + go func() { + err = mng.Run(tsets) + require.NoError(t, err) + }() + defer mng.Stop() + + // Get the static targets and apply them to the scrape manager. + require.Len(t, cfg.ScrapeConfigs, 1) + scrapeCfg := cfg.ScrapeConfigs[0] + require.Len(t, scrapeCfg.ServiceDiscoveryConfigs, 1) + staticDiscovery, ok := scrapeCfg.ServiceDiscoveryConfigs[0].(discovery.StaticConfig) + require.True(t, ok) + require.Len(t, staticDiscovery, 1) + tsets <- map[string][]*targetgroup.Group{"test": staticDiscovery} + + // Wait for the scrape loop to scrape the target. + require.Eventually(t, func() bool { + q, err := s.Querier(0, math.MaxInt64) + require.NoError(t, err) + seriesS := q.Select(context.Background(), false, nil, labels.MustNewMatcher(labels.MatchEqual, "__name__", "testing_example_native_histogram")) + countSeries := 0 + for seriesS.Next() { + countSeries++ + } + return countSeries > 0 + }, 15*time.Second, 100*time.Millisecond) + + // Check that native histogram schema is as expected. + q, err := s.Querier(0, math.MaxInt64) + require.NoError(t, err) + seriesS := q.Select(context.Background(), false, nil, labels.MustNewMatcher(labels.MatchEqual, "__name__", "testing_example_native_histogram")) + histogramSamples := []*histogram.Histogram{} + for seriesS.Next() { + series := seriesS.At() + it := series.Iterator(nil) + for vt := it.Next(); vt != chunkenc.ValNone; vt = it.Next() { + if vt != chunkenc.ValHistogram { + // don't care about other samples + continue + } + _, h := it.AtHistogram(nil) + histogramSamples = append(histogramSamples, h) + } + } + require.NoError(t, seriesS.Err()) + require.NotEmpty(t, histogramSamples) + for _, h := range histogramSamples { + require.Equal(t, expectedSchema, h.Schema) + } +} From 855b5ac4b80956874eb1790a04c92327f2f99e38 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 28 Mar 2024 09:23:45 +0000 Subject: [PATCH 4/4] Cut release 2.51.1 (#13853) * Cut release 2.51.1 Co-authored-by: George Krajcsovits Signed-off-by: Bryan Boreham --- CHANGELOG.md | 8 ++++++++ VERSION | 2 +- web/ui/module/codemirror-promql/package.json | 4 ++-- web/ui/module/lezer-promql/package.json | 2 +- web/ui/package-lock.json | 14 +++++++------- web/ui/package.json | 2 +- web/ui/react-app/package.json | 4 ++-- 7 files changed, 22 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8951866819..046d3ea60a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 2.51.1 / 2024-03-27 + +Bugfix release. + +* [BUGFIX] PromQL: Re-instate validation of label_join destination label #13803 +* [BUGFIX] Scraping (experimental native histograms): Fix handling of the min bucket factor on sync of targets #13846 +* [BUGFIX] PromQL: Some queries could return the same series twice (library use only) #13845 + ## 2.51.0 / 2024-03-18 This version is built with Go 1.22.1. diff --git a/VERSION b/VERSION index d66b738716..8e00e241f3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.51.0 +2.51.1 diff --git a/web/ui/module/codemirror-promql/package.json b/web/ui/module/codemirror-promql/package.json index 5139bce620..c364e05584 100644 --- a/web/ui/module/codemirror-promql/package.json +++ b/web/ui/module/codemirror-promql/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/codemirror-promql", - "version": "0.51.0", + "version": "0.51.1", "description": "a CodeMirror mode for the PromQL language", "types": "dist/esm/index.d.ts", "module": "dist/esm/index.js", @@ -29,7 +29,7 @@ }, "homepage": "https://github.com/prometheus/prometheus/blob/main/web/ui/module/codemirror-promql/README.md", "dependencies": { - "@prometheus-io/lezer-promql": "0.51.0", + "@prometheus-io/lezer-promql": "0.51.1", "lru-cache": "^7.18.3" }, "devDependencies": { diff --git a/web/ui/module/lezer-promql/package.json b/web/ui/module/lezer-promql/package.json index 2983206f5c..f5ea66602f 100644 --- a/web/ui/module/lezer-promql/package.json +++ b/web/ui/module/lezer-promql/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/lezer-promql", - "version": "0.51.0", + "version": "0.51.1", "description": "lezer-based PromQL grammar", "main": "dist/index.cjs", "type": "module", diff --git a/web/ui/package-lock.json b/web/ui/package-lock.json index d2581baf40..fdb348d669 100644 --- a/web/ui/package-lock.json +++ b/web/ui/package-lock.json @@ -1,12 +1,12 @@ { "name": "prometheus-io", - "version": "0.51.0", + "version": "0.51.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "prometheus-io", - "version": "0.51.0", + "version": "0.51.1", "workspaces": [ "react-app", "module/*" @@ -30,10 +30,10 @@ }, "module/codemirror-promql": { "name": "@prometheus-io/codemirror-promql", - "version": "0.51.0", + "version": "0.51.1", "license": "Apache-2.0", "dependencies": { - "@prometheus-io/lezer-promql": "0.51.0", + "@prometheus-io/lezer-promql": "0.51.1", "lru-cache": "^7.18.3" }, "devDependencies": { @@ -69,7 +69,7 @@ }, "module/lezer-promql": { "name": "@prometheus-io/lezer-promql", - "version": "0.51.0", + "version": "0.51.1", "license": "Apache-2.0", "devDependencies": { "@lezer/generator": "^1.5.1", @@ -19233,7 +19233,7 @@ }, "react-app": { "name": "@prometheus-io/app", - "version": "0.51.0", + "version": "0.51.1", "dependencies": { "@codemirror/autocomplete": "^6.11.1", "@codemirror/commands": "^6.3.2", @@ -19251,7 +19251,7 @@ "@lezer/lr": "^1.3.14", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "0.51.0", + "@prometheus-io/codemirror-promql": "0.51.1", "bootstrap": "^4.6.2", "css.escape": "^1.5.1", "downshift": "^7.6.2", diff --git a/web/ui/package.json b/web/ui/package.json index 7eedd0b217..0772aad7de 100644 --- a/web/ui/package.json +++ b/web/ui/package.json @@ -28,5 +28,5 @@ "ts-jest": "^29.1.1", "typescript": "^4.9.5" }, - "version": "0.51.0" + "version": "0.51.1" } diff --git a/web/ui/react-app/package.json b/web/ui/react-app/package.json index 4d2b5be920..b47c9180a3 100644 --- a/web/ui/react-app/package.json +++ b/web/ui/react-app/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/app", - "version": "0.51.0", + "version": "0.51.1", "private": true, "dependencies": { "@codemirror/autocomplete": "^6.11.1", @@ -19,7 +19,7 @@ "@lezer/lr": "^1.3.14", "@nexucis/fuzzy": "^0.4.1", "@nexucis/kvsearch": "^0.8.1", - "@prometheus-io/codemirror-promql": "0.51.0", + "@prometheus-io/codemirror-promql": "0.51.1", "bootstrap": "^4.6.2", "css.escape": "^1.5.1", "downshift": "^7.6.2",