Merge pull request #12366 from prometheus/release-2.44

Merge release 2.44 back to main

.promu.yml

@@ -14,8 +14,10 @@ build:
         all:
             - netgo
             - builtinassets
+            - stringlabels
         windows:
             - builtinassets
+            - stringlabels
     flags: -a
     ldflags: |
         -X github.com/prometheus/common/version.Version={{.Version}}

CHANGELOG.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## 2.44.0 / 2023-05-13
+
+This version is built with Go tag `stringlabels`, to use the smaller data
+structure for Labels that was optional in the previous release. For more
+details about this code change see #10991.
+
+* [CHANGE] Remote-write: Raise default samples per send to 2,000. #12203
+* [FEATURE] Remote-read: Handle native histograms. #12085, #12192
+* [FEATURE] Promtool: Health and readiness check of prometheus server in CLI. #12096
+* [FEATURE] PromQL: Add `query_samples_total` metric, the total number of samples loaded by all queries. #12251
+* [ENHANCEMENT] Storage: Optimise buffer used to iterate through samples. #12326
+* [ENHANCEMENT] Scrape: Reduce memory allocations on target labels. #12084
+* [ENHANCEMENT] PromQL: Use faster heap method for `topk()` / `bottomk()`. #12190
+* [ENHANCEMENT] Rules API: Allow filtering by rule name. #12270
+* [ENHANCEMENT] Native Histograms: Various fixes and improvements. #11687, #12264, #12272
+* [ENHANCEMENT] UI: Search of scraping pools is now case-insensitive. #12207
+* [ENHANCEMENT] TSDB: Add an affirmative log message for successful WAL repair. #12135
+* [BUGFIX] TSDB: Block compaction failed when shutting down. #12179
+* [BUGFIX] TSDB: Out-of-order chunks could be ignored if the write-behind log was deleted. #12127
+
 ## 2.43.0 / 2023-03-21
 
 We are working on some performance improvements in Prometheus, which are only

VERSION

@@ -1 +1 @@
-2.43.0
+2.44.0

model/labels/labels.go

@@ -533,16 +533,15 @@ func (b *Builder) Set(n, v string) *Builder {
 }
 
 func (b *Builder) Get(n string) string {
-	for _, d := range b.del {
+	// Del() removes entries from .add but Set() does not remove from .del, so check .add first.
-		if d == n {
-			return ""
-		}
-	}
 	for _, a := range b.add {
 		if a.Name == n {
 			return a.Value
 		}
 	}
+	if slices.Contains(b.del, n) {
+		return ""
+	}
 	return b.base.Get(n)
 }
 

model/labels/labels_string.go

@@ -593,14 +593,15 @@ func (b *Builder) Set(n, v string) *Builder {
 }
 
 func (b *Builder) Get(n string) string {
-	if slices.Contains(b.del, n) {
+	// Del() removes entries from .add but Set() does not remove from .del, so check .add first.
-		return ""
-	}
 	for _, a := range b.add {
 		if a.Name == n {
 			return a.Value
 		}
 	}
+	if slices.Contains(b.del, n) {
+		return ""
+	}
 	return b.base.Get(n)
 }
 

model/labels/labels_test.go

@@ -607,6 +607,13 @@ func TestBuilder(t *testing.T) {
 			require.Equal(t, tcase.want.BytesWithoutLabels(nil, "aaa", "bbb"), b.Labels().Bytes(nil))
 		})
 	}
+	t.Run("set_after_del", func(t *testing.T) {
+		b := NewBuilder(FromStrings("aaa", "111"))
+		b.Del("bbb")
+		b.Set("bbb", "222")
+		require.Equal(t, FromStrings("aaa", "111", "bbb", "222"), b.Labels())
+		require.Equal(t, "222", b.Get("bbb"))
+	})
 }
 
 func TestScratchBuilder(t *testing.T) {

model/relabel/relabel_test.go

@@ -397,6 +397,34 @@ func TestRelabel(t *testing.T) {
 				"foo":              "bar",
 			}),
 		},
+		{ // From https://github.com/prometheus/prometheus/issues/12283
+			input: labels.FromMap(map[string]string{
+				"__meta_kubernetes_pod_container_port_name":         "foo",
+				"__meta_kubernetes_pod_annotation_XXX_metrics_port": "9091",
+			}),
+			relabel: []*Config{
+				{
+					Regex:  MustNewRegexp("^__meta_kubernetes_pod_container_port_name$"),
+					Action: LabelDrop,
+				},
+				{
+					SourceLabels: model.LabelNames{"__meta_kubernetes_pod_annotation_XXX_metrics_port"},
+					Regex:        MustNewRegexp("(.+)"),
+					Action:       Replace,
+					Replacement:  "metrics",
+					TargetLabel:  "__meta_kubernetes_pod_container_port_name",
+				},
+				{
+					SourceLabels: model.LabelNames{"__meta_kubernetes_pod_container_port_name"},
+					Regex:        MustNewRegexp("^metrics$"),
+					Action:       Keep,
+				},
+			},
+			output: labels.FromMap(map[string]string{
+				"__meta_kubernetes_pod_annotation_XXX_metrics_port": "9091",
+				"__meta_kubernetes_pod_container_port_name":         "metrics",
+			}),
+		},
 		{
 			input: labels.FromMap(map[string]string{
 				"a":  "foo",

storage/buffer.go

@@ -242,15 +242,16 @@ func (s fhSample) Type() chunkenc.ValueType {
 type sampleRing struct {
 	delta int64
 
-	// Lookback buffers. We use buf for mixed samples, but one of the three
+	// Lookback buffers. We use iBuf for mixed samples, but one of the three
 	// concrete ones for homogenous samples. (Only one of the four bufs is
 	// allowed to be populated!) This avoids the overhead of the interface
 	// wrapper for the happy (and by far most common) case of homogenous
 	// samples.
-	buf   []tsdbutil.Sample
+	iBuf     []tsdbutil.Sample
-	fBuf  []fSample
+	fBuf     []fSample
-	hBuf  []hSample
+	hBuf     []hSample
-	fhBuf []fhSample
+	fhBuf    []fhSample
+	bufInUse bufType
 
 	i int // Position of most recent element in ring buffer.
 	f int // Position of first element in ring buffer.
@@ -259,6 +260,16 @@ type sampleRing struct {
 	it sampleRingIterator
 }
 
+type bufType int
+
+const (
+	noBuf bufType = iota // Nothing yet stored in sampleRing.
+	iBuf
+	fBuf
+	hBuf
+	fhBuf
+)
+
 // newSampleRing creates a new sampleRing. If you do not know the prefereed
 // value type yet, use a size of 0 (in which case the provided typ doesn't
 // matter). On the first add, a buffer of size 16 will be allocated with the
@@ -278,7 +289,7 @@ func newSampleRing(delta int64, size int, typ chunkenc.ValueType) *sampleRing {
 	case chunkenc.ValFloatHistogram:
 		r.fhBuf = make([]fhSample, size)
 	default:
-		r.buf = make([]tsdbutil.Sample, size)
+		r.iBuf = make([]tsdbutil.Sample, size)
 	}
 	return r
 }
@@ -287,6 +298,7 @@ func (r *sampleRing) reset() {
 	r.l = 0
 	r.i = -1
 	r.f = 0
+	r.bufInUse = noBuf
 }
 
 // Returns the current iterator. Invalidates previously returned iterators.
@@ -310,18 +322,18 @@ func (it *sampleRingIterator) Next() chunkenc.ValueType {
 	if it.i >= it.r.l {
 		return chunkenc.ValNone
 	}
-	switch {
+	switch it.r.bufInUse {
-	case len(it.r.fBuf) > 0:
+	case fBuf:
 		s := it.r.atF(it.i)
 		it.t = s.t
 		it.f = s.f
 		return chunkenc.ValFloat
-	case len(it.r.hBuf) > 0:
+	case hBuf:
 		s := it.r.atH(it.i)
 		it.t = s.t
 		it.h = s.h
 		return chunkenc.ValHistogram
-	case len(it.r.fhBuf) > 0:
+	case fhBuf:
 		s := it.r.atFH(it.i)
 		it.t = s.t
 		it.fh = s.fh
@@ -372,8 +384,8 @@ func (it *sampleRingIterator) AtT() int64 {
 }
 
 func (r *sampleRing) at(i int) tsdbutil.Sample {
-	j := (r.f + i) % len(r.buf)
+	j := (r.f + i) % len(r.iBuf)
-	return r.buf[j]
+	return r.iBuf[j]
 }
 
 func (r *sampleRing) atF(i int) fSample {
@@ -397,91 +409,113 @@ func (r *sampleRing) atFH(i int) fhSample {
 // from this package (fSample, hSample, fhSample), call one of the specialized
 // methods addF, addH, or addFH for better performance.
 func (r *sampleRing) add(s tsdbutil.Sample) {
-	if len(r.buf) == 0 {
+	if r.bufInUse == noBuf {
+		// First sample.
+		switch s := s.(type) {
+		case fSample:
+			r.bufInUse = fBuf
+			r.fBuf = addF(s, r.fBuf, r)
+		case hSample:
+			r.bufInUse = hBuf
+			r.hBuf = addH(s, r.hBuf, r)
+		case fhSample:
+			r.bufInUse = fhBuf
+			r.fhBuf = addFH(s, r.fhBuf, r)
+		}
+		return
+	}
+	if r.bufInUse != iBuf {
 		// Nothing added to the interface buf yet. Let's check if we can
 		// stay specialized.
 		switch s := s.(type) {
 		case fSample:
-			if len(r.hBuf)+len(r.fhBuf) == 0 {
+			if r.bufInUse == fBuf {
 				r.fBuf = addF(s, r.fBuf, r)
 				return
 			}
 		case hSample:
-			if len(r.fBuf)+len(r.fhBuf) == 0 {
+			if r.bufInUse == hBuf {
 				r.hBuf = addH(s, r.hBuf, r)
 				return
 			}
 		case fhSample:
-			if len(r.fBuf)+len(r.hBuf) == 0 {
+			if r.bufInUse == fhBuf {
 				r.fhBuf = addFH(s, r.fhBuf, r)
 				return
 			}
 		}
 		// The new sample isn't a fit for the already existing
 		// ones. Copy the latter into the interface buffer where needed.
-		switch {
+		switch r.bufInUse {
-		case len(r.fBuf) > 0:
+		case fBuf:
 			for _, s := range r.fBuf {
-				r.buf = append(r.buf, s)
+				r.iBuf = append(r.iBuf, s)
 			}
 			r.fBuf = nil
-		case len(r.hBuf) > 0:
+		case hBuf:
 			for _, s := range r.hBuf {
-				r.buf = append(r.buf, s)
+				r.iBuf = append(r.iBuf, s)
 			}
 			r.hBuf = nil
-		case len(r.fhBuf) > 0:
+		case fhBuf:
 			for _, s := range r.fhBuf {
-				r.buf = append(r.buf, s)
+				r.iBuf = append(r.iBuf, s)
 			}
 			r.fhBuf = nil
 		}
+		r.bufInUse = iBuf
 	}
-	r.buf = addSample(s, r.buf, r)
+	r.iBuf = addSample(s, r.iBuf, r)
 }
 
 // addF is a version of the add method specialized for fSample.
 func (r *sampleRing) addF(s fSample) {
-	switch {
+	switch r.bufInUse {
-	case len(r.buf) > 0:
+	case fBuf: // Add to existing fSamples.
-		// Already have interface samples. Add to the interface buf.
+		r.fBuf = addF(s, r.fBuf, r)
-		r.buf = addSample(s, r.buf, r)
+	case noBuf: // Add first sample.
-	case len(r.hBuf)+len(r.fhBuf) > 0:
+		r.fBuf = addF(s, r.fBuf, r)
+		r.bufInUse = fBuf
+	case iBuf: // Already have interface samples. Add to the interface buf.
+		r.iBuf = addSample(s, r.iBuf, r)
+	default:
 		// Already have specialized samples that are not fSamples.
 		// Need to call the checked add method for conversion.
 		r.add(s)
-	default:
-		r.fBuf = addF(s, r.fBuf, r)
 	}
 }
 
 // addH is a version of the add method specialized for hSample.
 func (r *sampleRing) addH(s hSample) {
-	switch {
+	switch r.bufInUse {
-	case len(r.buf) > 0:
+	case hBuf: // Add to existing hSamples.
-		// Already have interface samples. Add to the interface buf.
+		r.hBuf = addH(s, r.hBuf, r)
-		r.buf = addSample(s, r.buf, r)
+	case noBuf: // Add first sample.
-	case len(r.fBuf)+len(r.fhBuf) > 0:
+		r.hBuf = addH(s, r.hBuf, r)
-		// Already have samples that are not hSamples.
+		r.bufInUse = hBuf
+	case iBuf: // Already have interface samples. Add to the interface buf.
+		r.iBuf = addSample(s, r.iBuf, r)
+	default:
+		// Already have specialized samples that are not hSamples.
 		// Need to call the checked add method for conversion.
 		r.add(s)
-	default:
-		r.hBuf = addH(s, r.hBuf, r)
 	}
 }
 
 // addFH is a version of the add method specialized for fhSample.
 func (r *sampleRing) addFH(s fhSample) {
-	switch {
+	switch r.bufInUse {
-	case len(r.buf) > 0:
+	case fhBuf: // Add to existing fhSamples.
-		// Already have interface samples. Add to the interface buf.
+		r.fhBuf = addFH(s, r.fhBuf, r)
-		r.buf = addSample(s, r.buf, r)
+	case noBuf: // Add first sample.
-	case len(r.fBuf)+len(r.hBuf) > 0:
+		r.fhBuf = addFH(s, r.fhBuf, r)
-		// Already have samples that are not fhSamples.
+		r.bufInUse = fhBuf
+	case iBuf: // Already have interface samples. Add to the interface buf.
+		r.iBuf = addSample(s, r.iBuf, r)
+	default:
+		// Already have specialized samples that are not fhSamples.
 		// Need to call the checked add method for conversion.
 		r.add(s)
-	default:
-		r.fhBuf = addFH(s, r.fhBuf, r)
 	}
 }
 
@@ -701,15 +735,15 @@ func (r *sampleRing) reduceDelta(delta int64) bool {
 		return true
 	}
 
-	switch {
+	switch r.bufInUse {
-	case len(r.fBuf) > 0:
+	case fBuf:
 		genericReduceDelta(r.fBuf, r)
-	case len(r.hBuf) > 0:
+	case hBuf:
 		genericReduceDelta(r.hBuf, r)
-	case len(r.fhBuf) > 0:
+	case fhBuf:
 		genericReduceDelta(r.fhBuf, r)
 	default:
-		genericReduceDelta(r.buf, r)
+		genericReduceDelta(r.iBuf, r)
 	}
 	return true
 }
@@ -733,12 +767,12 @@ func (r *sampleRing) nthLast(n int) (tsdbutil.Sample, bool) {
 		return fSample{}, false
 	}
 	i := r.l - n
-	switch {
+	switch r.bufInUse {
-	case len(r.fBuf) > 0:
+	case fBuf:
 		return r.atF(i), true
-	case len(r.hBuf) > 0:
+	case hBuf:
 		return r.atH(i), true
-	case len(r.fhBuf) > 0:
+	case fhBuf:
 		return r.atFH(i), true
 	default:
 		return r.at(i), true
@@ -751,15 +785,15 @@ func (r *sampleRing) samples() []tsdbutil.Sample {
 	k := r.f + r.l
 	var j int
 
-	switch {
+	switch r.bufInUse {
-	case len(r.buf) > 0:
+	case iBuf:
-		if k > len(r.buf) {
+		if k > len(r.iBuf) {
-			k = len(r.buf)
+			k = len(r.iBuf)
 			j = r.l - k + r.f
 		}
-		n := copy(res, r.buf[r.f:k])
+		n := copy(res, r.iBuf[r.f:k])
-		copy(res[n:], r.buf[:j])
+		copy(res[n:], r.iBuf[:j])
-	case len(r.fBuf) > 0:
+	case fBuf:
 		if k > len(r.fBuf) {
 			k = len(r.fBuf)
 			j = r.l - k + r.f
@@ -770,7 +804,7 @@ func (r *sampleRing) samples() []tsdbutil.Sample {
 		for i, s := range resF {
 			res[i] = s
 		}
-	case len(r.hBuf) > 0:
+	case hBuf:
 		if k > len(r.hBuf) {
 			k = len(r.hBuf)
 			j = r.l - k + r.f
@@ -781,7 +815,7 @@ func (r *sampleRing) samples() []tsdbutil.Sample {
 		for i, s := range resH {
 			res[i] = s
 		}
-	case len(r.fhBuf) > 0:
+	case fhBuf:
 		if k > len(r.fhBuf) {
 			k = len(r.fhBuf)
 			j = r.l - k + r.f

web/ui/module/codemirror-promql/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@prometheus-io/codemirror-promql",
-  "version": "0.43.0",
+  "version": "0.44.0",
   "description": "a CodeMirror mode for the PromQL language",
   "types": "dist/esm/index.d.ts",
   "module": "dist/esm/index.js",
@@ -29,7 +29,7 @@
   },
   "homepage": "https://github.com/prometheus/prometheus/blob/main/web/ui/module/codemirror-promql/README.md",
   "dependencies": {
-    "@prometheus-io/lezer-promql": "0.43.0",
+    "@prometheus-io/lezer-promql": "0.44.0",
     "lru-cache": "^6.0.0"
   },
   "devDependencies": {

web/ui/module/lezer-promql/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@prometheus-io/lezer-promql",
-  "version": "0.43.0",
+  "version": "0.44.0",
   "description": "lezer-based PromQL grammar",
   "main": "dist/index.cjs",
   "type": "module",

web/ui/package-lock.json

@@ -28,10 +28,10 @@
     },
     "module/codemirror-promql": {
       "name": "@prometheus-io/codemirror-promql",
-      "version": "0.43.0",
+      "version": "0.44.0",
       "license": "Apache-2.0",
       "dependencies": {
-        "@prometheus-io/lezer-promql": "0.43.0",
+        "@prometheus-io/lezer-promql": "0.44.0",
         "lru-cache": "^6.0.0"
       },
       "devDependencies": {
@@ -61,7 +61,7 @@
     },
     "module/lezer-promql": {
       "name": "@prometheus-io/lezer-promql",
-      "version": "0.43.0",
+      "version": "0.44.0",
       "license": "Apache-2.0",
       "devDependencies": {
         "@lezer/generator": "^1.2.2",
@@ -20763,7 +20763,7 @@
     },
     "react-app": {
       "name": "@prometheus-io/app",
-      "version": "0.43.0",
+      "version": "0.44.0",
       "dependencies": {
         "@codemirror/autocomplete": "^6.4.0",
         "@codemirror/commands": "^6.2.0",
@@ -20781,7 +20781,7 @@
         "@lezer/lr": "^1.3.1",
         "@nexucis/fuzzy": "^0.4.1",
         "@nexucis/kvsearch": "^0.8.1",
-        "@prometheus-io/codemirror-promql": "0.43.0",
+        "@prometheus-io/codemirror-promql": "0.44.0",
         "bootstrap": "^4.6.2",
         "css.escape": "^1.5.1",
         "downshift": "^7.2.0",
@@ -23417,7 +23417,7 @@
         "@lezer/lr": "^1.3.1",
         "@nexucis/fuzzy": "^0.4.1",
         "@nexucis/kvsearch": "^0.8.1",
-        "@prometheus-io/codemirror-promql": "0.43.0",
+        "@prometheus-io/codemirror-promql": "0.44.0",
         "@testing-library/react-hooks": "^7.0.2",
         "@types/enzyme": "^3.10.12",
         "@types/flot": "0.0.32",
@@ -23468,7 +23468,7 @@
         "@lezer/common": "^1.0.2",
         "@lezer/highlight": "^1.1.3",
         "@lezer/lr": "^1.3.1",
-        "@prometheus-io/lezer-promql": "0.43.0",
+        "@prometheus-io/lezer-promql": "0.44.0",
         "@types/lru-cache": "^5.1.1",
         "isomorphic-fetch": "^3.0.0",
         "lru-cache": "^6.0.0",

web/ui/react-app/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@prometheus-io/app",
-  "version": "0.43.0",
+  "version": "0.44.0",
   "private": true,
   "dependencies": {
     "@codemirror/autocomplete": "^6.4.0",
@@ -19,7 +19,7 @@
     "@lezer/common": "^1.0.2",
     "@nexucis/fuzzy": "^0.4.1",
     "@nexucis/kvsearch": "^0.8.1",
-    "@prometheus-io/codemirror-promql": "0.43.0",
+    "@prometheus-io/codemirror-promql": "0.44.0",
     "bootstrap": "^4.6.2",
     "css.escape": "^1.5.1",
     "downshift": "^7.2.0",