Merge pull request #13057 from prometheus/release-2.48

Merge release-2.48 back into main

CHANGELOG.md

@@ -1,5 +1,40 @@
 # Changelog
 
+## 2.48.0-rc.1 / 2023-10-24
+
+* [BUGFIX] PromQL: Reduce inefficiency introduced by warnings/annotations and temporarily remove possible non-counter warnings. #13012
+
+## 2.48.0-rc.0 / 2023-10-10
+
+* [CHANGE] Remote-write: respect Retry-After header on 5xx errors. #12677
+* [FEATURE] Alerting: Add AWS SigV4 authentication support for Alertmanager endpoints. #12774
+* [FEATURE] Promtool: Add support for histograms in the TSDB dump command. #12775
+* [FEATURE] PromQL: Add warnings (and annotations) to PromQL query results. #12152 #12982 #12988
+* [FEATURE] Remote-write: Add Azure AD OAuth authentication support for remote write requests. #12572
+* [ENHANCEMENT] Remote-write: Add a header to count retried remote write requests. #12729
+* [ENHANCEMENT] TSDB: Improve query performance by re-using iterator when moving between series. #12757
+* [ENHANCEMENT] UI: Move /targets page discovered labels to expandable section #12824
+* [ENHANCEMENT] TSDB: Optimize WBL loading by not sending empty buffers over channel. #12808
+* [ENHANCEMENT] TSDB: Reply WBL mmap markers concurrently. #12801
+* [ENHANCEMENT] Promtool: Add support for specifying series matchers in the TSDB analyze command. #12842
+* [ENHANCEMENT] PromQL: Prevent Prometheus from overallocating memory on subquery with large amount of steps. #12734
+* [ENHANCEMENT] PromQL: Add warning when monotonicity is forced in the input to histogram_quantile. #12931
+* [ENHANCEMENT] Scraping: Optimize sample appending by reducing garbage. #12939
+* [ENHANCEMENT] Storage: Reduce memory allocations in queries that merge series sets. #12938
+* [ENHANCEMENT] UI: Show group interval in rules display. #12943
+* [ENHANCEMENT] Scraping: Save memory when scraping by delaying creation of buffer. #12953
+* [ENHANCEMENT] Agent: Allow ingestion of out-of-order samples. #12897
+* [ENHANCEMENT] Promtool: Improve support for native histograms in TSDB analyze command. #12869
+* [BUGFIX] SD: Ensure that discovery managers are properly canceled. #10569
+* [BUGFIX] TSDB: Fix PostingsForMatchers race with creating new series. #12558
+* [BUGFIX] TSDB: Fix handling of explicit counter reset header in histograms. #12772
+* [BUGFIX] SD: Validate HTTP client configuration in HTTP, EC2, Azure, Uyuni, PuppetDB, and Lightsail SDs. #12762 #12811 #12812 #12815 #12814 #12816
+* [BUGFIX] TSDB: Fix counter reset edgecases causing native histogram panics. #12838
+* [BUGFIX] TSDB: Fix duplicate sample detection at chunk size limit. #12874
+* [BUGFIX] Promtool: Fix errors not being reported in check rules command. #12715
+* [BUGFIX] TSDB: Avoid panics reported in logs when head initialization takes a long time. #12876
+* [BUGFIX] TSDB: Ensure that WBL is repaired when possible. #12406
+
 ## 2.47.1 / 2023-10-04
 
 * [BUGFIX] Fix duplicate sample detection at chunk size limit #12874

VERSION

@@ -1 +1 @@
-2.47.1
+2.48.0-rc.1

promql/engine.go

@@ -2546,7 +2546,7 @@ type groupedAggregation struct {
 func (ev *evaluator) aggregation(e *parser.AggregateExpr, grouping []string, param interface{}, vec Vector, seriesHelper []EvalSeriesHelper, enh *EvalNodeHelper) (Vector, annotations.Annotations) {
 	op := e.Op
 	without := e.Without
-	annos := annotations.Annotations{}
+	var annos annotations.Annotations
 	result := map[uint64]*groupedAggregation{}
 	orderedResult := []*groupedAggregation{}
 	var k int64

promql/functions.go

@@ -76,7 +76,7 @@ func extrapolatedRate(vals []parser.Value, args parser.Expressions, enh *EvalNod
 		resultHistogram    *histogram.FloatHistogram
 		firstT, lastT      int64
 		numSamplesMinusOne int
-		annos              = annotations.Annotations{}
+		annos              annotations.Annotations
 	)
 
 	// We need either at least two Histograms and no Floats, or at least two
@@ -87,14 +87,6 @@ func extrapolatedRate(vals []parser.Value, args parser.Expressions, enh *EvalNod
 		return enh.Out, annos.Add(annotations.NewMixedFloatsHistogramsWarning(metricName, args[0].PositionRange()))
 	}
 
-	if isCounter && metricName != "" && len(samples.Floats) > 0 &&
-		!strings.HasSuffix(metricName, "_total") &&
-		!strings.HasSuffix(metricName, "_sum") &&
-		!strings.HasSuffix(metricName, "_count") &&
-		!strings.HasSuffix(metricName, "_bucket") {
-		annos.Add(annotations.NewPossibleNonCounterInfo(metricName, args[0].PositionRange()))
-	}
-
 	switch {
 	case len(samples.Histograms) > 1:
 		numSamplesMinusOne = len(samples.Histograms) - 1
@@ -641,7 +633,7 @@ func funcQuantileOverTime(vals []parser.Value, args parser.Expressions, enh *Eva
 		return enh.Out, nil
 	}
 
-	annos := annotations.Annotations{}
+	var annos annotations.Annotations
 	if math.IsNaN(q) || q < 0 || q > 1 {
 		annos.Add(annotations.NewInvalidQuantileWarning(q, args[0].PositionRange()))
 	}
@@ -1104,7 +1096,7 @@ func funcHistogramFraction(vals []parser.Value, args parser.Expressions, enh *Ev
 func funcHistogramQuantile(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) {
 	q := vals[0].(Vector)[0].F
 	inVec := vals[1].(Vector)
-	annos := annotations.Annotations{}
+	var annos annotations.Annotations
 
 	if math.IsNaN(q) || q < 0 || q > 1 {
 		annos.Add(annotations.NewInvalidQuantileWarning(q, args[0].PositionRange()))

storage/buffer.go

@@ -284,7 +284,8 @@ func newSampleRing(delta int64, size int, typ chunkenc.ValueType) *sampleRing {
 	case chunkenc.ValFloatHistogram:
 		r.fhBuf = make([]fhSample, size)
 	default:
-		r.iBuf = make([]chunks.Sample, size)
+		// Do not initialize anything because the 1st sample will be
+		// added to one of the other bufs anyway.
 	}
 	return r
 }
@@ -294,6 +295,12 @@ func (r *sampleRing) reset() {
 	r.i = -1
 	r.f = 0
 	r.bufInUse = noBuf
+
+	// The first sample after the reset will always go to a specialized
+	// buffer. If we later need to change to the interface buffer, we'll
+	// copy from the specialized buffer to the interface buffer. For that to
+	// work properly, we have to reset the interface buffer here, too.
+	r.iBuf = r.iBuf[:0]
 }
 
 // Returns the current iterator. Invalidates previously returned iterators.
@@ -441,6 +448,7 @@ func (r *sampleRing) add(s chunks.Sample) {
 		}
 		// The new sample isn't a fit for the already existing
 		// ones. Copy the latter into the interface buffer where needed.
+		// The interface buffer is assumed to be of length zero at this point.
 		switch r.bufInUse {
 		case fBuf:
 			for _, s := range r.fBuf {

storage/buffer_test.go

@@ -90,6 +90,54 @@ func TestSampleRing(t *testing.T) {
 	}
 }
 
+func TestSampleRingMixed(t *testing.T) {
+	h1 := tsdbutil.GenerateTestHistogram(1)
+	h2 := tsdbutil.GenerateTestHistogram(2)
+
+	// With ValNone as the preferred type, nothing should be initialized.
+	r := newSampleRing(10, 2, chunkenc.ValNone)
+	require.Zero(t, len(r.fBuf))
+	require.Zero(t, len(r.hBuf))
+	require.Zero(t, len(r.fhBuf))
+	require.Zero(t, len(r.iBuf))
+
+	// But then mixed adds should work as expected.
+	r.addF(fSample{t: 1, f: 3.14})
+	r.addH(hSample{t: 2, h: h1})
+
+	it := r.iterator()
+
+	require.Equal(t, chunkenc.ValFloat, it.Next())
+	ts, f := it.At()
+	require.Equal(t, int64(1), ts)
+	require.Equal(t, 3.14, f)
+	require.Equal(t, chunkenc.ValHistogram, it.Next())
+	var h *histogram.Histogram
+	ts, h = it.AtHistogram()
+	require.Equal(t, int64(2), ts)
+	require.Equal(t, h1, h)
+	require.Equal(t, chunkenc.ValNone, it.Next())
+
+	r.reset()
+	it = r.iterator()
+	require.Equal(t, chunkenc.ValNone, it.Next())
+
+	r.addF(fSample{t: 3, f: 4.2})
+	r.addH(hSample{t: 4, h: h2})
+
+	it = r.iterator()
+
+	require.Equal(t, chunkenc.ValFloat, it.Next())
+	ts, f = it.At()
+	require.Equal(t, int64(3), ts)
+	require.Equal(t, 4.2, f)
+	require.Equal(t, chunkenc.ValHistogram, it.Next())
+	ts, h = it.AtHistogram()
+	require.Equal(t, int64(4), ts)
+	require.Equal(t, h2, h)
+	require.Equal(t, chunkenc.ValNone, it.Next())
+}
+
 func TestBufferedSeriesIterator(t *testing.T) {
 	var it *BufferedSeriesIterator
 

tsdb/db_test.go

@@ -1946,7 +1946,8 @@ func TestQuerierWithBoundaryChunks(t *testing.T) {
 	// The requested interval covers 2 blocks, so the querier's label values for blockID should give us 2 values, one from each block.
 	b, ws, err := q.LabelValues(ctx, "blockID")
 	require.NoError(t, err)
-	require.Equal(t, annotations.Annotations{}, ws)
+	var nilAnnotations annotations.Annotations
+	require.Equal(t, nilAnnotations, ws)
 	require.Equal(t, []string{"1", "2"}, b)
 }
 

util/annotations/annotations.go

@@ -50,6 +50,9 @@ func (a *Annotations) Add(err error) Annotations {
 // the first in-place, and returns the merged first Annotation for convenience.
 func (a *Annotations) Merge(aa Annotations) Annotations {
 	if *a == nil {
+		if aa == nil {
+			return nil
+		}
 		*a = Annotations{}
 	}
 	for key, val := range aa {

web/ui/module/codemirror-promql/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@prometheus-io/codemirror-promql",
-  "version": "0.47.0",
+  "version": "0.48.0-rc.1",
   "description": "a CodeMirror mode for the PromQL language",
   "types": "dist/esm/index.d.ts",
   "module": "dist/esm/index.js",
@@ -29,7 +29,7 @@
   },
   "homepage": "https://github.com/prometheus/prometheus/blob/main/web/ui/module/codemirror-promql/README.md",
   "dependencies": {
-    "@prometheus-io/lezer-promql": "0.47.0",
+    "@prometheus-io/lezer-promql": "0.48.0-rc.1",
     "lru-cache": "^7.18.3"
   },
   "devDependencies": {

web/ui/module/lezer-promql/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@prometheus-io/lezer-promql",
-  "version": "0.47.0",
+  "version": "0.48.0-rc.1",
   "description": "lezer-based PromQL grammar",
   "main": "dist/index.cjs",
   "type": "module",

web/ui/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "prometheus-io",
-  "version": "0.46.0",
+  "version": "0.48.0-rc.1",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "prometheus-io",
-      "version": "0.46.0",
+      "version": "0.48.0-rc.1",
       "workspaces": [
         "react-app",
         "module/*"
@@ -30,10 +30,10 @@
     },
     "module/codemirror-promql": {
       "name": "@prometheus-io/codemirror-promql",
-      "version": "0.47.0",
+      "version": "0.48.0-rc.1",
       "license": "Apache-2.0",
       "dependencies": {
-        "@prometheus-io/lezer-promql": "0.47.0",
+        "@prometheus-io/lezer-promql": "0.48.0-rc.1",
         "lru-cache": "^7.18.3"
       },
       "devDependencies": {
@@ -70,7 +70,7 @@
     },
     "module/lezer-promql": {
       "name": "@prometheus-io/lezer-promql",
-      "version": "0.47.0",
+      "version": "0.48.0-rc.1",
       "license": "Apache-2.0",
       "devDependencies": {
         "@lezer/generator": "^1.2.3",
@@ -20764,7 +20764,7 @@
     },
     "react-app": {
       "name": "@prometheus-io/app",
-      "version": "0.47.0",
+      "version": "0.48.0-rc.1",
       "dependencies": {
         "@codemirror/autocomplete": "^6.7.1",
         "@codemirror/commands": "^6.2.4",
@@ -20782,7 +20782,7 @@
         "@lezer/lr": "^1.3.6",
         "@nexucis/fuzzy": "^0.4.1",
         "@nexucis/kvsearch": "^0.8.1",
-        "@prometheus-io/codemirror-promql": "0.47.0",
+        "@prometheus-io/codemirror-promql": "0.48.0-rc.1",
         "bootstrap": "^4.6.2",
         "css.escape": "^1.5.1",
         "downshift": "^7.6.0",
@@ -23422,7 +23422,7 @@
         "@lezer/lr": "^1.3.6",
         "@nexucis/fuzzy": "^0.4.1",
         "@nexucis/kvsearch": "^0.8.1",
-        "@prometheus-io/codemirror-promql": "0.47.0",
+        "@prometheus-io/codemirror-promql": "0.48.0-rc.1",
         "@testing-library/react-hooks": "^7.0.2",
         "@types/enzyme": "^3.10.13",
         "@types/flot": "0.0.32",
@@ -23486,7 +23486,7 @@
         "@lezer/common": "^1.0.3",
         "@lezer/highlight": "^1.1.6",
         "@lezer/lr": "^1.3.6",
-        "@prometheus-io/lezer-promql": "0.47.0",
+        "@prometheus-io/lezer-promql": "0.48.0-rc.1",
         "isomorphic-fetch": "^3.0.0",
         "lru-cache": "^7.18.3",
         "nock": "^13.3.1"

web/ui/package.json

@@ -28,5 +28,5 @@
     "ts-jest": "^29.1.0",
     "typescript": "^4.9.5"
   },
-  "version": "0.46.0"
+  "version": "0.48.0-rc.1"
 }

web/ui/react-app/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@prometheus-io/app",
-  "version": "0.47.0",
+  "version": "0.48.0-rc.1",
   "private": true,
   "dependencies": {
     "@codemirror/autocomplete": "^6.7.1",
@@ -19,7 +19,7 @@
     "@lezer/lr": "^1.3.6",
     "@nexucis/fuzzy": "^0.4.1",
     "@nexucis/kvsearch": "^0.8.1",
-    "@prometheus-io/codemirror-promql": "0.47.0",
+    "@prometheus-io/codemirror-promql": "0.48.0-rc.1",
     "bootstrap": "^4.6.2",
     "css.escape": "^1.5.1",
     "downshift": "^7.6.0",