Merge pull request #13623 from prometheus/njpm/rw2-sync-main

remote write 2.0: sync with `main` branch

Makefile.common

@@ -62,10 +62,10 @@ SKIP_GOLANGCI_LINT :=
 GOLANGCI_LINT :=
 GOLANGCI_LINT_OPTS ?=
 GOLANGCI_LINT_VERSION ?= v1.55.2
-# golangci-lint only supports linux, darwin and windows platforms on i386/amd64.
+# golangci-lint only supports linux, darwin and windows platforms on i386/amd64/arm64.
 # windows isn't included here because of the path separator being different.
 ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin))
-	ifeq ($(GOHOSTARCH),$(filter $(GOHOSTARCH),amd64 i386))
+	ifeq ($(GOHOSTARCH),$(filter $(GOHOSTARCH),amd64 i386 arm64))
 		# If we're in CI and there is an Actions file, that means the linter
 		# is being run in Actions, so we don't need to run it here.
 		ifneq (,$(SKIP_GOLANGCI_LINT))

README.md

@@ -149,7 +149,7 @@ We are publishing our Remote Write protobuf independently at
 You can use that as a library:
 
 ```shell
-go get go.buf.build/protocolbuffers/go/prometheus/prometheus
+go get buf.build/gen/go/prometheus/prometheus/protocolbuffers/go@latest
 ```
 
 This is experimental.

cmd/prometheus/main_test.go

@@ -126,12 +126,9 @@ func TestFailedStartupExitCode(t *testing.T) {
 	require.Error(t, err)
 
 	var exitError *exec.ExitError
-	if errors.As(err, &exitError) {
+	require.ErrorAs(t, err, &exitError)
-		status := exitError.Sys().(syscall.WaitStatus)
+	status := exitError.Sys().(syscall.WaitStatus)
-		require.Equal(t, expectedExitStatus, status.ExitStatus())
+	require.Equal(t, expectedExitStatus, status.ExitStatus())
-	} else {
-		t.Errorf("unable to retrieve the exit status for prometheus: %v", err)
-	}
 }
 
 type senderFunc func(alerts ...*notifier.Alert)
@@ -194,9 +191,7 @@ func TestSendAlerts(t *testing.T) {
 		tc := tc
 		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
 			senderFunc := senderFunc(func(alerts ...*notifier.Alert) {
-				if len(tc.in) == 0 {
+				require.NotEmpty(t, tc.in, "sender called with 0 alert")
-					t.Fatalf("sender called with 0 alert")
-				}
 				require.Equal(t, tc.exp, alerts)
 			})
 			rules.SendAlerts(senderFunc, "http://localhost:9090")(context.TODO(), "up", tc.in...)
@@ -228,7 +223,7 @@ func TestWALSegmentSizeBounds(t *testing.T) {
 			go func() { done <- prom.Wait() }()
 			select {
 			case err := <-done:
-				t.Errorf("prometheus should be still running: %v", err)
+				require.Fail(t, "prometheus should be still running: %v", err)
 			case <-time.After(startupTime):
 				prom.Process.Kill()
 				<-done
@@ -239,12 +234,9 @@ func TestWALSegmentSizeBounds(t *testing.T) {
 		err = prom.Wait()
 		require.Error(t, err)
 		var exitError *exec.ExitError
-		if errors.As(err, &exitError) {
+		require.ErrorAs(t, err, &exitError)
-			status := exitError.Sys().(syscall.WaitStatus)
+		status := exitError.Sys().(syscall.WaitStatus)
-			require.Equal(t, expectedExitStatus, status.ExitStatus())
+		require.Equal(t, expectedExitStatus, status.ExitStatus())
-		} else {
-			t.Errorf("unable to retrieve the exit status for prometheus: %v", err)
-		}
 	}
 }
 
@@ -274,7 +266,7 @@ func TestMaxBlockChunkSegmentSizeBounds(t *testing.T) {
 			go func() { done <- prom.Wait() }()
 			select {
 			case err := <-done:
-				t.Errorf("prometheus should be still running: %v", err)
+				require.Fail(t, "prometheus should be still running: %v", err)
 			case <-time.After(startupTime):
 				prom.Process.Kill()
 				<-done
@@ -285,12 +277,9 @@ func TestMaxBlockChunkSegmentSizeBounds(t *testing.T) {
 		err = prom.Wait()
 		require.Error(t, err)
 		var exitError *exec.ExitError
-		if errors.As(err, &exitError) {
+		require.ErrorAs(t, err, &exitError)
-			status := exitError.Sys().(syscall.WaitStatus)
+		status := exitError.Sys().(syscall.WaitStatus)
-			require.Equal(t, expectedExitStatus, status.ExitStatus())
+		require.Equal(t, expectedExitStatus, status.ExitStatus())
-		} else {
-			t.Errorf("unable to retrieve the exit status for prometheus: %v", err)
-		}
 	}
 }
 
@@ -347,10 +336,8 @@ func getCurrentGaugeValuesFor(t *testing.T, reg prometheus.Gatherer, metricNames
 			}
 
 			require.Len(t, g.GetMetric(), 1)
-			if _, ok := res[m]; ok {
+			_, ok := res[m]
-				t.Error("expected only one metric family for", m)
+			require.False(t, ok, "expected only one metric family for", m)
-				t.FailNow()
-			}
 			res[m] = *g.GetMetric()[0].GetGauge().Value
 		}
 	}

cmd/prometheus/main_unix_test.go

@@ -23,6 +23,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/prometheus/prometheus/util/testutil"
 )
 
@@ -37,9 +39,7 @@ func TestStartupInterrupt(t *testing.T) {
 
 	prom := exec.Command(promPath, "-test.main", "--config.file="+promConfig, "--storage.tsdb.path="+t.TempDir(), "--web.listen-address=0.0.0.0"+port)
 	err := prom.Start()
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("execution error: %v", err)
-	}
 
 	done := make(chan error, 1)
 	go func() {
@@ -68,14 +68,11 @@ Loop:
 		time.Sleep(500 * time.Millisecond)
 	}
 
-	if !startedOk {
+	require.True(t, startedOk, "prometheus didn't start in the specified timeout")
-		t.Fatal("prometheus didn't start in the specified timeout")
+	err = prom.Process.Kill()
-	}
+	require.Error(t, err, "prometheus didn't shutdown gracefully after sending the Interrupt signal")
-	switch err := prom.Process.Kill(); {
+	// TODO - find a better way to detect when the process didn't exit as expected!
-	case err == nil:
+	if stoppedErr != nil {
-		t.Errorf("prometheus didn't shutdown gracefully after sending the Interrupt signal")
+		require.EqualError(t, stoppedErr, "signal: interrupt", "prometheus exit")
-	case stoppedErr != nil && stoppedErr.Error() != "signal: interrupt":
-		// TODO: find a better way to detect when the process didn't exit as expected!
-		t.Errorf("prometheus exited with an unexpected error: %v", stoppedErr)
 	}
 }

cmd/promtool/backfill_test.go

@@ -26,6 +26,7 @@ import (
 	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/tsdb"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
+	"github.com/prometheus/prometheus/util/testutil"
 )
 
 type backfillSample struct {
@@ -76,7 +77,7 @@ func testBlocks(t *testing.T, db *tsdb.DB, expectedMinTime, expectedMaxTime, exp
 	allSamples := queryAllSeries(t, q, expectedMinTime, expectedMaxTime)
 	sortSamples(allSamples)
 	sortSamples(expectedSamples)
-	require.Equal(t, expectedSamples, allSamples, "did not create correct samples")
+	testutil.RequireEqual(t, expectedSamples, allSamples, "did not create correct samples")
 
 	if len(allSamples) > 0 {
 		require.Equal(t, expectedMinTime, allSamples[0].Timestamp, "timestamp of first sample is not the expected minimum time")

cmd/promtool/sd.go

@@ -18,10 +18,10 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
-	"reflect"
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/google/go-cmp/cmp"
 	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/prometheus/prometheus/config"
@@ -153,7 +153,7 @@ func getSDCheckResult(targetGroups []*targetgroup.Group, scrapeConfig *config.Sc
 
 			duplicateRes := false
 			for _, sdCheckRes := range sdCheckResults {
-				if reflect.DeepEqual(sdCheckRes, result) {
+				if cmp.Equal(sdCheckRes, result, cmp.Comparer(labels.Equal)) {
 					duplicateRes = true
 					break
 				}

cmd/promtool/sd_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/relabel"
+	"github.com/prometheus/prometheus/util/testutil"
 
 	"github.com/stretchr/testify/require"
 )
@@ -69,5 +70,5 @@ func TestSDCheckResult(t *testing.T) {
 		},
 	}
 
-	require.Equal(t, expectedSDCheckResult, getSDCheckResult(targetGroups, scrapeConfig, true))
+	testutil.RequireEqual(t, expectedSDCheckResult, getSDCheckResult(targetGroups, scrapeConfig, true))
 }

cmd/promtool/unittest.go

@@ -20,13 +20,13 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"reflect"
 	"sort"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/google/go-cmp/cmp"
 	"github.com/grafana/regexp"
 	"github.com/nsf/jsondiff"
 	"github.com/prometheus/common/model"
@@ -340,7 +340,7 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
 				sort.Sort(gotAlerts)
 				sort.Sort(expAlerts)
 
-				if !reflect.DeepEqual(expAlerts, gotAlerts) {
+				if !cmp.Equal(expAlerts, gotAlerts, cmp.Comparer(labels.Equal)) {
 					var testName string
 					if tg.TestGroupName != "" {
 						testName = fmt.Sprintf("    name: %s,\n", tg.TestGroupName)
@@ -448,7 +448,7 @@ Outer:
 		sort.Slice(gotSamples, func(i, j int) bool {
 			return labels.Compare(gotSamples[i].Labels, gotSamples[j].Labels) <= 0
 		})
-		if !reflect.DeepEqual(expSamples, gotSamples) {
+		if !cmp.Equal(expSamples, gotSamples, cmp.Comparer(labels.Equal)) {
 			errs = append(errs, fmt.Errorf("    expr: %q, time: %s,\n        exp: %v\n        got: %v", testCase.Expr,
 				testCase.EvalTime.String(), parsedSamplesString(expSamples), parsedSamplesString(gotSamples)))
 		}

cmd/promtool/unittest_test.go

@@ -16,6 +16,8 @@ package main
 import (
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/prometheus/prometheus/promql"
 )
 
@@ -178,9 +180,8 @@ func TestRulesUnitTestRun(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := RulesUnitTest(tt.queryOpts, tt.args.run, false, tt.args.files...); got != tt.want {
+			got := RulesUnitTest(tt.queryOpts, tt.args.run, false, tt.args.files...)
-				t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want)
+			require.Equal(t, tt.want, got)
-			}
 		})
 	}
 }

config/config_test.go

@@ -58,6 +58,7 @@ import (
 	"github.com/prometheus/prometheus/discovery/zookeeper"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/relabel"
+	"github.com/prometheus/prometheus/util/testutil"
 )
 
 func mustParseURL(u string) *config.URL {
@@ -2037,16 +2038,16 @@ func TestExpandExternalLabels(t *testing.T) {
 
 	c, err := LoadFile("testdata/external_labels.good.yml", false, false, log.NewNopLogger())
 	require.NoError(t, err)
-	require.Equal(t, labels.FromStrings("bar", "foo", "baz", "foo${TEST}bar", "foo", "${TEST}", "qux", "foo$${TEST}", "xyz", "foo$$bar"), c.GlobalConfig.ExternalLabels)
+	testutil.RequireEqual(t, labels.FromStrings("bar", "foo", "baz", "foo${TEST}bar", "foo", "${TEST}", "qux", "foo$${TEST}", "xyz", "foo$$bar"), c.GlobalConfig.ExternalLabels)
 
 	c, err = LoadFile("testdata/external_labels.good.yml", false, true, log.NewNopLogger())
 	require.NoError(t, err)
-	require.Equal(t, labels.FromStrings("bar", "foo", "baz", "foobar", "foo", "", "qux", "foo${TEST}", "xyz", "foo$bar"), c.GlobalConfig.ExternalLabels)
+	testutil.RequireEqual(t, labels.FromStrings("bar", "foo", "baz", "foobar", "foo", "", "qux", "foo${TEST}", "xyz", "foo$bar"), c.GlobalConfig.ExternalLabels)
 
 	os.Setenv("TEST", "TestValue")
 	c, err = LoadFile("testdata/external_labels.good.yml", false, true, log.NewNopLogger())
 	require.NoError(t, err)
-	require.Equal(t, labels.FromStrings("bar", "foo", "baz", "fooTestValuebar", "foo", "TestValue", "qux", "foo${TEST}", "xyz", "foo$bar"), c.GlobalConfig.ExternalLabels)
+	testutil.RequireEqual(t, labels.FromStrings("bar", "foo", "baz", "fooTestValuebar", "foo", "TestValue", "qux", "foo${TEST}", "xyz", "foo$bar"), c.GlobalConfig.ExternalLabels)
 }
 
 func TestAgentMode(t *testing.T) {

discovery/consul/consul_test.go

@@ -56,15 +56,11 @@ func TestConfiguredService(t *testing.T) {
 	metrics := NewTestMetrics(t, conf, prometheus.NewRegistry())
 
 	consulDiscovery, err := NewDiscovery(conf, nil, metrics)
-	if err != nil {
+	require.NoError(t, err, "when initializing discovery")
-		t.Errorf("Unexpected error when initializing discovery %v", err)
+	require.True(t, consulDiscovery.shouldWatch("configuredServiceName", []string{""}),
-	}
+		"Expected service %s to be watched", "configuredServiceName")
-	if !consulDiscovery.shouldWatch("configuredServiceName", []string{""}) {
+	require.False(t, consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{""}),
-		t.Errorf("Expected service %s to be watched", "configuredServiceName")
+		"Expected service %s to not be watched", "nonConfiguredServiceName")
-	}
-	if consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{""}) {
-		t.Errorf("Expected service %s to not be watched", "nonConfiguredServiceName")
-	}
 }
 
 func TestConfiguredServiceWithTag(t *testing.T) {
@@ -76,21 +72,18 @@ func TestConfiguredServiceWithTag(t *testing.T) {
 	metrics := NewTestMetrics(t, conf, prometheus.NewRegistry())
 
 	consulDiscovery, err := NewDiscovery(conf, nil, metrics)
-	if err != nil {
+	require.NoError(t, err, "when initializing discovery")
-		t.Errorf("Unexpected error when initializing discovery %v", err)
+	require.False(t, consulDiscovery.shouldWatch("configuredServiceName", []string{""}),
-	}
+		"Expected service %s to not be watched without tag", "configuredServiceName")
-	if consulDiscovery.shouldWatch("configuredServiceName", []string{""}) {
+
-		t.Errorf("Expected service %s to not be watched without tag", "configuredServiceName")
+	require.True(t, consulDiscovery.shouldWatch("configuredServiceName", []string{"http"}),
-	}
+		"Expected service %s to be watched with tag %s", "configuredServiceName", "http")
-	if !consulDiscovery.shouldWatch("configuredServiceName", []string{"http"}) {
+
-		t.Errorf("Expected service %s to be watched with tag %s", "configuredServiceName", "http")
+	require.False(t, consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{""}),
-	}
+		"Expected service %s to not be watched without tag", "nonConfiguredServiceName")
-	if consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{""}) {
+
-		t.Errorf("Expected service %s to not be watched without tag", "nonConfiguredServiceName")
+	require.False(t, consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{"http"}),
-	}
+		"Expected service %s to not be watched with tag %s", "nonConfiguredServiceName", "http")
-	if consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{"http"}) {
-		t.Errorf("Expected service %s to not be watched with tag %s", "nonConfiguredServiceName", "http")
-	}
 }
 
 func TestConfiguredServiceWithTags(t *testing.T) {
@@ -173,13 +166,10 @@ func TestConfiguredServiceWithTags(t *testing.T) {
 		metrics := NewTestMetrics(t, tc.conf, prometheus.NewRegistry())
 
 		consulDiscovery, err := NewDiscovery(tc.conf, nil, metrics)
-		if err != nil {
+		require.NoError(t, err, "when initializing discovery")
-			t.Errorf("Unexpected error when initializing discovery %v", err)
-		}
 		ret := consulDiscovery.shouldWatch(tc.serviceName, tc.serviceTags)
-		if ret != tc.shouldWatch {
+		require.Equal(t, tc.shouldWatch, ret, "Watched service and tags: %s %+v, input was %s %+v",
-			t.Errorf("Expected should watch? %t, got %t. Watched service and tags: %s %+v, input was %s %+v", tc.shouldWatch, ret, tc.conf.Services, tc.conf.ServiceTags, tc.serviceName, tc.serviceTags)
+			tc.conf.Services, tc.conf.ServiceTags, tc.serviceName, tc.serviceTags)
-		}
 	}
 }
 
@@ -189,12 +179,8 @@ func TestNonConfiguredService(t *testing.T) {
 	metrics := NewTestMetrics(t, conf, prometheus.NewRegistry())
 
 	consulDiscovery, err := NewDiscovery(conf, nil, metrics)
-	if err != nil {
+	require.NoError(t, err, "when initializing discovery")
-		t.Errorf("Unexpected error when initializing discovery %v", err)
+	require.True(t, consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{""}), "Expected service %s to be watched", "nonConfiguredServiceName")
-	}
-	if !consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{""}) {
-		t.Errorf("Expected service %s to be watched", "nonConfiguredServiceName")
-	}
 }
 
 const (
@@ -502,13 +488,10 @@ oauth2:
 			var config SDConfig
 			err := config.UnmarshalYAML(unmarshal([]byte(test.config)))
 			if err != nil {
-				require.Equalf(t, err.Error(), test.errMessage, "Expected error '%s', got '%v'", test.errMessage, err)
+				require.EqualError(t, err, test.errMessage)
-				return
-			}
-			if test.errMessage != "" {
-				t.Errorf("Expected error %s, got none", test.errMessage)
 				return
 			}
+			require.Empty(t, test.errMessage, "Expected error.")
 
 			require.Equal(t, test.expected, config)
 		})

discovery/file/file_test.go

@@ -358,9 +358,7 @@ func TestInvalidFile(t *testing.T) {
 
 			// Verify that we've received nothing.
 			time.Sleep(defaultWait)
-			if runner.lastReceive().After(now) {
+			require.False(t, runner.lastReceive().After(now), "unexpected targets received: %v", runner.targets())
-				t.Fatalf("unexpected targets received: %v", runner.targets())
-			}
 		})
 	}
 }

discovery/kubernetes/kubernetes_test.go

@@ -131,14 +131,8 @@ func (d k8sDiscoveryTest) Run(t *testing.T) {
 	go readResultWithTimeout(t, ch, d.expectedMaxItems, time.Second, resChan)
 
 	dd, ok := d.discovery.(hasSynced)
-	if !ok {
+	require.True(t, ok, "discoverer does not implement hasSynced interface")
-		t.Errorf("discoverer does not implement hasSynced interface")
+	require.True(t, cache.WaitForCacheSync(ctx.Done(), dd.hasSynced), "discoverer failed to sync: %v", dd)
-		return
-	}
-	if !cache.WaitForCacheSync(ctx.Done(), dd.hasSynced) {
-		t.Errorf("discoverer failed to sync: %v", dd)
-		return
-	}
 
 	if d.afterStart != nil {
 		d.afterStart()

discovery/manager_test.go

@@ -694,7 +694,7 @@ func TestTargetUpdatesOrder(t *testing.T) {
 			for x := 0; x < totalUpdatesCount; x++ {
 				select {
 				case <-ctx.Done():
-					t.Fatalf("%d: no update arrived within the timeout limit", x)
+					require.FailNow(t, "%d: no update arrived within the timeout limit", x)
 				case tgs := <-provUpdates:
 					discoveryManager.updateGroup(poolKey{setName: strconv.Itoa(i), provider: tc.title}, tgs)
 					for _, got := range discoveryManager.allGroups() {
@@ -756,10 +756,8 @@ func verifySyncedPresence(t *testing.T, tGroups map[string][]*targetgroup.Group,
 
 func verifyPresence(t *testing.T, tSets map[poolKey]map[string]*targetgroup.Group, poolKey poolKey, label string, present bool) {
 	t.Helper()
-	if _, ok := tSets[poolKey]; !ok {
+	_, ok := tSets[poolKey]
-		t.Fatalf("'%s' should be present in Pool keys: %v", poolKey, tSets)
+	require.True(t, ok, "'%s' should be present in Pool keys: %v", poolKey, tSets)
-		return
-	}
 
 	match := false
 	var mergedTargets string
@@ -776,7 +774,7 @@ func verifyPresence(t *testing.T, tSets map[poolKey]map[string]*targetgroup.Grou
 		if !present {
 			msg = "not"
 		}
-		t.Fatalf("%q should %s be present in Targets labels: %q", label, msg, mergedTargets)
+		require.FailNow(t, "%q should %s be present in Targets labels: %q", label, msg, mergedTargets)
 	}
 }
 
@@ -1088,22 +1086,14 @@ func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
 	syncedTargets = <-discoveryManager.SyncCh()
 	p = pk("static", "prometheus", 1)
 	targetGroups, ok := discoveryManager.targets[p]
-	if !ok {
+	require.True(t, ok, "'%v' should be present in target groups", p)
-		t.Fatalf("'%v' should be present in target groups", p)
-	}
 	group, ok := targetGroups[""]
-	if !ok {
+	require.True(t, ok, "missing '' key in target groups %v", targetGroups)
-		t.Fatalf("missing '' key in target groups %v", targetGroups)
-	}
 
-	if len(group.Targets) != 0 {
+	require.Empty(t, group.Targets, "Invalid number of targets.")
-		t.Fatalf("Invalid number of targets: expected 0, got %d", len(group.Targets))
-	}
 	require.Len(t, syncedTargets, 1)
 	require.Len(t, syncedTargets["prometheus"], 1)
-	if lbls := syncedTargets["prometheus"][0].Labels; lbls != nil {
+	require.Nil(t, syncedTargets["prometheus"][0].Labels)
-		t.Fatalf("Unexpected Group: expected nil Labels, got %v", lbls)
-	}
 }
 
 func TestIdenticalConfigurationsAreCoalesced(t *testing.T) {
@@ -1131,9 +1121,7 @@ func TestIdenticalConfigurationsAreCoalesced(t *testing.T) {
 	syncedTargets := <-discoveryManager.SyncCh()
 	verifyPresence(t, discoveryManager.targets, pk("static", "prometheus", 0), "{__address__=\"foo:9090\"}", true)
 	verifyPresence(t, discoveryManager.targets, pk("static", "prometheus2", 0), "{__address__=\"foo:9090\"}", true)
-	if len(discoveryManager.providers) != 1 {
+	require.Len(t, discoveryManager.providers, 1, "Invalid number of providers.")
-		t.Fatalf("Invalid number of providers: expected 1, got %d", len(discoveryManager.providers))
-	}
 	require.Len(t, syncedTargets, 2)
 	verifySyncedPresence(t, syncedTargets, "prometheus", "{__address__=\"foo:9090\"}", true)
 	require.Len(t, syncedTargets["prometheus"], 1)
@@ -1231,9 +1219,7 @@ func TestGaugeFailedConfigs(t *testing.T) {
 	<-discoveryManager.SyncCh()
 
 	failedCount := client_testutil.ToFloat64(discoveryManager.metrics.FailedConfigs)
-	if failedCount != 3 {
+	require.Equal(t, 3.0, failedCount, "Expected to have 3 failed configs.")
-		t.Fatalf("Expected to have 3 failed configs, got: %v", failedCount)
-	}
 
 	c["prometheus"] = Configs{
 		staticConfig("foo:9090"),
@@ -1242,9 +1228,7 @@ func TestGaugeFailedConfigs(t *testing.T) {
 	<-discoveryManager.SyncCh()
 
 	failedCount = client_testutil.ToFloat64(discoveryManager.metrics.FailedConfigs)
-	if failedCount != 0 {
+	require.Equal(t, 0.0, failedCount, "Expected to get no failed config.")
-		t.Fatalf("Expected to get no failed config, got: %v", failedCount)
-	}
 }
 
 func TestCoordinationWithReceiver(t *testing.T) {
@@ -1388,19 +1372,14 @@ func TestCoordinationWithReceiver(t *testing.T) {
 				time.Sleep(expected.delay)
 				select {
 				case <-ctx.Done():
-					t.Fatalf("step %d: no update received in the expected timeframe", i)
+					require.FailNow(t, "step %d: no update received in the expected timeframe", i)
 				case tgs, ok := <-mgr.SyncCh():
-					if !ok {
+					require.True(t, ok, "step %d: discovery manager channel is closed", i)
-						t.Fatalf("step %d: discovery manager channel is closed", i)
+					require.Equal(t, len(expected.tgs), len(tgs), "step %d: targets mismatch", i)
-					}
+
-					if len(tgs) != len(expected.tgs) {
-						t.Fatalf("step %d: target groups mismatch, got: %d, expected: %d\ngot: %#v\nexpected: %#v",
-							i, len(tgs), len(expected.tgs), tgs, expected.tgs)
-					}
 					for k := range expected.tgs {
-						if _, ok := tgs[k]; !ok {
+						_, ok := tgs[k]
-							t.Fatalf("step %d: target group not found: %s\ngot: %#v", i, k, tgs)
+						require.True(t, ok, "step %d: target group not found: %s", i, k)
-						}
 						assertEqualGroups(t, tgs[k], expected.tgs[k])
 					}
 				}

discovery/marathon/marathon_test.go

@@ -69,23 +69,15 @@ func TestMarathonSDHandleError(t *testing.T) {
 		}
 	)
 	tgs, err := testUpdateServices(client)
-	if !errors.Is(err, errTesting) {
+	require.ErrorIs(t, err, errTesting)
-		t.Fatalf("Expected error: %s", err)
+	require.Empty(t, tgs, "Expected no target groups.")
-	}
-	if len(tgs) != 0 {
-		t.Fatalf("Got group: %s", tgs)
-	}
 }
 
 func TestMarathonSDEmptyList(t *testing.T) {
 	client := func(_ context.Context, _ *http.Client, _ string) (*appList, error) { return &appList{}, nil }
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Empty(t, tgs, "Expected no target groups.")
-	}
-	if len(tgs) > 0 {
-		t.Fatalf("Got group: %v", tgs)
-	}
 }
 
 func marathonTestAppList(labels map[string]string, runningTasks int) *appList {
@@ -119,28 +111,16 @@ func TestMarathonSDSendGroup(t *testing.T) {
 		return marathonTestAppList(marathonValidLabel, 1), nil
 	}
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 target group.")
-	}
-	if len(tgs) != 1 {
-		t.Fatal("Expected 1 target group, got", len(tgs))
-	}
 
 	tg := tgs[0]
+	require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
+	require.Len(t, tg.Targets, 1, "Expected 1 target.")
 
-	if tg.Source != "test-service" {
-		t.Fatalf("Wrong target group name: %s", tg.Source)
-	}
-	if len(tg.Targets) != 1 {
-		t.Fatalf("Wrong number of targets: %v", tg.Targets)
-	}
 	tgt := tg.Targets[0]
-	if tgt[model.AddressLabel] != "mesos-slave1:31000" {
+	require.Equal(t, "mesos-slave1:31000", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "yes", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.")
-	}
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" {
-		t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
-	}
 }
 
 func TestMarathonSDRemoveApp(t *testing.T) {
@@ -153,40 +133,27 @@ func TestMarathonSDRemoveApp(t *testing.T) {
 	defer refreshMetrics.Unregister()
 
 	md, err := NewDiscovery(cfg, nil, metrics)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("%s", err)
-	}
 
 	md.appsClient = func(_ context.Context, _ *http.Client, _ string) (*appList, error) {
 		return marathonTestAppList(marathonValidLabel, 1), nil
 	}
 	tgs, err := md.refresh(context.Background())
-	if err != nil {
+	require.NoError(t, err, "Got error on first update.")
-		t.Fatalf("Got error on first update: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 targetgroup.")
-	}
-	if len(tgs) != 1 {
-		t.Fatal("Expected 1 targetgroup, got", len(tgs))
-	}
 	tg1 := tgs[0]
 
 	md.appsClient = func(_ context.Context, _ *http.Client, _ string) (*appList, error) {
 		return marathonTestAppList(marathonValidLabel, 0), nil
 	}
 	tgs, err = md.refresh(context.Background())
-	if err != nil {
+	require.NoError(t, err, "Got error on second update.")
-		t.Fatalf("Got error on second update: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 targetgroup.")
-	}
+
-	if len(tgs) != 1 {
-		t.Fatal("Expected 1 targetgroup, got", len(tgs))
-	}
 	tg2 := tgs[0]
 
-	if tg2.Source != tg1.Source {
+	require.NotEmpty(t, tg2.Targets, "Got a non-empty target set.")
-		if len(tg2.Targets) > 0 {
+	require.Equal(t, tg1.Source, tg2.Source, "Source is different.")
-			t.Errorf("Got a non-empty target set: %s", tg2.Targets)
-		}
-		t.Fatalf("Source is different: %s != %s", tg1.Source, tg2.Source)
-	}
 }
 
 func marathonTestAppListWithMultiplePorts(labels map[string]string, runningTasks int) *appList {
@@ -221,34 +188,22 @@ func TestMarathonSDSendGroupWithMultiplePort(t *testing.T) {
 		return marathonTestAppListWithMultiplePorts(marathonValidLabel, 1), nil
 	}
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 target group.")
-	}
+
-	if len(tgs) != 1 {
+	tg := tgs[0]
-		t.Fatal("Expected 1 target group, got", len(tgs))
+	require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
-	}
+	require.Len(t, tg.Targets, 2, "Wrong number of targets.")
-	tg := tgs[0]
 
-	if tg.Source != "test-service" {
-		t.Fatalf("Wrong target group name: %s", tg.Source)
-	}
-	if len(tg.Targets) != 2 {
-		t.Fatalf("Wrong number of targets: %v", tg.Targets)
-	}
 	tgt := tg.Targets[0]
-	if tgt[model.AddressLabel] != "mesos-slave1:31000" {
+	require.Equal(t, "mesos-slave1:31000", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "yes", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]),
-	}
+		"Wrong portMappings label from the first port: %s", tgt[model.AddressLabel])
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" {
+
-		t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
-	}
 	tgt = tg.Targets[1]
-	if tgt[model.AddressLabel] != "mesos-slave1:32000" {
+	require.Equal(t, "mesos-slave1:32000", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]),
-	}
+		"Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
-	}
 }
 
 func marathonTestZeroTaskPortAppList(labels map[string]string, runningTasks int) *appList {
@@ -278,20 +233,12 @@ func TestMarathonZeroTaskPorts(t *testing.T) {
 		return marathonTestZeroTaskPortAppList(marathonValidLabel, 1), nil
 	}
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 target group.")
-	}
-	if len(tgs) != 1 {
-		t.Fatal("Expected 1 target group, got", len(tgs))
-	}
-	tg := tgs[0]
 
-	if tg.Source != "test-service-zero-ports" {
+	tg := tgs[0]
-		t.Fatalf("Wrong target group name: %s", tg.Source)
+	require.Equal(t, "test-service-zero-ports", tg.Source, "Wrong target group name.")
-	}
+	require.Empty(t, tg.Targets, "Wrong number of targets.")
-	if len(tg.Targets) != 0 {
-		t.Fatalf("Wrong number of targets: %v", tg.Targets)
-	}
 }
 
 func Test500ErrorHttpResponseWithValidJSONBody(t *testing.T) {
@@ -306,9 +253,7 @@ func Test500ErrorHttpResponseWithValidJSONBody(t *testing.T) {
 	defer ts.Close()
 	// Execute test case and validate behavior.
 	_, err := testUpdateServices(nil)
-	if err == nil {
+	require.Error(t, err, "Expected error for 5xx HTTP response from marathon server.")
-		t.Fatalf("Expected error for 5xx HTTP response from marathon server, got nil")
-	}
 }
 
 func marathonTestAppListWithPortDefinitions(labels map[string]string, runningTasks int) *appList {
@@ -346,40 +291,24 @@ func TestMarathonSDSendGroupWithPortDefinitions(t *testing.T) {
 		return marathonTestAppListWithPortDefinitions(marathonValidLabel, 1), nil
 	}
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 target group.")
-	}
+
-	if len(tgs) != 1 {
+	tg := tgs[0]
-		t.Fatal("Expected 1 target group, got", len(tgs))
+	require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
-	}
+	require.Len(t, tg.Targets, 2, "Wrong number of targets.")
-	tg := tgs[0]
 
-	if tg.Source != "test-service" {
-		t.Fatalf("Wrong target group name: %s", tg.Source)
-	}
-	if len(tg.Targets) != 2 {
-		t.Fatalf("Wrong number of targets: %v", tg.Targets)
-	}
 	tgt := tg.Targets[0]
-	if tgt[model.AddressLabel] != "mesos-slave1:1234" {
+	require.Equal(t, "mesos-slave1:1234", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]),
-	}
+		"Wrong portMappings label from the first port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]),
-		t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
+		"Wrong portDefinitions label from the first port.")
-	}
+
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
-	}
 	tgt = tg.Targets[1]
-	if tgt[model.AddressLabel] != "mesos-slave1:5678" {
+	require.Equal(t, "mesos-slave1:5678", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Empty(t, tgt[model.LabelName(portMappingLabelPrefix+"prometheus")], "Wrong portMappings label from the second port.")
-	}
+	require.Equal(t, "yes", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "yes" {
-		t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
-	}
 }
 
 func marathonTestAppListWithPortDefinitionsRequirePorts(labels map[string]string, runningTasks int) *appList {
@@ -416,40 +345,22 @@ func TestMarathonSDSendGroupWithPortDefinitionsRequirePorts(t *testing.T) {
 		return marathonTestAppListWithPortDefinitionsRequirePorts(marathonValidLabel, 1), nil
 	}
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 target group.")
-	}
+
-	if len(tgs) != 1 {
+	tg := tgs[0]
-		t.Fatal("Expected 1 target group, got", len(tgs))
+	require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
-	}
+	require.Len(t, tg.Targets, 2, "Wrong number of targets.")
-	tg := tgs[0]
 
-	if tg.Source != "test-service" {
-		t.Fatalf("Wrong target group name: %s", tg.Source)
-	}
-	if len(tg.Targets) != 2 {
-		t.Fatalf("Wrong number of targets: %v", tg.Targets)
-	}
 	tgt := tg.Targets[0]
-	if tgt[model.AddressLabel] != "mesos-slave1:31000" {
+	require.Equal(t, "mesos-slave1:31000", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.")
-	}
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
+
-		t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
-	}
 	tgt = tg.Targets[1]
-	if tgt[model.AddressLabel] != "mesos-slave1:32000" {
+	require.Equal(t, "mesos-slave1:32000", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.")
-	}
+	require.Equal(t, "yes", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "yes" {
-		t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
-	}
 }
 
 func marathonTestAppListWithPorts(labels map[string]string, runningTasks int) *appList {
@@ -481,40 +392,22 @@ func TestMarathonSDSendGroupWithPorts(t *testing.T) {
 		return marathonTestAppListWithPorts(marathonValidLabel, 1), nil
 	}
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 target group.")
-	}
+
-	if len(tgs) != 1 {
+	tg := tgs[0]
-		t.Fatal("Expected 1 target group, got", len(tgs))
+	require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
-	}
+	require.Len(t, tg.Targets, 2, "Wrong number of targets.")
-	tg := tgs[0]
 
-	if tg.Source != "test-service" {
-		t.Fatalf("Wrong target group name: %s", tg.Source)
-	}
-	if len(tg.Targets) != 2 {
-		t.Fatalf("Wrong number of targets: %v", tg.Targets)
-	}
 	tgt := tg.Targets[0]
-	if tgt[model.AddressLabel] != "mesos-slave1:31000" {
+	require.Equal(t, "mesos-slave1:31000", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.")
-	}
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
+
-		t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
-	}
 	tgt = tg.Targets[1]
-	if tgt[model.AddressLabel] != "mesos-slave1:32000" {
+	require.Equal(t, "mesos-slave1:32000", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.")
-	}
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
-	}
 }
 
 func marathonTestAppListWithContainerPortMappings(labels map[string]string, runningTasks int) *appList {
@@ -555,40 +448,22 @@ func TestMarathonSDSendGroupWithContainerPortMappings(t *testing.T) {
 		return marathonTestAppListWithContainerPortMappings(marathonValidLabel, 1), nil
 	}
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 target group.")
-	}
+
-	if len(tgs) != 1 {
+	tg := tgs[0]
-		t.Fatal("Expected 1 target group, got", len(tgs))
+	require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
-	}
+	require.Len(t, tg.Targets, 2, "Wrong number of targets.")
-	tg := tgs[0]
 
-	if tg.Source != "test-service" {
-		t.Fatalf("Wrong target group name: %s", tg.Source)
-	}
-	if len(tg.Targets) != 2 {
-		t.Fatalf("Wrong number of targets: %v", tg.Targets)
-	}
 	tgt := tg.Targets[0]
-	if tgt[model.AddressLabel] != "mesos-slave1:12345" {
+	require.Equal(t, "mesos-slave1:12345", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "yes", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.")
-	}
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" {
+
-		t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
-	}
 	tgt = tg.Targets[1]
-	if tgt[model.AddressLabel] != "mesos-slave1:32000" {
+	require.Equal(t, "mesos-slave1:32000", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.")
-	}
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
-	}
 }
 
 func marathonTestAppListWithDockerContainerPortMappings(labels map[string]string, runningTasks int) *appList {
@@ -629,40 +504,22 @@ func TestMarathonSDSendGroupWithDockerContainerPortMappings(t *testing.T) {
 		return marathonTestAppListWithDockerContainerPortMappings(marathonValidLabel, 1), nil
 	}
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 target group.")
-	}
+
-	if len(tgs) != 1 {
+	tg := tgs[0]
-		t.Fatal("Expected 1 target group, got", len(tgs))
+	require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
-	}
+	require.Len(t, tg.Targets, 2, "Wrong number of targets.")
-	tg := tgs[0]
 
-	if tg.Source != "test-service" {
-		t.Fatalf("Wrong target group name: %s", tg.Source)
-	}
-	if len(tg.Targets) != 2 {
-		t.Fatalf("Wrong number of targets: %v", tg.Targets)
-	}
 	tgt := tg.Targets[0]
-	if tgt[model.AddressLabel] != "mesos-slave1:31000" {
+	require.Equal(t, "mesos-slave1:31000", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "yes", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.")
-	}
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" {
+
-		t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
-	}
 	tgt = tg.Targets[1]
-	if tgt[model.AddressLabel] != "mesos-slave1:12345" {
+	require.Equal(t, "mesos-slave1:12345", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.")
-	}
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
-	}
 }
 
 func marathonTestAppListWithContainerNetworkAndPortMappings(labels map[string]string, runningTasks int) *appList {
@@ -707,38 +564,20 @@ func TestMarathonSDSendGroupWithContainerNetworkAndPortMapping(t *testing.T) {
 		return marathonTestAppListWithContainerNetworkAndPortMappings(marathonValidLabel, 1), nil
 	}
 	tgs, err := testUpdateServices(client)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatalf("Got error: %s", err)
+	require.Len(t, tgs, 1, "Expected 1 target group.")
-	}
+
-	if len(tgs) != 1 {
+	tg := tgs[0]
-		t.Fatal("Expected 1 target group, got", len(tgs))
+	require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
-	}
+	require.Len(t, tg.Targets, 2, "Wrong number of targets.")
-	tg := tgs[0]
 
-	if tg.Source != "test-service" {
-		t.Fatalf("Wrong target group name: %s", tg.Source)
-	}
-	if len(tg.Targets) != 2 {
-		t.Fatalf("Wrong number of targets: %v", tg.Targets)
-	}
 	tgt := tg.Targets[0]
-	if tgt[model.AddressLabel] != "1.2.3.4:8080" {
+	require.Equal(t, "1.2.3.4:8080", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "yes", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.")
-	}
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" {
+
-		t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
-	}
 	tgt = tg.Targets[1]
-	if tgt[model.AddressLabel] != "1.2.3.4:1234" {
+	require.Equal(t, "1.2.3.4:1234", string(tgt[model.AddressLabel]), "Wrong target address.")
-		t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+	require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.")
-	}
+	require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.")
-	if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
-	}
-	if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-		t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
-	}
 }

discovery/openstack/mock_test.go

@@ -18,6 +18,8 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"testing"
+
+	"github.com/stretchr/testify/require"
 )
 
 // SDMock is the interface for the OpenStack mock.
@@ -49,15 +51,13 @@ func (m *SDMock) Setup() {
 const tokenID = "cbc36478b0bd8e67e89469c7749d4127"
 
 func testMethod(t *testing.T, r *http.Request, expected string) {
-	if expected != r.Method {
+	require.Equal(t, expected, r.Method, "Unexpected request method.")
-		t.Errorf("Request method = %v, expected %v", r.Method, expected)
-	}
 }
 
 func testHeader(t *testing.T, r *http.Request, header, expected string) {
-	if actual := r.Header.Get(header); expected != actual {
+	t.Helper()
-		t.Errorf("Header %s = %s, expected %s", header, actual, expected)
+	actual := r.Header.Get(header)
-	}
+	require.Equal(t, expected, actual, "Unexpected value for request header %s.", header)
 }
 
 // HandleVersionsSuccessfully mocks version call.

discovery/refresh/refresh_test.go

@@ -97,7 +97,7 @@ func TestRefresh(t *testing.T) {
 	defer tick.Stop()
 	select {
 	case <-ch:
-		t.Fatal("Unexpected target group")
+		require.FailNow(t, "Unexpected target group")
 	case <-tick.C:
 	}
 }

discovery/zookeeper/zookeeper_test.go

@@ -18,6 +18,7 @@ import (
 	"time"
 
 	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
 )
 
@@ -31,7 +32,5 @@ func TestNewDiscoveryError(t *testing.T) {
 		time.Second, []string{"/"},
 		nil,
 		func(data []byte, path string) (model.LabelSet, error) { return nil, nil })
-	if err == nil {
+	require.Error(t, err)
-		t.Fatalf("expected error, got nil")
-	}
 }

docs/feature_flags.md

@@ -34,7 +34,7 @@ Activating the remote write receiver via a feature flag is deprecated. Use `--we
 
 [OpenMetrics](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#exemplars) introduces the ability for scrape targets to add exemplars to certain metrics. Exemplars are references to data outside of the MetricSet. A common use case are IDs of program traces.
 
-Exemplar storage is implemented as a fixed size circular buffer that stores exemplars in memory for all series. Enabling this feature will enable the storage of exemplars scraped by Prometheus. The config file block [storage](configuration/configuration.md#configuration-file)/[exemplars](configuration/configuration.md#exemplars) can be used to control the size of circular buffer by # of exemplars. An exemplar with just a `traceID=<jaeger-trace-id>` uses roughly 100 bytes of memory via the in-memory exemplar storage. If the exemplar storage is enabled, we will also append the exemplars to WAL for local persistence (for WAL duration).
+Exemplar storage is implemented as a fixed size circular buffer that stores exemplars in memory for all series. Enabling this feature will enable the storage of exemplars scraped by Prometheus. The config file block [storage](configuration/configuration.md#configuration-file)/[exemplars](configuration/configuration.md#exemplars) can be used to control the size of circular buffer by # of exemplars. An exemplar with just a `trace_id=<jaeger-trace-id>` uses roughly 100 bytes of memory via the in-memory exemplar storage. If the exemplar storage is enabled, we will also append the exemplars to WAL for local persistence (for WAL duration).
 
 ## Memory snapshot on shutdown
 

docs/querying/api.md

@@ -404,7 +404,7 @@ $ curl -g 'http://localhost:9090/api/v1/query_exemplars?query=test_exemplar_metr
             "exemplars": [
                 {
                     "labels": {
-                        "traceID": "EpTxMJ40fUus7aGY"
+                        "trace_id": "EpTxMJ40fUus7aGY"
                     },
                     "value": "6",
                     "timestamp": 1600096945.479
@@ -421,14 +421,14 @@ $ curl -g 'http://localhost:9090/api/v1/query_exemplars?query=test_exemplar_metr
             "exemplars": [
                 {
                     "labels": {
-                        "traceID": "Olp9XHlq763ccsfa"
+                        "trace_id": "Olp9XHlq763ccsfa"
                     },
                     "value": "19",
                     "timestamp": 1600096955.479
                 },
                 {
                     "labels": {
-                        "traceID": "hCtjygkIHwAN9vs4"
+                        "trace_id": "hCtjygkIHwAN9vs4"
                     },
                     "value": "20",
                     "timestamp": 1600096965.489

docs/querying/basics.md

@@ -14,7 +14,7 @@ systems via the [HTTP API](api.md).
 
 ## Examples
 
-This document is meant as a reference. For learning, it might be easier to
+This document is a Prometheus basic language reference. For learning, it may be easier to
 start with a couple of [examples](examples.md).
 
 ## Expression language data types
@@ -28,9 +28,9 @@ evaluate to one of four types:
 * **String** - a simple string value; currently unused
 
 Depending on the use-case (e.g. when graphing vs. displaying the output of an
-expression), only some of these types are legal as the result from a
+expression), only some of these types are legal as the result of a
 user-specified expression. For example, an expression that returns an instant
-vector is the only type that can be directly graphed.
+vector is the only type which can be graphed.
 
 _Notes about the experimental native histograms:_
 
@@ -46,16 +46,15 @@ _Notes about the experimental native histograms:_
 
 ### String literals
 
-Strings may be specified as literals in single quotes, double quotes or
+String literals are designated by single quotes, double quotes or backticks.
-backticks.
 
 PromQL follows the same [escaping rules as
-Go](https://golang.org/ref/spec#String_literals). In single or double quotes a
+Go](https://golang.org/ref/spec#String_literals). For string literals in single or double quotes, a
 backslash begins an escape sequence, which may be followed by `a`, `b`, `f`,
-`n`, `r`, `t`, `v` or `\`. Specific characters can be provided using octal
+`n`, `r`, `t`, `v` or `\`.  Specific characters can be provided using octal
-(`\nnn`) or hexadecimal (`\xnn`, `\unnnn` and `\Unnnnnnnn`).
+(`\nnn`) or hexadecimal (`\xnn`, `\unnnn` and `\Unnnnnnnn`) notations.
 
-No escaping is processed inside backticks. Unlike Go, Prometheus does not discard newlines inside backticks.
+Conversely, escape characters are not parsed in string literals designated by backticks. It is important to note that, unlike Go, Prometheus does not discard newlines inside backticks.
 
 Example:
 
@@ -83,13 +82,17 @@ Examples:
     -Inf
     NaN
 
-## Time series Selectors
+## Time series selectors
+
+Time series selectors are responsible for selecting the times series and raw or inferred sample timestamps and values.
+
+Time series *selectors* are not to be confused with higher level concept of instant and range *queries* that can execute the time series *selectors*. A higher level instant query would evaluate the given selector at one point in time, however the range query would evaluate the selector at multiple different times in between a minimum and maximum timestamp at regular steps.
 
 ### Instant vector selectors
 
 Instant vector selectors allow the selection of a set of time series and a
-single sample value for each at a given timestamp (instant): in the simplest
+single sample value for each at a given timestamp (point in time).  In the simplest
-form, only a metric name is specified. This results in an instant vector
+form, only a metric name is specified, which results in an instant vector
 containing elements for all time series that have this metric name.
 
 This example selects all time series that have the `http_requests_total` metric
@@ -97,7 +100,7 @@ name:
 
     http_requests_total
 
-It is possible to filter these time series further by appending a comma separated list of label
+It is possible to filter these time series further by appending a comma-separated list of label
 matchers in curly braces (`{}`).
 
 This example selects only those time series with the `http_requests_total`
@@ -124,6 +127,33 @@ For example, this selects all `http_requests_total` time series for `staging`,
 Label matchers that match empty label values also select all time series that
 do not have the specific label set at all. It is possible to have multiple matchers for the same label name.
 
+For example, given the dataset:
+
+    http_requests_total
+    http_requests_total{replica="rep-a"}
+    http_requests_total{replica="rep-b"}
+    http_requests_total{environment="development"}
+
+The query `http_requests_total{environment=""}` would match and return:
+
+    http_requests_total
+    http_requests_total{replica="rep-a"}
+    http_requests_total{replica="rep-b"}
+
+and would exclude:
+
+    http_requests_total{environment="development"}
+
+Multiple matchers can be used for the same label name; they all must pass for a result to be returned. 
+
+The query:
+
+    http_requests_total{replica!="rep-a",replica=~"rep.*"}
+
+Would then match:
+
+    http_requests_total{replica="rep-b"}
+
 Vector selectors must either specify a name or at least one label matcher
 that does not match the empty string. The following expression is illegal:
 
@@ -178,11 +208,13 @@ following units:
 * `s` - seconds
 * `m` - minutes
 * `h` - hours
-* `d` - days - assuming a day has always 24h
+* `d` - days - assuming a day always has 24h
-* `w` - weeks - assuming a week has always 7d
+* `w` - weeks - assuming a week always has 7d
-* `y` - years - assuming a year has always 365d
+* `y` - years - assuming a year always has 365d<sup>1</sup>
 
-Time durations can be combined, by concatenation. Units must be ordered from the
+<sup>1</sup> For days in a year, the leap day is ignored, and conversely, for a minute, a leap second is ignored.
+
+Time durations can be combined by concatenation. Units must be ordered from the
 longest to the shortest. A given unit must only appear once in a time duration.
 
 Here are some examples of valid time durations:
@@ -217,8 +249,7 @@ that `http_requests_total` had a week ago:
 
     rate(http_requests_total[5m] offset 1w)
 
-For comparisons with temporal shifts forward in time, a negative offset
+When querying for samples in the past, a negative offset will enable temporal comparisons forward in time:
-can be specified:
 
     rate(http_requests_total[5m] offset -1w)
 
@@ -249,11 +280,11 @@ The same works for range vectors. This returns the 5-minute rate that
 
     rate(http_requests_total[5m] @ 1609746000)
 
-The `@` modifier supports all representation of float literals described
+The `@` modifier supports all representations of numeric literals described above.
-above within the limits of `int64`. It can also be used along
+It works with the `offset` modifier where the offset is applied relative to the `@`
-with the `offset` modifier where the offset is applied relative to the `@`
+modifier time.  The results are the same irrespective of the order of the modifiers.
-modifier time irrespective of which modifier is written first.
+
-These 2 queries will produce the same result.
+For example, these two queries will produce the same result:
 
     # offset after @
     http_requests_total @ 1609746000 offset 5m
@@ -299,33 +330,35 @@ PromQL supports line comments that start with `#`. Example:
 
 ### Staleness
 
-When queries are run, timestamps at which to sample data are selected
+The timestamps at which to sample data, during a query, are selected
 independently of the actual present time series data. This is mainly to support
 cases like aggregation (`sum`, `avg`, and so on), where multiple aggregated
-time series do not exactly align in time. Because of their independence,
+time series do not precisely align in time. Because of their independence,
 Prometheus needs to assign a value at those timestamps for each relevant time
-series. It does so by simply taking the newest sample before this timestamp.
+series. It does so by taking the newest sample before this timestamp within the lookback period.
+The lookback period is 5 minutes by default.
 
 If a target scrape or rule evaluation no longer returns a sample for a time
-series that was previously present, that time series will be marked as stale.
+series that was previously present, this time series will be marked as stale.
-If a target is removed, its previously returned time series will be marked as
+If a target is removed, the previously retrieved time series will be marked as
-stale soon afterwards.
+stale soon after removal.
 
 If a query is evaluated at a sampling timestamp after a time series is marked
-stale, then no value is returned for that time series. If new samples are
+as stale, then no value is returned for that time series. If new samples are
-subsequently ingested for that time series, they will be returned as normal.
+subsequently ingested for that time series, they will be returned as expected.
 
-If no sample is found (by default) 5 minutes before a sampling timestamp,
+A time series will go stale when it is no longer exported, or the target no
-no value is returned for that time series at this point in time. This
+longer exists. Such time series will disappear from graphs 
-effectively means that time series "disappear" from graphs at times where their
+at the times of their latest collected sample, and they will not be returned
-latest collected sample is older than 5 minutes or after they are marked stale.
+in queries after they are marked stale.
 
-Staleness will not be marked for time series that have timestamps included in
+Some exporters, which put their own timestamps on samples, get a different behaviour: 
-their scrapes. Only the 5 minute threshold will be applied in that case.
+series that stop being exported take the last value for (by default) 5 minutes before
+disappearing. The `track_timestamps_staleness` setting can change this.
 
 ### Avoiding slow queries and overloads
 
-If a query needs to operate on a very large amount of data, graphing it might
+If a query needs to operate on a substantial amount of data, graphing it might
 time out or overload the server or browser. Thus, when constructing queries
 over unknown data, always start building the query in the tabular view of
 Prometheus's expression browser until the result set seems reasonable
@@ -336,7 +369,7 @@ rule](../configuration/recording_rules.md#recording-rules).
 
 This is especially relevant for Prometheus's query language, where a bare
 metric name selector like `api_http_requests_total` could expand to thousands
-of time series with different labels. Also keep in mind that expressions which
+of time series with different labels. Also, keep in mind that expressions that
 aggregate over many time series will generate load on the server even if the
 output is only a small number of time series. This is similar to how it would
 be slow to sum all values of a column in a relational database, even if the

docs/querying/functions.md

@@ -175,6 +175,27 @@ Special cases are:
 `floor(v instant-vector)` rounds the sample values of all elements in `v` down
 to the nearest integer.
 
+## `histogram_avg()`
+
+_This function only acts on native histograms, which are an experimental
+feature. The behavior of this function may change in future versions of
+Prometheus, including its removal from PromQL._
+
+`histogram_avg(v instant-vector)` returns the arithmetic average of observed values stored in
+a native histogram. Samples that are not native histograms are ignored and do
+not show up in the returned vector.
+
+Use `histogram_avg` as demonstrated below to compute the average request duration
+over a 5-minute window from a native histogram:
+
+    histogram_avg(rate(http_request_duration_seconds[5m]))
+
+Which is equivalent to the following query:
+
+      histogram_sum(rate(http_request_duration_seconds[5m]))
+    /
+      histogram_count(rate(http_request_duration_seconds[5m]))
+
 ## `histogram_count()` and `histogram_sum()`
 
 _Both functions only act on native histograms, which are an experimental
@@ -193,13 +214,6 @@ Use `histogram_count` in the following way to calculate a rate of observations
 
     histogram_count(rate(http_request_duration_seconds[10m]))
 
-The additional use of `histogram_sum` enables the calculation of the average of
-observed values (in this case corresponding to “average request duration”):
-
-      histogram_sum(rate(http_request_duration_seconds[10m]))
-	/
-      histogram_count(rate(http_request_duration_seconds[10m]))
-
 ## `histogram_fraction()`
 
 _This function only acts on native histograms, which are an experimental

documentation/prometheus-mixin/alerts.libsonnet

@@ -122,7 +122,7 @@
             alert: 'PrometheusNotIngestingSamples',
             expr: |||
               (
-                rate(prometheus_tsdb_head_samples_appended_total{%(prometheusSelector)s}[5m]) <= 0
+                sum without(type) (rate(prometheus_tsdb_head_samples_appended_total{%(prometheusSelector)s}[5m])) <= 0
               and
                 (
                   sum without(scrape_job) (prometheus_target_metadata_cache_entries{%(prometheusSelector)s}) > 0

go.mod

@@ -28,6 +28,7 @@ require (
 	github.com/go-zookeeper/zk v1.0.3
 	github.com/gogo/protobuf v1.3.2
 	github.com/golang/snappy v0.0.4
+	github.com/google/go-cmp v0.6.0
 	github.com/google/pprof v0.0.0-20240117000934-35fc243c5815
 	github.com/google/uuid v1.5.0
 	github.com/gophercloud/gophercloud v1.8.0
@@ -51,7 +52,7 @@ require (
 	github.com/prometheus/alertmanager v0.26.0
 	github.com/prometheus/client_golang v1.18.0
 	github.com/prometheus/client_model v0.5.0
-	github.com/prometheus/common v0.46.0
+	github.com/prometheus/common v0.47.0
 	github.com/prometheus/common/assets v0.2.0
 	github.com/prometheus/common/sigv4 v0.1.0
 	github.com/prometheus/exporter-toolkit v0.11.0
@@ -135,7 +136,6 @@ require (
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
-	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect

go.sum

@@ -670,8 +670,8 @@ github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8b
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
 github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
-github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y=
+github.com/prometheus/common v0.47.0 h1:p5Cz0FNHo7SnWOmWmoRozVcjEp0bIVU8cV7OShpjL1k=
-github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ=
+github.com/prometheus/common v0.47.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
 github.com/prometheus/common/assets v0.2.0 h1:0P5OrzoHrYBOSM1OigWL3mY8ZvV2N4zIE/5AahrSrfM=
 github.com/prometheus/common/assets v0.2.0/go.mod h1:D17UVUE12bHbim7HzwUvtqm6gwBEaDQ0F+hIGbFbccI=
 github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4=

model/labels/labels_stringlabels.go

@@ -450,14 +450,12 @@ func (ls Labels) DropMetricName() Labels {
 	return ls
 }
 
-// InternStrings calls intern on every string value inside ls, replacing them with what it returns.
+// InternStrings is a no-op because it would only save when the whole set of labels is identical.
 func (ls *Labels) InternStrings(intern func(string) string) {
-	ls.data = intern(ls.data)
 }
 
-// ReleaseStrings calls release on every string value inside ls.
+// ReleaseStrings is a no-op for the same reason as InternStrings.
 func (ls Labels) ReleaseStrings(release func(string)) {
-	release(ls.data)
 }
 
 // Labels returns the labels from the builder.

model/labels/labels_test.go

@@ -708,7 +708,8 @@ func TestScratchBuilder(t *testing.T) {
 
 func TestLabels_Hash(t *testing.T) {
 	lbls := FromStrings("foo", "bar", "baz", "qux")
-	require.Equal(t, lbls.Hash(), lbls.Hash())
+	hash1, hash2 := lbls.Hash(), lbls.Hash()
+	require.Equal(t, hash1, hash2)
 	require.NotEqual(t, lbls.Hash(), FromStrings("foo", "bar").Hash(), "different labels match.")
 }
 

model/relabel/relabel_test.go

@@ -22,6 +22,7 @@ import (
 	"gopkg.in/yaml.v2"
 
 	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/util/testutil"
 )
 
 func TestRelabel(t *testing.T) {
@@ -591,7 +592,7 @@ func TestRelabel(t *testing.T) {
 		res, keep := Process(test.input, test.relabel...)
 		require.Equal(t, !test.drop, keep)
 		if keep {
-			require.Equal(t, test.output, res)
+			testutil.RequireEqual(t, test.output, res)
 		}
 	}
 }

model/textparse/openmetricslex.l

@@ -50,12 +50,15 @@ S     [ ]
 <sComment>TYPE{S}                     l.state = sMeta1; return tType
 <sComment>UNIT{S}                     l.state = sMeta1; return tUnit
 <sComment>"EOF"\n?                    l.state = sInit; return tEOFWord
+<sMeta1>\"(\\.|[^\\"])*\"             l.state = sMeta2; return tMName
 <sMeta1>{M}({M}|{D})*                 l.state = sMeta2; return tMName
 <sMeta2>{S}{C}*\n                     l.state = sInit; return tText
 
 {M}({M}|{D})*                         l.state = sValue; return tMName
 <sValue>\{                            l.state = sLabels; return tBraceOpen
+\{                                    l.state = sLabels; return tBraceOpen
 <sLabels>{L}({L}|{D})*                return tLName
+<sLabels>\"(\\.|[^\\"])*\"            l.state = sLabels; return tQString
 <sLabels>\}                           l.state = sValue; return tBraceClose
 <sLabels>=                            l.state = sLValue; return tEqual
 <sLabels>,                            return tComma

model/textparse/openmetricsparse.go

@@ -81,6 +81,12 @@ type OpenMetricsParser struct {
 	ts      int64
 	hasTS   bool
 	start   int
+	// offsets is a list of offsets into series that describe the positions
+	// of the metric name and label names and values for this series.
+	// p.offsets[0] is the start character of the metric name.
+	// p.offsets[1] is the end of the metric name.
+	// Subsequently, p.offsets is a pair of pair of offsets for the positions
+	// of the label name and value start and end characters.
 	offsets []int
 
 	eOffsets      []int
@@ -153,20 +159,18 @@ func (p *OpenMetricsParser) Metric(l *labels.Labels) string {
 	s := string(p.series)
 
 	p.builder.Reset()
-	p.builder.Add(labels.MetricName, s[:p.offsets[0]-p.start])
+	metricName := unreplace(s[p.offsets[0]-p.start : p.offsets[1]-p.start])
+	p.builder.Add(labels.MetricName, metricName)
 
-	for i := 1; i < len(p.offsets); i += 4 {
+	for i := 2; i < len(p.offsets); i += 4 {
 		a := p.offsets[i] - p.start
 		b := p.offsets[i+1] - p.start
+		label := unreplace(s[a:b])
 		c := p.offsets[i+2] - p.start
 		d := p.offsets[i+3] - p.start
+		value := unreplace(s[c:d])
 
-		value := s[c:d]
+		p.builder.Add(label, value)
-		// Replacer causes allocations. Replace only when necessary.
-		if strings.IndexByte(s[c:d], byte('\\')) >= 0 {
-			value = lvalReplacer.Replace(value)
-		}
-		p.builder.Add(s[a:b], value)
 	}
 
 	p.builder.Sort()
@@ -255,7 +259,13 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
 	case tHelp, tType, tUnit:
 		switch t2 := p.nextToken(); t2 {
 		case tMName:
-			p.offsets = append(p.offsets, p.l.start, p.l.i)
+			mStart := p.l.start
+			mEnd := p.l.i
+			if p.l.b[mStart] == '"' && p.l.b[mEnd-1] == '"' {
+				mStart++
+				mEnd--
+			}
+			p.offsets = append(p.offsets, mStart, mEnd)
 		default:
 			return EntryInvalid, p.parseError("expected metric name after "+t.String(), t2)
 		}
@@ -312,58 +322,33 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
 			return EntryUnit, nil
 		}
 
+	case tBraceOpen:
+		// We found a brace, so make room for the eventual metric name. If these
+		// values aren't updated, then the metric name was not set inside the
+		// braces and we can return an error.
+		if len(p.offsets) == 0 {
+			p.offsets = []int{-1, -1}
+		}
+		if p.offsets, err = p.parseLVals(p.offsets, false); err != nil {
+			return EntryInvalid, err
+		}
+
+		p.series = p.l.b[p.start:p.l.i]
+		return p.parseMetricSuffix(p.nextToken())
 	case tMName:
-		p.offsets = append(p.offsets, p.l.i)
+		p.offsets = append(p.offsets, p.start, p.l.i)
 		p.series = p.l.b[p.start:p.l.i]
 
 		t2 := p.nextToken()
 		if t2 == tBraceOpen {
-			p.offsets, err = p.parseLVals(p.offsets)
+			p.offsets, err = p.parseLVals(p.offsets, false)
 			if err != nil {
 				return EntryInvalid, err
 			}
 			p.series = p.l.b[p.start:p.l.i]
 			t2 = p.nextToken()
 		}
-		p.val, err = p.getFloatValue(t2, "metric")
+		return p.parseMetricSuffix(t2)
-		if err != nil {
-			return EntryInvalid, err
-		}
-
-		p.hasTS = false
-		switch t2 := p.nextToken(); t2 {
-		case tEOF:
-			return EntryInvalid, errors.New("data does not end with # EOF")
-		case tLinebreak:
-			break
-		case tComment:
-			if err := p.parseComment(); err != nil {
-				return EntryInvalid, err
-			}
-		case tTimestamp:
-			p.hasTS = true
-			var ts float64
-			// A float is enough to hold what we need for millisecond resolution.
-			if ts, err = parseFloat(yoloString(p.l.buf()[1:])); err != nil {
-				return EntryInvalid, fmt.Errorf("%w while parsing: %q", err, p.l.b[p.start:p.l.i])
-			}
-			if math.IsNaN(ts) || math.IsInf(ts, 0) {
-				return EntryInvalid, fmt.Errorf("invalid timestamp %f", ts)
-			}
-			p.ts = int64(ts * 1000)
-			switch t3 := p.nextToken(); t3 {
-			case tLinebreak:
-			case tComment:
-				if err := p.parseComment(); err != nil {
-					return EntryInvalid, err
-				}
-			default:
-				return EntryInvalid, p.parseError("expected next entry after timestamp", t3)
-			}
-		default:
-			return EntryInvalid, p.parseError("expected timestamp or # symbol", t2)
-		}
-		return EntrySeries, nil
 
 	default:
 		err = p.parseError("expected a valid start token", t)
@@ -374,7 +359,7 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
 func (p *OpenMetricsParser) parseComment() error {
 	var err error
 	// Parse the labels.
-	p.eOffsets, err = p.parseLVals(p.eOffsets)
+	p.eOffsets, err = p.parseLVals(p.eOffsets, true)
 	if err != nil {
 		return err
 	}
@@ -415,38 +400,47 @@ func (p *OpenMetricsParser) parseComment() error {
 	return nil
 }
 
-func (p *OpenMetricsParser) parseLVals(offsets []int) ([]int, error) {
+func (p *OpenMetricsParser) parseLVals(offsets []int, isExemplar bool) ([]int, error) {
-	first := true
+	t := p.nextToken()
 	for {
-		t := p.nextToken()
+		curTStart := p.l.start
+		curTI := p.l.i
 		switch t {
 		case tBraceClose:
 			return offsets, nil
-		case tComma:
+		case tLName:
-			if first {
+		case tQString:
-				return nil, p.parseError("expected label name or left brace", t)
+		default:
-			}
+			return nil, p.parseError("expected label name", t)
-			t = p.nextToken()
+		}
-			if t != tLName {
+
+		t = p.nextToken()
+		// A quoted string followed by a comma or brace is a metric name. Set the
+		// offsets and continue processing. If this is an exemplar, this format
+		// is not allowed.
+		if t == tComma || t == tBraceClose {
+			if isExemplar {
 				return nil, p.parseError("expected label name", t)
 			}
-		case tLName:
+			if offsets[0] != -1 || offsets[1] != -1 {
-			if !first {
+				return nil, fmt.Errorf("metric name already set while parsing: %q", p.l.b[p.start:p.l.i])
-				return nil, p.parseError("expected comma", t)
 			}
-		default:
+			offsets[0] = curTStart + 1
-			if first {
+			offsets[1] = curTI - 1
-				return nil, p.parseError("expected label name or left brace", t)
+			if t == tBraceClose {
+				return offsets, nil
 			}
-			return nil, p.parseError("expected comma or left brace", t)
+			t = p.nextToken()
-
+			continue
 		}
-		first = false
+		// We have a label name, and it might be quoted.
-		// t is now a label name.
+		if p.l.b[curTStart] == '"' {
+			curTStart++
+			curTI--
+		}
+		offsets = append(offsets, curTStart, curTI)
 
-		offsets = append(offsets, p.l.start, p.l.i)
+		if t != tEqual {
-
-		if t := p.nextToken(); t != tEqual {
 			return nil, p.parseError("expected equal", t)
 		}
 		if t := p.nextToken(); t != tLValue {
@@ -459,9 +453,64 @@ func (p *OpenMetricsParser) parseLVals(offsets []int) ([]int, error) {
 		// The openMetricsLexer ensures the value string is quoted. Strip first
 		// and last character.
 		offsets = append(offsets, p.l.start+1, p.l.i-1)
+
+		// Free trailing commas are allowed.
+		t = p.nextToken()
+		if t == tComma {
+			t = p.nextToken()
+		} else if t != tBraceClose {
+			return nil, p.parseError("expected comma or brace close", t)
+		}
 	}
 }
 
+// parseMetricSuffix parses the end of the line after the metric name and
+// labels. It starts parsing with the provided token.
+func (p *OpenMetricsParser) parseMetricSuffix(t token) (Entry, error) {
+	if p.offsets[0] == -1 {
+		return EntryInvalid, fmt.Errorf("metric name not set while parsing: %q", p.l.b[p.start:p.l.i])
+	}
+
+	var err error
+	p.val, err = p.getFloatValue(t, "metric")
+	if err != nil {
+		return EntryInvalid, err
+	}
+
+	p.hasTS = false
+	switch t2 := p.nextToken(); t2 {
+	case tEOF:
+		return EntryInvalid, errors.New("data does not end with # EOF")
+	case tLinebreak:
+		break
+	case tComment:
+		if err := p.parseComment(); err != nil {
+			return EntryInvalid, err
+		}
+	case tTimestamp:
+		p.hasTS = true
+		var ts float64
+		// A float is enough to hold what we need for millisecond resolution.
+		if ts, err = parseFloat(yoloString(p.l.buf()[1:])); err != nil {
+			return EntryInvalid, fmt.Errorf("%w while parsing: %q", err, p.l.b[p.start:p.l.i])
+		}
+		if math.IsNaN(ts) || math.IsInf(ts, 0) {
+			return EntryInvalid, fmt.Errorf("invalid timestamp %f", ts)
+		}
+		p.ts = int64(ts * 1000)
+		switch t3 := p.nextToken(); t3 {
+		case tLinebreak:
+		case tComment:
+			if err := p.parseComment(); err != nil {
+				return EntryInvalid, err
+			}
+		default:
+			return EntryInvalid, p.parseError("expected next entry after timestamp", t3)
+		}
+	}
+	return EntrySeries, nil
+}
+
 func (p *OpenMetricsParser) getFloatValue(t token, after string) (float64, error) {
 	if t != tValue {
 		return 0, p.parseError(fmt.Sprintf("expected value after %v", after), t)

model/textparse/openmetricsparse_test.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/prometheus/prometheus/model/exemplar"
 	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/util/testutil"
 )
 
 func TestOpenMetricsParse(t *testing.T) {
@@ -251,6 +252,137 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5`
 
 	var res labels.Labels
 
+	for {
+		et, err := p.Next()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		require.NoError(t, err)
+
+		switch et {
+		case EntrySeries:
+			m, ts, v := p.Series()
+
+			var e exemplar.Exemplar
+			p.Metric(&res)
+			found := p.Exemplar(&e)
+			require.Equal(t, exp[i].m, string(m))
+			require.Equal(t, exp[i].t, ts)
+			require.Equal(t, exp[i].v, v)
+			testutil.RequireEqual(t, exp[i].lset, res)
+			if exp[i].e == nil {
+				require.False(t, found)
+			} else {
+				require.True(t, found)
+				testutil.RequireEqual(t, *exp[i].e, e)
+			}
+
+		case EntryType:
+			m, typ := p.Type()
+			require.Equal(t, exp[i].m, string(m))
+			require.Equal(t, exp[i].typ, typ)
+
+		case EntryHelp:
+			m, h := p.Help()
+			require.Equal(t, exp[i].m, string(m))
+			require.Equal(t, exp[i].help, string(h))
+
+		case EntryUnit:
+			m, u := p.Unit()
+			require.Equal(t, exp[i].m, string(m))
+			require.Equal(t, exp[i].unit, string(u))
+
+		case EntryComment:
+			require.Equal(t, exp[i].comment, string(p.Comment()))
+		}
+
+		i++
+	}
+	require.Len(t, exp, i)
+}
+
+func TestUTF8OpenMetricsParse(t *testing.T) {
+	oldValidationScheme := model.NameValidationScheme
+	model.NameValidationScheme = model.UTF8Validation
+	defer func() {
+		model.NameValidationScheme = oldValidationScheme
+	}()
+
+	input := `# HELP "go.gc_duration_seconds" A summary of the GC invocation durations.
+# TYPE "go.gc_duration_seconds" summary
+# UNIT "go.gc_duration_seconds" seconds
+{"go.gc_duration_seconds",quantile="0"} 4.9351e-05
+{"go.gc_duration_seconds",quantile="0.25"} 7.424100000000001e-05
+{"go.gc_duration_seconds",quantile="0.5",a="b"} 8.3835e-05
+{"http.status",q="0.9",a="b"} 8.3835e-05
+{"http.status",q="0.9",a="b"} 8.3835e-05
+{q="0.9","http.status",a="b"} 8.3835e-05
+{"go.gc_duration_seconds_sum"} 0.004304266
+{"Heizölrückstoßabdämpfung 10€ metric with \"interesting\" {character\nchoices}","strange©™\n'quoted' \"name\""="6"} 10.0`
+
+	input += "\n# EOF\n"
+
+	exp := []struct {
+		lset    labels.Labels
+		m       string
+		t       *int64
+		v       float64
+		typ     model.MetricType
+		help    string
+		unit    string
+		comment string
+		e       *exemplar.Exemplar
+	}{
+		{
+			m:    "go.gc_duration_seconds",
+			help: "A summary of the GC invocation durations.",
+		}, {
+			m:   "go.gc_duration_seconds",
+			typ: model.MetricTypeSummary,
+		}, {
+			m:    "go.gc_duration_seconds",
+			unit: "seconds",
+		}, {
+			m:    `{"go.gc_duration_seconds",quantile="0"}`,
+			v:    4.9351e-05,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0"),
+		}, {
+			m:    `{"go.gc_duration_seconds",quantile="0.25"}`,
+			v:    7.424100000000001e-05,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0.25"),
+		}, {
+			m:    `{"go.gc_duration_seconds",quantile="0.5",a="b"}`,
+			v:    8.3835e-05,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0.5", "a", "b"),
+		}, {
+			m:    `{"http.status",q="0.9",a="b"}`,
+			v:    8.3835e-05,
+			lset: labels.FromStrings("__name__", "http.status", "q", "0.9", "a", "b"),
+		}, {
+			m:    `{"http.status",q="0.9",a="b"}`,
+			v:    8.3835e-05,
+			lset: labels.FromStrings("__name__", "http.status", "q", "0.9", "a", "b"),
+		}, {
+			m:    `{q="0.9","http.status",a="b"}`,
+			v:    8.3835e-05,
+			lset: labels.FromStrings("__name__", "http.status", "q", "0.9", "a", "b"),
+		}, {
+			m:    `{"go.gc_duration_seconds_sum"}`,
+			v:    0.004304266,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds_sum"),
+		}, {
+			m: `{"Heizölrückstoßabdämpfung 10€ metric with \"interesting\" {character\nchoices}","strange©™\n'quoted' \"name\""="6"}`,
+			v: 10.0,
+			lset: labels.FromStrings("__name__", `Heizölrückstoßabdämpfung 10€ metric with "interesting" {character
+choices}`, "strange©™\n'quoted' \"name\"", "6"),
+		},
+	}
+
+	p := NewOpenMetricsParser([]byte(input))
+	i := 0
+
+	var res labels.Labels
+
 	for {
 		et, err := p.Next()
 		if errors.Is(err, io.EOF) {
@@ -456,17 +588,13 @@ func TestOpenMetricsParseErrors(t *testing.T) {
 			input: "a{b='c'} 1\n# EOF\n",
 			err:   "expected label value, got \"'\" (\"INVALID\") while parsing: \"a{b='\"",
 		},
-		{
-			input: "a{b=\"c\",} 1\n# EOF\n",
-			err:   "expected label name, got \"} \" (\"BCLOSE\") while parsing: \"a{b=\\\"c\\\",} \"",
-		},
 		{
 			input: "a{,b=\"c\"} 1\n# EOF\n",
-			err:   "expected label name or left brace, got \",b\" (\"COMMA\") while parsing: \"a{,b\"",
+			err:   "expected label name, got \",b\" (\"COMMA\") while parsing: \"a{,b\"",
 		},
 		{
 			input: "a{b=\"c\"d=\"e\"} 1\n# EOF\n",
-			err:   "expected comma, got \"d=\" (\"LNAME\") while parsing: \"a{b=\\\"c\\\"d=\"",
+			err:   "expected comma or brace close, got \"d=\" (\"LNAME\") while parsing: \"a{b=\\\"c\\\"d=\"",
 		},
 		{
 			input: "a{b=\"c\",,d=\"e\"} 1\n# EOF\n",
@@ -478,12 +606,24 @@ func TestOpenMetricsParseErrors(t *testing.T) {
 		},
 		{
 			input: "a{\xff=\"foo\"} 1\n# EOF\n",
-			err:   "expected label name or left brace, got \"\\xff\" (\"INVALID\") while parsing: \"a{\\xff\"",
+			err:   "expected label name, got \"\\xff\" (\"INVALID\") while parsing: \"a{\\xff\"",
 		},
 		{
 			input: "a{b=\"\xff\"} 1\n# EOF\n",
 			err:   "invalid UTF-8 label value: \"\\\"\\xff\\\"\"",
 		},
+		{
+			input: `{"a","b = "c"}
+# EOF
+`,
+			err: "expected equal, got \"c\\\"\" (\"LNAME\") while parsing: \"{\\\"a\\\",\\\"b = \\\"c\\\"\"",
+		},
+		{
+			input: `{"a",b\nc="d"} 1
+# EOF
+`,
+			err: "expected equal, got \"\\\\\" (\"INVALID\") while parsing: \"{\\\"a\\\",b\\\\\"",
+		},
 		{
 			input: "a true\n",
 			err:   "strconv.ParseFloat: parsing \"true\": invalid syntax while parsing: \"a true\"",
@@ -494,7 +634,7 @@ func TestOpenMetricsParseErrors(t *testing.T) {
 		},
 		{
 			input: "empty_label_name{=\"\"} 0\n# EOF\n",
-			err:   "expected label name or left brace, got \"=\\\"\" (\"EQUAL\") while parsing: \"empty_label_name{=\\\"\"",
+			err:   "expected label name, got \"=\\\"\" (\"EQUAL\") while parsing: \"empty_label_name{=\\\"\"",
 		},
 		{
 			input: "foo 1_2\n\n# EOF\n",
@@ -524,6 +664,14 @@ func TestOpenMetricsParseErrors(t *testing.T) {
 			input: `custom_metric_total 1 # {aa="bb"}`,
 			err:   "expected value after exemplar labels, got \"}\" (\"EOF\") while parsing: \"custom_metric_total 1 # {aa=\\\"bb\\\"}\"",
 		},
+		{
+			input: `custom_metric_total 1 # {bb}`,
+			err:   "expected label name, got \"}\" (\"BCLOSE\") while parsing: \"custom_metric_total 1 # {bb}\"",
+		},
+		{
+			input: `custom_metric_total 1 # {bb, a="dd"}`,
+			err:   "expected label name, got \", \" (\"COMMA\") while parsing: \"custom_metric_total 1 # {bb, \"",
+		},
 		{
 			input: `custom_metric_total 1 # {aa="bb",,cc="dd"} 1`,
 			err:   "expected label name, got \",c\" (\"COMMA\") while parsing: \"custom_metric_total 1 # {aa=\\\"bb\\\",,c\"",
@@ -550,7 +698,7 @@ func TestOpenMetricsParseErrors(t *testing.T) {
 		},
 		{
 			input: `{b="c",} 1`,
-			err:   "expected a valid start token, got \"{\" (\"INVALID\") while parsing: \"{\"",
+			err:   "metric name not set while parsing: \"{b=\\\"c\\\",} 1\"",
 		},
 		{
 			input: `a 1 NaN`,

model/textparse/promlex.l

@@ -66,12 +66,15 @@ C     [^\n]
 #                                     return l.consumeComment()
 <sComment>HELP[\t ]+                  l.state = sMeta1; return tHelp
 <sComment>TYPE[\t ]+                  l.state = sMeta1; return tType
+<sMeta1>\"(\\.|[^\\"])*\"             l.state = sMeta2; return tMName
 <sMeta1>{M}({M}|{D})*                 l.state = sMeta2; return tMName
 <sMeta2>{C}*                          l.state = sInit; return tText
 
 {M}({M}|{D})*                         l.state = sValue; return tMName
 <sValue>\{                            l.state = sLabels; return tBraceOpen
+\{                                    l.state = sLabels; return tBraceOpen
 <sLabels>{L}({L}|{D})*                return tLName
+<sLabels>\"(\\.|[^\\"])*\"            l.state = sLabels; return tQString
 <sLabels>\}                           l.state = sValue; return tBraceClose
 <sLabels>=                            l.state = sLValue; return tEqual
 <sLabels>,                            return tComma

model/textparse/promlex.l.go

@@ -51,19 +51,19 @@ yystate0:
 	case 0: // start condition: INITIAL
 		goto yystart1
 	case 1: // start condition: sComment
-		goto yystart8
+		goto yystart9
 	case 2: // start condition: sMeta1
-		goto yystart19
+		goto yystart20
 	case 3: // start condition: sMeta2
-		goto yystart21
+		goto yystart25
 	case 4: // start condition: sLabels
-		goto yystart24
+		goto yystart28
 	case 5: // start condition: sLValue
-		goto yystart29
-	case 6: // start condition: sValue
-		goto yystart33
-	case 7: // start condition: sTimestamp
 		goto yystart36
+	case 6: // start condition: sValue
+		goto yystart40
+	case 7: // start condition: sTimestamp
+		goto yystart43
 	}
 
 yystate1:
@@ -82,6 +82,8 @@ yystart1:
 		goto yystate3
 	case c == '\x00':
 		goto yystate2
+	case c == '{':
+		goto yystate8
 	}
 
 yystate2:
@@ -123,40 +125,35 @@ yystate7:
 	c = l.next()
 	switch {
 	default:
-		goto yyrule10
+		goto yyrule11
 	case c >= '0' && c <= ':' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
 		goto yystate7
 	}
 
 yystate8:
 	c = l.next()
-yystart8:
+	goto yyrule13
+
+yystate9:
+	c = l.next()
+yystart9:
 	switch {
 	default:
 		goto yyabort
 	case c == 'H':
-		goto yystate9
+		goto yystate10
 	case c == 'T':
-		goto yystate14
+		goto yystate15
 	case c == '\t' || c == ' ':
 		goto yystate3
 	}
 
-yystate9:
-	c = l.next()
-	switch {
-	default:
-		goto yyabort
-	case c == 'E':
-		goto yystate10
-	}
-
 yystate10:
 	c = l.next()
 	switch {
 	default:
 		goto yyabort
-	case c == 'L':
+	case c == 'E':
 		goto yystate11
 	}
 
@@ -165,7 +162,7 @@ yystate11:
 	switch {
 	default:
 		goto yyabort
-	case c == 'P':
+	case c == 'L':
 		goto yystate12
 	}
 
@@ -174,7 +171,7 @@ yystate12:
 	switch {
 	default:
 		goto yyabort
-	case c == '\t' || c == ' ':
+	case c == 'P':
 		goto yystate13
 	}
 
@@ -182,18 +179,18 @@ yystate13:
 	c = l.next()
 	switch {
 	default:
-		goto yyrule6
+		goto yyabort
 	case c == '\t' || c == ' ':
-		goto yystate13
+		goto yystate14
 	}
 
 yystate14:
 	c = l.next()
 	switch {
 	default:
-		goto yyabort
+		goto yyrule6
-	case c == 'Y':
+	case c == '\t' || c == ' ':
-		goto yystate15
+		goto yystate14
 	}
 
 yystate15:
@@ -201,7 +198,7 @@ yystate15:
 	switch {
 	default:
 		goto yyabort
-	case c == 'P':
+	case c == 'Y':
 		goto yystate16
 	}
 
@@ -210,7 +207,7 @@ yystate16:
 	switch {
 	default:
 		goto yyabort
-	case c == 'E':
+	case c == 'P':
 		goto yystate17
 	}
 
@@ -219,7 +216,7 @@ yystate17:
 	switch {
 	default:
 		goto yyabort
-	case c == '\t' || c == ' ':
+	case c == 'E':
 		goto yystate18
 	}
 
@@ -227,167 +224,167 @@ yystate18:
 	c = l.next()
 	switch {
 	default:
-		goto yyrule7
+		goto yyabort
 	case c == '\t' || c == ' ':
-		goto yystate18
+		goto yystate19
 	}
 
 yystate19:
 	c = l.next()
-yystart19:
 	switch {
 	default:
-		goto yyabort
+		goto yyrule7
-	case c == ':' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
-		goto yystate20
 	case c == '\t' || c == ' ':
-		goto yystate3
+		goto yystate19
 	}
 
 yystate20:
 	c = l.next()
+yystart20:
 	switch {
 	default:
-		goto yyrule8
+		goto yyabort
-	case c >= '0' && c <= ':' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
+	case c == '"':
-		goto yystate20
+		goto yystate21
+	case c == ':' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
+		goto yystate24
+	case c == '\t' || c == ' ':
+		goto yystate3
 	}
 
 yystate21:
 	c = l.next()
-yystart21:
 	switch {
 	default:
-		goto yyrule9
+		goto yyabort
-	case c == '\t' || c == ' ':
+	case c == '"':
-		goto yystate23
-	case c >= '\x01' && c <= '\b' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'ÿ':
 		goto yystate22
+	case c == '\\':
+		goto yystate23
+	case c >= '\x01' && c <= '!' || c >= '#' && c <= '[' || c >= ']' && c <= 'ÿ':
+		goto yystate21
 	}
 
 yystate22:
 	c = l.next()
-	switch {
+	goto yyrule8
-	default:
-		goto yyrule9
-	case c >= '\x01' && c <= '\t' || c >= '\v' && c <= 'ÿ':
-		goto yystate22
-	}
 
 yystate23:
 	c = l.next()
 	switch {
 	default:
-		goto yyrule3
+		goto yyabort
-	case c == '\t' || c == ' ':
+	case c >= '\x01' && c <= '\t' || c >= '\v' && c <= 'ÿ':
-		goto yystate23
+		goto yystate21
-	case c >= '\x01' && c <= '\b' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'ÿ':
-		goto yystate22
 	}
 
 yystate24:
 	c = l.next()
-yystart24:
 	switch {
 	default:
-		goto yyabort
+		goto yyrule9
-	case c == ',':
+	case c >= '0' && c <= ':' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
-		goto yystate25
+		goto yystate24
-	case c == '=':
-		goto yystate26
-	case c == '\t' || c == ' ':
-		goto yystate3
-	case c == '}':
-		goto yystate28
-	case c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
-		goto yystate27
 	}
 
 yystate25:
 	c = l.next()
-	goto yyrule15
+yystart25:
+	switch {
+	default:
+		goto yyrule10
+	case c == '\t' || c == ' ':
+		goto yystate27
+	case c >= '\x01' && c <= '\b' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'ÿ':
+		goto yystate26
+	}
 
 yystate26:
 	c = l.next()
-	goto yyrule14
+	switch {
+	default:
+		goto yyrule10
+	case c >= '\x01' && c <= '\t' || c >= '\v' && c <= 'ÿ':
+		goto yystate26
+	}
 
 yystate27:
 	c = l.next()
 	switch {
 	default:
-		goto yyrule12
+		goto yyrule3
-	case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
+	case c == '\t' || c == ' ':
 		goto yystate27
+	case c >= '\x01' && c <= '\b' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'ÿ':
+		goto yystate26
 	}
 
 yystate28:
 	c = l.next()
-	goto yyrule13
+yystart28:
+	switch {
+	default:
+		goto yyabort
+	case c == '"':
+		goto yystate29
+	case c == ',':
+		goto yystate32
+	case c == '=':
+		goto yystate33
+	case c == '\t' || c == ' ':
+		goto yystate3
+	case c == '}':
+		goto yystate35
+	case c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
+		goto yystate34
+	}
 
 yystate29:
 	c = l.next()
-yystart29:
 	switch {
 	default:
 		goto yyabort
 	case c == '"':
 		goto yystate30
-	case c == '\t' || c == ' ':
+	case c == '\\':
-		goto yystate3
+		goto yystate31
+	case c >= '\x01' && c <= '!' || c >= '#' && c <= '[' || c >= ']' && c <= 'ÿ':
+		goto yystate29
 	}
 
 yystate30:
 	c = l.next()
-	switch {
+	goto yyrule15
-	default:
-		goto yyabort
-	case c == '"':
-		goto yystate31
-	case c == '\\':
-		goto yystate32
-	case c >= '\x01' && c <= '!' || c >= '#' && c <= '[' || c >= ']' && c <= 'ÿ':
-		goto yystate30
-	}
 
 yystate31:
-	c = l.next()
-	goto yyrule16
-
-yystate32:
 	c = l.next()
 	switch {
 	default:
 		goto yyabort
 	case c >= '\x01' && c <= '\t' || c >= '\v' && c <= 'ÿ':
-		goto yystate30
+		goto yystate29
 	}
 
+yystate32:
+	c = l.next()
+	goto yyrule18
+
 yystate33:
 	c = l.next()
-yystart33:
+	goto yyrule17
-	switch {
-	default:
-		goto yyabort
-	case c == '\t' || c == ' ':
-		goto yystate3
-	case c == '{':
-		goto yystate35
-	case c >= '\x01' && c <= '\b' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'z' || c >= '|' && c <= 'ÿ':
-		goto yystate34
-	}
 
 yystate34:
 	c = l.next()
 	switch {
 	default:
-		goto yyrule17
+		goto yyrule14
-	case c >= '\x01' && c <= '\b' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'z' || c >= '|' && c <= 'ÿ':
+	case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
 		goto yystate34
 	}
 
 yystate35:
 	c = l.next()
-	goto yyrule11
+	goto yyrule16
 
 yystate36:
 	c = l.next()
@@ -395,25 +392,90 @@ yystart36:
 	switch {
 	default:
 		goto yyabort
-	case c == '\n':
+	case c == '"':
 		goto yystate37
 	case c == '\t' || c == ' ':
 		goto yystate3
-	case c >= '0' && c <= '9':
-		goto yystate38
 	}
 
 yystate37:
 	c = l.next()
-	goto yyrule19
+	switch {
+	default:
+		goto yyabort
+	case c == '"':
+		goto yystate38
+	case c == '\\':
+		goto yystate39
+	case c >= '\x01' && c <= '!' || c >= '#' && c <= '[' || c >= ']' && c <= 'ÿ':
+		goto yystate37
+	}
 
 yystate38:
+	c = l.next()
+	goto yyrule19
+
+yystate39:
 	c = l.next()
 	switch {
 	default:
-		goto yyrule18
+		goto yyabort
+	case c >= '\x01' && c <= '\t' || c >= '\v' && c <= 'ÿ':
+		goto yystate37
+	}
+
+yystate40:
+	c = l.next()
+yystart40:
+	switch {
+	default:
+		goto yyabort
+	case c == '\t' || c == ' ':
+		goto yystate3
+	case c == '{':
+		goto yystate42
+	case c >= '\x01' && c <= '\b' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'z' || c >= '|' && c <= 'ÿ':
+		goto yystate41
+	}
+
+yystate41:
+	c = l.next()
+	switch {
+	default:
+		goto yyrule20
+	case c >= '\x01' && c <= '\b' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'z' || c >= '|' && c <= 'ÿ':
+		goto yystate41
+	}
+
+yystate42:
+	c = l.next()
+	goto yyrule12
+
+yystate43:
+	c = l.next()
+yystart43:
+	switch {
+	default:
+		goto yyabort
+	case c == '\n':
+		goto yystate44
+	case c == '\t' || c == ' ':
+		goto yystate3
 	case c >= '0' && c <= '9':
-		goto yystate38
+		goto yystate45
+	}
+
+yystate44:
+	c = l.next()
+	goto yyrule22
+
+yystate45:
+	c = l.next()
+	switch {
+	default:
+		goto yyrule21
+	case c >= '0' && c <= '9':
+		goto yystate45
 	}
 
 yyrule1: // \0
@@ -451,67 +513,85 @@ yyrule7: // TYPE[\t ]+
 		return tType
 		goto yystate0
 	}
-yyrule8: // {M}({M}|{D})*
+yyrule8: // \"(\\.|[^\\"])*\"
 	{
 		l.state = sMeta2
 		return tMName
 		goto yystate0
 	}
-yyrule9: // {C}*
+yyrule9: // {M}({M}|{D})*
+	{
+		l.state = sMeta2
+		return tMName
+		goto yystate0
+	}
+yyrule10: // {C}*
 	{
 		l.state = sInit
 		return tText
 		goto yystate0
 	}
-yyrule10: // {M}({M}|{D})*
+yyrule11: // {M}({M}|{D})*
 	{
 		l.state = sValue
 		return tMName
 		goto yystate0
 	}
-yyrule11: // \{
+yyrule12: // \{
 	{
 		l.state = sLabels
 		return tBraceOpen
 		goto yystate0
 	}
-yyrule12: // {L}({L}|{D})*
+yyrule13: // \{
+	{
+		l.state = sLabels
+		return tBraceOpen
+		goto yystate0
+	}
+yyrule14: // {L}({L}|{D})*
 	{
 		return tLName
 	}
-yyrule13: // \}
+yyrule15: // \"(\\.|[^\\"])*\"
+	{
+		l.state = sLabels
+		return tQString
+		goto yystate0
+	}
+yyrule16: // \}
 	{
 		l.state = sValue
 		return tBraceClose
 		goto yystate0
 	}
-yyrule14: // =
+yyrule17: // =
 	{
 		l.state = sLValue
 		return tEqual
 		goto yystate0
 	}
-yyrule15: // ,
+yyrule18: // ,
 	{
 		return tComma
 	}
-yyrule16: // \"(\\.|[^\\"])*\"
+yyrule19: // \"(\\.|[^\\"])*\"
 	{
 		l.state = sLabels
 		return tLValue
 		goto yystate0
 	}
-yyrule17: // [^{ \t\n]+
+yyrule20: // [^{ \t\n]+
 	{
 		l.state = sTimestamp
 		return tValue
 		goto yystate0
 	}
-yyrule18: // {D}+
+yyrule21: // {D}+
 	{
 		return tTimestamp
 	}
-yyrule19: // \n
+yyrule22: // \n
 	if true { // avoid go vet determining the below panic will not be reached
 		l.state = sInit
 		return tLinebreak
@@ -520,9 +600,7 @@ yyrule19: // \n
 	panic("unreachable")
 
 yyabort: // no lexem recognized
-	//
 	// silence unused label errors for build and satisfy go vet reachability analysis
-	//
 	{
 		if false {
 			goto yyabort
@@ -534,26 +612,26 @@ yyabort: // no lexem recognized
 			goto yystate1
 		}
 		if false {
-			goto yystate8
+			goto yystate9
 		}
 		if false {
-			goto yystate19
+			goto yystate20
 		}
 		if false {
-			goto yystate21
+			goto yystate25
 		}
 		if false {
-			goto yystate24
+			goto yystate28
-		}
-		if false {
-			goto yystate29
-		}
-		if false {
-			goto yystate33
 		}
 		if false {
 			goto yystate36
 		}
+		if false {
+			goto yystate40
+		}
+		if false {
+			goto yystate43
+		}
 	}
 
 	// Workaround to gobble up comments that started with a HELP or TYPE

model/textparse/promparse.go

@@ -57,6 +57,7 @@ const (
 	tComment
 	tBlank
 	tMName
+	tQString
 	tBraceOpen
 	tBraceClose
 	tLName
@@ -93,6 +94,8 @@ func (t token) String() string {
 		return "BLANK"
 	case tMName:
 		return "MNAME"
+	case tQString:
+		return "QSTRING"
 	case tBraceOpen:
 		return "BOPEN"
 	case tBraceClose:
@@ -153,6 +156,12 @@ type PromParser struct {
 	ts      int64
 	hasTS   bool
 	start   int
+	// offsets is a list of offsets into series that describe the positions
+	// of the metric name and label names and values for this series.
+	// p.offsets[0] is the start character of the metric name.
+	// p.offsets[1] is the end of the metric name.
+	// Subsequently, p.offsets is a pair of pair of offsets for the positions
+	// of the label name and value start and end characters.
 	offsets []int
 }
 
@@ -218,20 +227,17 @@ func (p *PromParser) Metric(l *labels.Labels) string {
 	s := string(p.series)
 
 	p.builder.Reset()
-	p.builder.Add(labels.MetricName, s[:p.offsets[0]-p.start])
+	metricName := unreplace(s[p.offsets[0]-p.start : p.offsets[1]-p.start])
+	p.builder.Add(labels.MetricName, metricName)
 
-	for i := 1; i < len(p.offsets); i += 4 {
+	for i := 2; i < len(p.offsets); i += 4 {
 		a := p.offsets[i] - p.start
 		b := p.offsets[i+1] - p.start
+		label := unreplace(s[a:b])
 		c := p.offsets[i+2] - p.start
 		d := p.offsets[i+3] - p.start
-
+		value := unreplace(s[c:d])
-		value := s[c:d]
+		p.builder.Add(label, value)
-		// Replacer causes allocations. Replace only when necessary.
-		if strings.IndexByte(s[c:d], byte('\\')) >= 0 {
-			value = lvalReplacer.Replace(value)
-		}
-		p.builder.Add(s[a:b], value)
 	}
 
 	p.builder.Sort()
@@ -289,7 +295,13 @@ func (p *PromParser) Next() (Entry, error) {
 	case tHelp, tType:
 		switch t2 := p.nextToken(); t2 {
 		case tMName:
-			p.offsets = append(p.offsets, p.l.start, p.l.i)
+			mStart := p.l.start
+			mEnd := p.l.i
+			if p.l.b[mStart] == '"' && p.l.b[mEnd-1] == '"' {
+				mStart++
+				mEnd--
+			}
+			p.offsets = append(p.offsets, mStart, mEnd)
 		default:
 			return EntryInvalid, p.parseError("expected metric name after "+t.String(), t2)
 		}
@@ -301,7 +313,7 @@ func (p *PromParser) Next() (Entry, error) {
 				p.text = []byte{}
 			}
 		default:
-			return EntryInvalid, fmt.Errorf("expected text in %s", t.String())
+			return EntryInvalid, fmt.Errorf("expected text in %s, got %v", t.String(), t2.String())
 		}
 		switch t {
 		case tType:
@@ -339,12 +351,24 @@ func (p *PromParser) Next() (Entry, error) {
 			return EntryInvalid, p.parseError("linebreak expected after comment", t)
 		}
 		return EntryComment, nil
+	case tBraceOpen:
+		// We found a brace, so make room for the eventual metric name. If these
+		// values aren't updated, then the metric name was not set inside the
+		// braces and we can return an error.
+		if len(p.offsets) == 0 {
+			p.offsets = []int{-1, -1}
+		}
+		if err := p.parseLVals(); err != nil {
+			return EntryInvalid, err
+		}
 
-	case tMName:
-		p.offsets = append(p.offsets, p.l.i)
 		p.series = p.l.b[p.start:p.l.i]
-
+		return p.parseMetricSuffix(p.nextToken())
+	case tMName:
+		p.offsets = append(p.offsets, p.start, p.l.i)
+		p.series = p.l.b[p.start:p.l.i]
 		t2 := p.nextToken()
+		// If there's a brace, consume and parse the label values.
 		if t2 == tBraceOpen {
 			if err := p.parseLVals(); err != nil {
 				return EntryInvalid, err
@@ -352,32 +376,7 @@ func (p *PromParser) Next() (Entry, error) {
 			p.series = p.l.b[p.start:p.l.i]
 			t2 = p.nextToken()
 		}
-		if t2 != tValue {
+		return p.parseMetricSuffix(t2)
-			return EntryInvalid, p.parseError("expected value after metric", t2)
-		}
-		if p.val, err = parseFloat(yoloString(p.l.buf())); err != nil {
-			return EntryInvalid, fmt.Errorf("%w while parsing: %q", err, p.l.b[p.start:p.l.i])
-		}
-		// Ensure canonical NaN value.
-		if math.IsNaN(p.val) {
-			p.val = math.Float64frombits(value.NormalNaN)
-		}
-		p.hasTS = false
-		switch t := p.nextToken(); t {
-		case tLinebreak:
-			break
-		case tTimestamp:
-			p.hasTS = true
-			if p.ts, err = strconv.ParseInt(yoloString(p.l.buf()), 10, 64); err != nil {
-				return EntryInvalid, fmt.Errorf("%w while parsing: %q", err, p.l.b[p.start:p.l.i])
-			}
-			if t2 := p.nextToken(); t2 != tLinebreak {
-				return EntryInvalid, p.parseError("expected next entry after timestamp", t2)
-			}
-		default:
-			return EntryInvalid, p.parseError("expected timestamp or new record", t)
-		}
-		return EntrySeries, nil
 
 	default:
 		err = p.parseError("expected a valid start token", t)
@@ -385,19 +384,43 @@ func (p *PromParser) Next() (Entry, error) {
 	return EntryInvalid, err
 }
 
+// parseLVals parses the contents inside the braces.
 func (p *PromParser) parseLVals() error {
 	t := p.nextToken()
 	for {
+		curTStart := p.l.start
+		curTI := p.l.i
 		switch t {
 		case tBraceClose:
 			return nil
 		case tLName:
+		case tQString:
 		default:
 			return p.parseError("expected label name", t)
 		}
-		p.offsets = append(p.offsets, p.l.start, p.l.i)
 
-		if t := p.nextToken(); t != tEqual {
+		t = p.nextToken()
+		// A quoted string followed by a comma or brace is a metric name. Set the
+		// offsets and continue processing.
+		if t == tComma || t == tBraceClose {
+			if p.offsets[0] != -1 || p.offsets[1] != -1 {
+				return fmt.Errorf("metric name already set while parsing: %q", p.l.b[p.start:p.l.i])
+			}
+			p.offsets[0] = curTStart + 1
+			p.offsets[1] = curTI - 1
+			if t == tBraceClose {
+				return nil
+			}
+			t = p.nextToken()
+			continue
+		}
+		// We have a label name, and it might be quoted.
+		if p.l.b[curTStart] == '"' {
+			curTStart++
+			curTI--
+		}
+		p.offsets = append(p.offsets, curTStart, curTI)
+		if t != tEqual {
 			return p.parseError("expected equal", t)
 		}
 		if t := p.nextToken(); t != tLValue {
@@ -411,13 +434,51 @@ func (p *PromParser) parseLVals() error {
 		// and last character.
 		p.offsets = append(p.offsets, p.l.start+1, p.l.i-1)
 
-		// Free trailing commas are allowed.
+		// Free trailing commas are allowed. NOTE: this allows spaces between label
+		// names, unlike in OpenMetrics. It is not clear if this is intended or an
+		// accidental bug.
 		if t = p.nextToken(); t == tComma {
 			t = p.nextToken()
 		}
 	}
 }
 
+// parseMetricSuffix parses the end of the line after the metric name and
+// labels. It starts parsing with the provided token.
+func (p *PromParser) parseMetricSuffix(t token) (Entry, error) {
+	if p.offsets[0] == -1 {
+		return EntryInvalid, fmt.Errorf("metric name not set while parsing: %q", p.l.b[p.start:p.l.i])
+	}
+	if t != tValue {
+		return EntryInvalid, p.parseError("expected value after metric", t)
+	}
+	var err error
+	if p.val, err = parseFloat(yoloString(p.l.buf())); err != nil {
+		return EntryInvalid, fmt.Errorf("%w while parsing: %q", err, p.l.b[p.start:p.l.i])
+	}
+	// Ensure canonical NaN value.
+	if math.IsNaN(p.val) {
+		p.val = math.Float64frombits(value.NormalNaN)
+	}
+	p.hasTS = false
+	switch t := p.nextToken(); t {
+	case tLinebreak:
+		break
+	case tTimestamp:
+		p.hasTS = true
+		if p.ts, err = strconv.ParseInt(yoloString(p.l.buf()), 10, 64); err != nil {
+			return EntryInvalid, fmt.Errorf("%w while parsing: %q", err, p.l.b[p.start:p.l.i])
+		}
+		if t2 := p.nextToken(); t2 != tLinebreak {
+			return EntryInvalid, p.parseError("expected next entry after timestamp", t2)
+		}
+	default:
+		return EntryInvalid, p.parseError("expected timestamp or new record", t)
+	}
+
+	return EntrySeries, nil
+}
+
 var lvalReplacer = strings.NewReplacer(
 	`\"`, "\"",
 	`\\`, "\\",
@@ -429,6 +490,14 @@ var helpReplacer = strings.NewReplacer(
 	`\n`, "\n",
 )
 
+func unreplace(s string) string {
+	// Replacer causes allocations. Replace only when necessary.
+	if strings.IndexByte(s, byte('\\')) >= 0 {
+		return lvalReplacer.Replace(s)
+	}
+	return s
+}
+
 func yoloString(b []byte) string {
 	return *((*string)(unsafe.Pointer(&b)))
 }

model/textparse/promparse_test.go

@@ -26,6 +26,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/util/testutil"
 )
 
 func TestPromParse(t *testing.T) {
@@ -47,6 +48,7 @@ go_gc_duration_seconds{ quantile="1.0", a="b" } 8.3835e-05
 go_gc_duration_seconds { quantile="1.0", a="b" } 8.3835e-05
 go_gc_duration_seconds { quantile= "1.0", a= "b", } 8.3835e-05
 go_gc_duration_seconds { quantile = "1.0", a = "b" } 8.3835e-05
+go_gc_duration_seconds { quantile = "2.0" a = "b" } 8.3835e-05
 go_gc_duration_seconds_count 99
 some:aggregate:rate5m{a_b="c"}	1
 # HELP go_goroutines Number of goroutines that currently exist.
@@ -129,6 +131,11 @@ testmetric{label="\"bar\""} 1`
 			m:    `go_gc_duration_seconds { quantile = "1.0", a = "b" }`,
 			v:    8.3835e-05,
 			lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "1.0", "a", "b"),
+		}, {
+			// NOTE: Unlike OpenMetrics, Promparse allows spaces between label terms. This appears to be unintended and should probably be fixed.
+			m:    `go_gc_duration_seconds { quantile = "2.0" a = "b" }`,
+			v:    8.3835e-05,
+			lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "2.0", "a", "b"),
 		}, {
 			m:    `go_gc_duration_seconds_count`,
 			v:    99,
@@ -175,6 +182,132 @@ testmetric{label="\"bar\""} 1`
 
 	var res labels.Labels
 
+	for {
+		et, err := p.Next()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		require.NoError(t, err)
+
+		switch et {
+		case EntrySeries:
+			m, ts, v := p.Series()
+
+			p.Metric(&res)
+
+			require.Equal(t, exp[i].m, string(m))
+			require.Equal(t, exp[i].t, ts)
+			require.Equal(t, exp[i].v, v)
+			testutil.RequireEqual(t, exp[i].lset, res)
+
+		case EntryType:
+			m, typ := p.Type()
+			require.Equal(t, exp[i].m, string(m))
+			require.Equal(t, exp[i].typ, typ)
+
+		case EntryHelp:
+			m, h := p.Help()
+			require.Equal(t, exp[i].m, string(m))
+			require.Equal(t, exp[i].help, string(h))
+
+		case EntryComment:
+			require.Equal(t, exp[i].comment, string(p.Comment()))
+		}
+
+		i++
+	}
+	require.Len(t, exp, i)
+}
+
+func TestUTF8PromParse(t *testing.T) {
+	oldValidationScheme := model.NameValidationScheme
+	model.NameValidationScheme = model.UTF8Validation
+	defer func() {
+		model.NameValidationScheme = oldValidationScheme
+	}()
+
+	input := `# HELP "go.gc_duration_seconds" A summary of the GC invocation durations.
+# 	TYPE "go.gc_duration_seconds" summary
+{"go.gc_duration_seconds",quantile="0"} 4.9351e-05
+{"go.gc_duration_seconds",quantile="0.25",} 7.424100000000001e-05
+{"go.gc_duration_seconds",quantile="0.5",a="b"} 8.3835e-05
+{"go.gc_duration_seconds",quantile="0.8", a="b"} 8.3835e-05
+{"go.gc_duration_seconds", quantile="0.9", a="b"} 8.3835e-05
+{"go.gc_duration_seconds", quantile="1.0", a="b" } 8.3835e-05
+{ "go.gc_duration_seconds", quantile="1.0", a="b" } 8.3835e-05
+{ "go.gc_duration_seconds", quantile= "1.0", a= "b", } 8.3835e-05
+{ "go.gc_duration_seconds", quantile = "1.0", a = "b" } 8.3835e-05
+{"go.gc_duration_seconds_count"} 99
+{"Heizölrückstoßabdämpfung 10€ metric with \"interesting\" {character\nchoices}","strange©™\n'quoted' \"name\""="6"} 10.0`
+
+	exp := []struct {
+		lset    labels.Labels
+		m       string
+		t       *int64
+		v       float64
+		typ     model.MetricType
+		help    string
+		comment string
+	}{
+		{
+			m:    "go.gc_duration_seconds",
+			help: "A summary of the GC invocation durations.",
+		}, {
+			m:   "go.gc_duration_seconds",
+			typ: model.MetricTypeSummary,
+		}, {
+			m:    `{"go.gc_duration_seconds",quantile="0"}`,
+			v:    4.9351e-05,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0"),
+		}, {
+			m:    `{"go.gc_duration_seconds",quantile="0.25",}`,
+			v:    7.424100000000001e-05,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0.25"),
+		}, {
+			m:    `{"go.gc_duration_seconds",quantile="0.5",a="b"}`,
+			v:    8.3835e-05,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0.5", "a", "b"),
+		}, {
+			m:    `{"go.gc_duration_seconds",quantile="0.8", a="b"}`,
+			v:    8.3835e-05,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0.8", "a", "b"),
+		}, {
+			m:    `{"go.gc_duration_seconds", quantile="0.9", a="b"}`,
+			v:    8.3835e-05,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0.9", "a", "b"),
+		}, {
+			m:    `{"go.gc_duration_seconds", quantile="1.0", a="b" }`,
+			v:    8.3835e-05,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "1.0", "a", "b"),
+		}, {
+			m:    `{ "go.gc_duration_seconds", quantile="1.0", a="b" }`,
+			v:    8.3835e-05,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "1.0", "a", "b"),
+		}, {
+			m:    `{ "go.gc_duration_seconds", quantile= "1.0", a= "b", }`,
+			v:    8.3835e-05,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "1.0", "a", "b"),
+		}, {
+			m:    `{ "go.gc_duration_seconds", quantile = "1.0", a = "b" }`,
+			v:    8.3835e-05,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "1.0", "a", "b"),
+		}, {
+			m:    `{"go.gc_duration_seconds_count"}`,
+			v:    99,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds_count"),
+		}, {
+			m: `{"Heizölrückstoßabdämpfung 10€ metric with \"interesting\" {character\nchoices}","strange©™\n'quoted' \"name\""="6"}`,
+			v: 10.0,
+			lset: labels.FromStrings("__name__", `Heizölrückstoßabdämpfung 10€ metric with "interesting" {character
+choices}`, "strange©™\n'quoted' \"name\"", "6"),
+		},
+	}
+
+	p := NewPromParser([]byte(input))
+	i := 0
+
+	var res labels.Labels
+
 	for {
 		et, err := p.Next()
 		if errors.Is(err, io.EOF) {
@@ -237,6 +370,14 @@ func TestPromParseErrors(t *testing.T) {
 			input: "a{b=\"\xff\"} 1\n",
 			err:   "invalid UTF-8 label value: \"\\\"\\xff\\\"\"",
 		},
+		{
+			input: `{"a", "b = "c"}`,
+			err:   "expected equal, got \"c\\\"\" (\"LNAME\") while parsing: \"{\\\"a\\\", \\\"b = \\\"c\\\"\"",
+		},
+		{
+			input: `{"a",b\nc="d"} 1`,
+			err:   "expected equal, got \"\\\\\" (\"INVALID\") while parsing: \"{\\\"a\\\",b\\\\\"",
+		},
 		{
 			input: "a true\n",
 			err:   "strconv.ParseFloat: parsing \"true\": invalid syntax while parsing: \"a true\"",
@@ -267,7 +408,7 @@ func TestPromParseErrors(t *testing.T) {
 		},
 		{
 			input: `{a="ok"} 1`,
-			err:   "expected a valid start token, got \"{\" (\"INVALID\") while parsing: \"{\"",
+			err:   "metric name not set while parsing: \"{a=\\\"ok\\\"} 1\"",
 		},
 		{
 			input: "# TYPE #\n#EOF\n",

model/textparse/protobufparse.go

@@ -56,6 +56,8 @@ type ProtobufParser struct {
 	fieldPos    int
 	fieldsDone  bool // true if no more fields of a Summary or (legacy) Histogram to be processed.
 	redoClassic bool // true after parsing a native histogram if we need to parse it again as a classic histogram.
+	// exemplarPos is the position within the exemplars slice of a native histogram.
+	exemplarPos int
 
 	// exemplarReturned is set to true each time an exemplar has been
 	// returned, and set back to false upon each Next() call.
@@ -304,8 +306,9 @@ func (p *ProtobufParser) Metric(l *labels.Labels) string {
 
 // Exemplar writes the exemplar of the current sample into the passed
 // exemplar. It returns if an exemplar exists or not. In case of a native
-// histogram, the legacy bucket section is still used for exemplars. To ingest
+// histogram, the exemplars in the native histogram will be returned.
-// all exemplars, call the Exemplar method repeatedly until it returns false.
+// If this field is empty, the classic bucket section is still used for exemplars.
+// To ingest all exemplars, call the Exemplar method repeatedly until it returns false.
 func (p *ProtobufParser) Exemplar(ex *exemplar.Exemplar) bool {
 	if p.exemplarReturned && p.state == EntrySeries {
 		// We only ever return one exemplar per (non-native-histogram) series.
@@ -317,28 +320,42 @@ func (p *ProtobufParser) Exemplar(ex *exemplar.Exemplar) bool {
 	case dto.MetricType_COUNTER:
 		exProto = m.GetCounter().GetExemplar()
 	case dto.MetricType_HISTOGRAM, dto.MetricType_GAUGE_HISTOGRAM:
-		bb := m.GetHistogram().GetBucket()
 		isClassic := p.state == EntrySeries
-		if p.fieldPos < 0 {
+		if !isClassic && len(m.GetHistogram().GetExemplars()) > 0 {
-			if isClassic {
+			exs := m.GetHistogram().GetExemplars()
-				return false // At _count or _sum.
+			for p.exemplarPos < len(exs) {
+				exProto = exs[p.exemplarPos]
+				p.exemplarPos++
+				if exProto != nil && exProto.GetTimestamp() != nil {
+					break
+				}
 			}
-			p.fieldPos = 0 // Start at 1st bucket for native histograms.
+			if exProto != nil && exProto.GetTimestamp() == nil {
-		}
+				return false
-		for p.fieldPos < len(bb) {
-			exProto = bb[p.fieldPos].GetExemplar()
-			if isClassic {
-				break
 			}
-			p.fieldPos++
+		} else {
-			// We deliberately drop exemplars with no timestamp only for native histograms.
+			bb := m.GetHistogram().GetBucket()
-			if exProto != nil && (isClassic || exProto.GetTimestamp() != nil) {
+			if p.fieldPos < 0 {
-				break // Found a classic histogram exemplar or a native histogram exemplar with a timestamp.
+				if isClassic {
+					return false // At _count or _sum.
+				}
+				p.fieldPos = 0 // Start at 1st bucket for native histograms.
+			}
+			for p.fieldPos < len(bb) {
+				exProto = bb[p.fieldPos].GetExemplar()
+				if isClassic {
+					break
+				}
+				p.fieldPos++
+				// We deliberately drop exemplars with no timestamp only for native histograms.
+				if exProto != nil && (isClassic || exProto.GetTimestamp() != nil) {
+					break // Found a classic histogram exemplar or a native histogram exemplar with a timestamp.
+				}
+			}
+			// If the last exemplar for native histograms has no timestamp, ignore it.
+			if !isClassic && exProto.GetTimestamp() == nil {
+				return false
 			}
-		}
-		// If the last exemplar for native histograms has no timestamp, ignore it.
-		if !isClassic && exProto.GetTimestamp() == nil {
-			return false
 		}
 	default:
 		return false

model/textparse/protobufparse_test.go

@@ -27,6 +27,7 @@ import (
 	"github.com/prometheus/prometheus/model/exemplar"
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/util/testutil"
 
 	dto "github.com/prometheus/prometheus/prompb/io/prometheus/client"
 )
@@ -595,6 +596,105 @@ metric: <
   >
 >
 
+`,
+		`name: "test_histogram_with_native_histogram_exemplars"
+help: "A histogram with native histogram exemplars."
+type: HISTOGRAM
+metric: <
+  histogram: <
+    sample_count: 175
+    sample_sum: 0.0008280461746287094
+    bucket: <
+      cumulative_count: 2
+      upper_bound: -0.0004899999999999998
+    >
+    bucket: <
+      cumulative_count: 4
+      upper_bound: -0.0003899999999999998
+      exemplar: <
+	    label: <
+	      name: "dummyID"
+	      value: "59727"
+	    >
+	    value: -0.00039
+	    timestamp: <
+	      seconds: 1625851155
+	      nanos: 146848499
+	    >
+      >
+    >
+    bucket: <
+      cumulative_count: 16
+      upper_bound: -0.0002899999999999998
+      exemplar: <
+	    label: <
+	      name: "dummyID"
+	      value: "5617"
+	    >
+	    value: -0.00029
+      >
+    >
+    schema: 3
+    zero_threshold: 2.938735877055719e-39
+    zero_count: 2
+    negative_span: <
+      offset: -162
+      length: 1
+    >
+    negative_span: <
+      offset: 23
+      length: 4
+    >
+    negative_delta: 1
+    negative_delta: 3
+    negative_delta: -2
+    negative_delta: -1
+    negative_delta: 1
+    positive_span: <
+      offset: -161
+      length: 1
+    >
+    positive_span: <
+      offset: 8
+      length: 3
+    >
+    positive_delta: 1
+    positive_delta: 2
+    positive_delta: -1
+    positive_delta: -1
+    exemplars: <
+      label: <
+        name: "dummyID"
+        value: "59780"
+      >
+      value: -0.00039
+      timestamp: <
+        seconds: 1625851155
+        nanos: 146848499
+      >
+    >
+    exemplars: <
+      label: <
+        name: "dummyID"
+        value: "5617"
+      >
+      value: -0.00029
+    >
+    exemplars: <
+      label: <
+        name: "dummyID"
+        value: "59772"
+      >
+      value: -0.00052
+      timestamp: <
+        seconds: 1625851160
+        nanos: 156848499
+      >
+    >
+  >
+  timestamp_ms: 1234568
+>
+
 `,
 	}
 
@@ -1140,6 +1240,42 @@ func TestProtobufParse(t *testing.T) {
 						"__name__", "test_gaugehistogram_with_createdtimestamp",
 					),
 				},
+				{
+					m:    "test_histogram_with_native_histogram_exemplars",
+					help: "A histogram with native histogram exemplars.",
+				},
+				{
+					m:   "test_histogram_with_native_histogram_exemplars",
+					typ: model.MetricTypeHistogram,
+				},
+				{
+					m: "test_histogram_with_native_histogram_exemplars",
+					t: 1234568,
+					shs: &histogram.Histogram{
+						Count:         175,
+						ZeroCount:     2,
+						Sum:           0.0008280461746287094,
+						ZeroThreshold: 2.938735877055719e-39,
+						Schema:        3,
+						PositiveSpans: []histogram.Span{
+							{Offset: -161, Length: 1},
+							{Offset: 8, Length: 3},
+						},
+						NegativeSpans: []histogram.Span{
+							{Offset: -162, Length: 1},
+							{Offset: 23, Length: 4},
+						},
+						PositiveBuckets: []int64{1, 2, -1, -1},
+						NegativeBuckets: []int64{1, 3, -2, -1, 1},
+					},
+					lset: labels.FromStrings(
+						"__name__", "test_histogram_with_native_histogram_exemplars",
+					),
+					e: []exemplar.Exemplar{
+						{Labels: labels.FromStrings("dummyID", "59780"), Value: -0.00039, HasTs: true, Ts: 1625851155146},
+						{Labels: labels.FromStrings("dummyID", "59772"), Value: -0.00052, HasTs: true, Ts: 1625851160156},
+					},
+				},
 			},
 		},
 		{
@@ -1958,6 +2094,100 @@ func TestProtobufParse(t *testing.T) {
 						"__name__", "test_gaugehistogram_with_createdtimestamp",
 					),
 				},
+				{ // 94
+					m:    "test_histogram_with_native_histogram_exemplars",
+					help: "A histogram with native histogram exemplars.",
+				},
+				{ // 95
+					m:   "test_histogram_with_native_histogram_exemplars",
+					typ: model.MetricTypeHistogram,
+				},
+				{ // 96
+					m: "test_histogram_with_native_histogram_exemplars",
+					t: 1234568,
+					shs: &histogram.Histogram{
+						Count:         175,
+						ZeroCount:     2,
+						Sum:           0.0008280461746287094,
+						ZeroThreshold: 2.938735877055719e-39,
+						Schema:        3,
+						PositiveSpans: []histogram.Span{
+							{Offset: -161, Length: 1},
+							{Offset: 8, Length: 3},
+						},
+						NegativeSpans: []histogram.Span{
+							{Offset: -162, Length: 1},
+							{Offset: 23, Length: 4},
+						},
+						PositiveBuckets: []int64{1, 2, -1, -1},
+						NegativeBuckets: []int64{1, 3, -2, -1, 1},
+					},
+					lset: labels.FromStrings(
+						"__name__", "test_histogram_with_native_histogram_exemplars",
+					),
+					e: []exemplar.Exemplar{
+						{Labels: labels.FromStrings("dummyID", "59780"), Value: -0.00039, HasTs: true, Ts: 1625851155146},
+						{Labels: labels.FromStrings("dummyID", "59772"), Value: -0.00052, HasTs: true, Ts: 1625851160156},
+					},
+				},
+				{ // 97
+					m: "test_histogram_with_native_histogram_exemplars_count",
+					t: 1234568,
+					v: 175,
+					lset: labels.FromStrings(
+						"__name__", "test_histogram_with_native_histogram_exemplars_count",
+					),
+				},
+				{ // 98
+					m: "test_histogram_with_native_histogram_exemplars_sum",
+					t: 1234568,
+					v: 0.0008280461746287094,
+					lset: labels.FromStrings(
+						"__name__", "test_histogram_with_native_histogram_exemplars_sum",
+					),
+				},
+				{ // 99
+					m: "test_histogram_with_native_histogram_exemplars_bucket\xffle\xff-0.0004899999999999998",
+					t: 1234568,
+					v: 2,
+					lset: labels.FromStrings(
+						"__name__", "test_histogram_with_native_histogram_exemplars_bucket",
+						"le", "-0.0004899999999999998",
+					),
+				},
+				{ // 100
+					m: "test_histogram_with_native_histogram_exemplars_bucket\xffle\xff-0.0003899999999999998",
+					t: 1234568,
+					v: 4,
+					lset: labels.FromStrings(
+						"__name__", "test_histogram_with_native_histogram_exemplars_bucket",
+						"le", "-0.0003899999999999998",
+					),
+					e: []exemplar.Exemplar{
+						{Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00039, HasTs: true, Ts: 1625851155146},
+					},
+				},
+				{ // 101
+					m: "test_histogram_with_native_histogram_exemplars_bucket\xffle\xff-0.0002899999999999998",
+					t: 1234568,
+					v: 16,
+					lset: labels.FromStrings(
+						"__name__", "test_histogram_with_native_histogram_exemplars_bucket",
+						"le", "-0.0002899999999999998",
+					),
+					e: []exemplar.Exemplar{
+						{Labels: labels.FromStrings("dummyID", "5617"), Value: -0.00029, HasTs: false},
+					},
+				},
+				{ // 102
+					m: "test_histogram_with_native_histogram_exemplars_bucket\xffle\xff+Inf",
+					t: 1234568,
+					v: 175,
+					lset: labels.FromStrings(
+						"__name__", "test_histogram_with_native_histogram_exemplars_bucket",
+						"le", "+Inf",
+					),
+				},
 			},
 		},
 	}
@@ -1993,12 +2223,12 @@ func TestProtobufParse(t *testing.T) {
 						require.Equal(t, int64(0), exp[i].t, "i: %d", i)
 					}
 					require.Equal(t, exp[i].v, v, "i: %d", i)
-					require.Equal(t, exp[i].lset, res, "i: %d", i)
+					testutil.RequireEqual(t, exp[i].lset, res, "i: %d", i)
 					if len(exp[i].e) == 0 {
 						require.False(t, eFound, "i: %d", i)
 					} else {
 						require.True(t, eFound, "i: %d", i)
-						require.Equal(t, exp[i].e[0], e, "i: %d", i)
+						testutil.RequireEqual(t, exp[i].e[0], e, "i: %d", i)
 						require.False(t, p.Exemplar(&e), "too many exemplars returned, i: %d", i)
 					}
 					if exp[i].ct != 0 {
@@ -2017,7 +2247,7 @@ func TestProtobufParse(t *testing.T) {
 					} else {
 						require.Equal(t, int64(0), exp[i].t, "i: %d", i)
 					}
-					require.Equal(t, exp[i].lset, res, "i: %d", i)
+					testutil.RequireEqual(t, exp[i].lset, res, "i: %d", i)
 					require.Equal(t, exp[i].m, string(m), "i: %d", i)
 					if shs != nil {
 						require.Equal(t, exp[i].shs, shs, "i: %d", i)
@@ -2026,7 +2256,7 @@ func TestProtobufParse(t *testing.T) {
 					}
 					j := 0
 					for e := (exemplar.Exemplar{}); p.Exemplar(&e); j++ {
-						require.Equal(t, exp[i].e[j], e, "i: %d", i)
+						testutil.RequireEqual(t, exp[i].e[j], e, "i: %d", i)
 						e = exemplar.Exemplar{}
 					}
 					require.Len(t, exp[i].e, j, "not enough exemplars found, i: %d", i)

promql/engine.go

@@ -1494,10 +1494,14 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, annotations.Annotatio
 						otherInArgs[j][0].F = otherArgs[j][0].Floats[step].F
 					}
 				}
-				maxt := ts - offset
+				// Evaluate the matrix selector for this series
-				mint := maxt - selRange
+				// for this step, but only if this is the 1st
-				// Evaluate the matrix selector for this series for this step.
+				// iteration or no @ modifier has been used.
-				floats, histograms = ev.matrixIterSlice(it, mint, maxt, floats, histograms)
+				if ts == ev.startTimestamp || selVS.Timestamp == nil {
+					maxt := ts - offset
+					mint := maxt - selRange
+					floats, histograms = ev.matrixIterSlice(it, mint, maxt, floats, histograms)
+				}
 				if len(floats)+len(histograms) == 0 {
 					continue
 				}

promql/engine_test.go

@@ -38,6 +38,7 @@ import (
 	"github.com/prometheus/prometheus/util/annotations"
 	"github.com/prometheus/prometheus/util/stats"
 	"github.com/prometheus/prometheus/util/teststorage"
+	"github.com/prometheus/prometheus/util/testutil"
 )
 
 func TestMain(m *testing.M) {
@@ -1631,7 +1632,7 @@ load 1ms
 				sort.Sort(expMat)
 				sort.Sort(res.Value.(Matrix))
 			}
-			require.Equal(t, c.result, res.Value, "query %q failed", c.query)
+			testutil.RequireEqual(t, c.result, res.Value, "query %q failed", c.query)
 		})
 	}
 }
@@ -1956,7 +1957,7 @@ func TestSubquerySelector(t *testing.T) {
 					require.Equal(t, c.Result.Err, res.Err)
 					mat := res.Value.(Matrix)
 					sort.Sort(mat)
-					require.Equal(t, c.Result.Value, mat)
+					testutil.RequireEqual(t, c.Result.Value, mat)
 				})
 			}
 		})
@@ -2001,7 +2002,7 @@ load 1m
 
 	res := qry.Exec(context.Background())
 	require.NoError(t, res.Err)
-	require.Equal(t, expectedResult, res.Value)
+	testutil.RequireEqual(t, expectedResult, res.Value)
 }
 
 type FakeQueryLogger struct {
@@ -3147,7 +3148,7 @@ func TestRangeQuery(t *testing.T) {
 
 			res := qry.Exec(context.Background())
 			require.NoError(t, res.Err)
-			require.Equal(t, c.Result, res.Value)
+			testutil.RequireEqual(t, c.Result, res.Value)
 		})
 	}
 }
@@ -4347,7 +4348,7 @@ func TestNativeHistogram_Sum_Count_Add_AvgOperator(t *testing.T) {
 					vector, err := res.Vector()
 					require.NoError(t, err)
 
-					require.Equal(t, exp, vector)
+					testutil.RequireEqual(t, exp, vector)
 				}
 
 				// sum().
@@ -4605,7 +4606,7 @@ func TestNativeHistogram_SubOperator(t *testing.T) {
 						}
 					}
 
-					require.Equal(t, exp, vector)
+					testutil.RequireEqual(t, exp, vector)
 				}
 
 				// - operator.
@@ -4753,7 +4754,7 @@ func TestNativeHistogram_MulDivOperator(t *testing.T) {
 					vector, err := res.Vector()
 					require.NoError(t, err)
 
-					require.Equal(t, exp, vector)
+					testutil.RequireEqual(t, exp, vector)
 				}
 
 				// histogram * scalar.

promql/functions.go

@@ -1081,6 +1081,23 @@ func funcHistogramSum(vals []parser.Value, args parser.Expressions, enh *EvalNod
 	return enh.Out, nil
 }
 
+// === histogram_avg(Vector parser.ValueTypeVector) (Vector, Annotations) ===
+func funcHistogramAvg(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) {
+	inVec := vals[0].(Vector)
+
+	for _, sample := range inVec {
+		// Skip non-histogram samples.
+		if sample.H == nil {
+			continue
+		}
+		enh.Out = append(enh.Out, Sample{
+			Metric: sample.Metric.DropMetricName(),
+			F:      sample.H.Sum / sample.H.Count,
+		})
+	}
+	return enh.Out, nil
+}
+
 // === histogram_stddev(Vector parser.ValueTypeVector) (Vector, Annotations)  ===
 func funcHistogramStdDev(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) {
 	inVec := vals[0].(Vector)
@@ -1532,6 +1549,7 @@ var FunctionCalls = map[string]FunctionCall{
 	"deriv":              funcDeriv,
 	"exp":                funcExp,
 	"floor":              funcFloor,
+	"histogram_avg":      funcHistogramAvg,
 	"histogram_count":    funcHistogramCount,
 	"histogram_fraction": funcHistogramFraction,
 	"histogram_quantile": funcHistogramQuantile,

promql/parser/functions.go

@@ -167,6 +167,11 @@ var Functions = map[string]*Function{
 		ArgTypes:   []ValueType{ValueTypeVector},
 		ReturnType: ValueTypeVector,
 	},
+	"histogram_avg": {
+		Name:       "histogram_avg",
+		ArgTypes:   []ValueType{ValueTypeVector},
+		ReturnType: ValueTypeVector,
+	},
 	"histogram_count": {
 		Name:       "histogram_count",
 		ArgTypes:   []ValueType{ValueTypeVector},

promql/parser/generated_parser.y

@@ -161,7 +161,7 @@ START_METRIC_SELECTOR
 // Type definitions for grammar rules.
 %type <matchers> label_match_list
 %type <matcher> label_matcher
-%type <item> aggregate_op grouping_label match_op maybe_label metric_identifier unary_op at_modifier_preprocessors
+%type <item> aggregate_op grouping_label match_op maybe_label metric_identifier unary_op at_modifier_preprocessors string_identifier
 %type <labels> label_set metric
 %type <lblList> label_set_list
 %type <label> label_set_item
@@ -582,7 +582,13 @@ label_match_list: label_match_list COMMA label_matcher
                 ;
 
 label_matcher   : IDENTIFIER match_op STRING
-                        { $$ = yylex.(*parser).newLabelMatcher($1, $2, $3);  }
+                        { $$ = yylex.(*parser).newLabelMatcher($1, $2, $3); }
+                | string_identifier match_op STRING
+                        { $$ = yylex.(*parser).newLabelMatcher($1, $2, $3); }
+                | string_identifier
+                        { $$ = yylex.(*parser).newMetricNameMatcher($1); }
+                | string_identifier match_op error
+                        { yylex.(*parser).unexpected("label matching", "string"); $$ = nil}
                 | IDENTIFIER match_op error
                         { yylex.(*parser).unexpected("label matching", "string"); $$ = nil}
                 | IDENTIFIER error
@@ -901,7 +907,17 @@ string_literal  : STRING
                                 PosRange: $1.PositionRange(),
                         }
                         }
-                        ;
+                ;
+
+string_identifier  : STRING
+                        {
+                        $$ = Item{
+                                Typ: METRIC_IDENTIFIER,
+                                Pos: $1.PositionRange().Start,
+                                Val: yylex.(*parser).unquoteString($1.Val),
+                        }
+                        }
+                ;
 
 /*
  * Wrappers for optional arguments.

promql/parser/lex_test.go

@@ -815,16 +815,10 @@ func TestLexer(t *testing.T) {
 							hasError = true
 						}
 					}
-					if !hasError {
+					require.True(t, hasError, "%d: input %q, expected lexing error but did not fail", i, test.input)
-						t.Logf("%d: input %q", i, test.input)
-						require.Fail(t, "expected lexing error but did not fail")
-					}
 					continue
 				}
-				if lastItem.Typ == ERROR {
+				require.NotEqual(t, ERROR, lastItem.Typ, "%d: input %q, unexpected lexing error at position %d: %s", i, test.input, lastItem.Pos, lastItem)
-					t.Logf("%d: input %q", i, test.input)
-					require.Fail(t, "unexpected lexing error at position %d: %s", lastItem.Pos, lastItem)
-				}
 
 				eofItem := Item{EOF, posrange.Pos(len(test.input)), ""}
 				require.Equal(t, lastItem, eofItem, "%d: input %q", i, test.input)

promql/parser/parse.go

@@ -417,6 +417,8 @@ func (p *parser) newBinaryExpression(lhs Node, op Item, modifiers, rhs Node) *Bi
 }
 
 func (p *parser) assembleVectorSelector(vs *VectorSelector) {
+	// If the metric name was set outside the braces, add a matcher for it.
+	// If the metric name was inside the braces we don't need to do anything.
 	if vs.Name != "" {
 		nameMatcher, err := labels.NewMatcher(labels.MatchEqual, labels.MetricName, vs.Name)
 		if err != nil {
@@ -789,6 +791,18 @@ func (p *parser) checkAST(node Node) (typ ValueType) {
 			// Skip the check for non-empty matchers because an explicit
 			// metric name is a non-empty matcher.
 			break
+		} else {
+			// We also have to make sure a metric name was not set twice inside the
+			// braces.
+			foundMetricName := ""
+			for _, m := range n.LabelMatchers {
+				if m != nil && m.Name == labels.MetricName {
+					if foundMetricName != "" {
+						p.addParseErrf(n.PositionRange(), "metric name must not be set twice: %q or %q", foundMetricName, m.Value)
+					}
+					foundMetricName = m.Value
+				}
+			}
 		}
 
 		// A Vector selector must contain at least one non-empty matcher to prevent
@@ -872,6 +886,15 @@ func (p *parser) newLabelMatcher(label, operator, value Item) *labels.Matcher {
 	return m
 }
 
+func (p *parser) newMetricNameMatcher(value Item) *labels.Matcher {
+	m, err := labels.NewMatcher(labels.MatchEqual, labels.MetricName, value.Val)
+	if err != nil {
+		p.addParseErr(value.PositionRange(), err)
+	}
+
+	return m
+}
+
 // addOffset is used to set the offset in the generated parser.
 func (p *parser) addOffset(e Node, offset time.Duration) {
 	var orgoffsetp *time.Duration

promql/parser/parse_test.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/util/testutil"
 
 	"github.com/prometheus/prometheus/promql/parser/posrange"
 )
@@ -473,6 +474,22 @@ var testExpr = []struct {
 			StartPos: 1,
 		},
 	},
+	{
+		input: ` +{"some_metric"}`,
+		expected: &UnaryExpr{
+			Op: ADD,
+			Expr: &VectorSelector{
+				LabelMatchers: []*labels.Matcher{
+					MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "some_metric"),
+				},
+				PosRange: posrange.PositionRange{
+					Start: 2,
+					End:   17,
+				},
+			},
+			StartPos: 1,
+		},
+	},
 	{
 		input:  "",
 		fail:   true,
@@ -1701,6 +1718,33 @@ var testExpr = []struct {
 			},
 		},
 	},
+	{
+		input: `{"foo"}`,
+		expected: &VectorSelector{
+			// When a metric is named inside the braces, the Name field is not set.
+			LabelMatchers: []*labels.Matcher{
+				MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "foo"),
+			},
+			PosRange: posrange.PositionRange{
+				Start: 0,
+				End:   7,
+			},
+		},
+	},
+	{
+		input: `{"foo", a="bc"}`,
+		expected: &VectorSelector{
+			// When a metric is named inside the braces, the Name field is not set.
+			LabelMatchers: []*labels.Matcher{
+				MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "foo"),
+				MustLabelMatcher(labels.MatchEqual, "a", "bc"),
+			},
+			PosRange: posrange.PositionRange{
+				Start: 0,
+				End:   15,
+			},
+		},
+	},
 	{
 		input: `foo{NaN='bc'}`,
 		expected: &VectorSelector{
@@ -1746,6 +1790,23 @@ var testExpr = []struct {
 			},
 		},
 	},
+	{
+		// Metric name in the middle of selector list is fine.
+		input: `{a="b", foo!="bar", "foo", test=~"test", bar!~"baz"}`,
+		expected: &VectorSelector{
+			LabelMatchers: []*labels.Matcher{
+				MustLabelMatcher(labels.MatchEqual, "a", "b"),
+				MustLabelMatcher(labels.MatchNotEqual, "foo", "bar"),
+				MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "foo"),
+				MustLabelMatcher(labels.MatchRegexp, "test", "test"),
+				MustLabelMatcher(labels.MatchNotRegexp, "bar", "baz"),
+			},
+			PosRange: posrange.PositionRange{
+				Start: 0,
+				End:   52,
+			},
+		},
+	},
 	{
 		input: `foo{a="b", foo!="bar", test=~"test", bar!~"baz",}`,
 		expected: &VectorSelector{
@@ -1870,6 +1931,11 @@ var testExpr = []struct {
 		fail:   true,
 		errMsg: `unexpected identifier "lol" in label matching, expected "," or "}"`,
 	},
+	{
+		input:  `foo{"a"=}`,
+		fail:   true,
+		errMsg: `unexpected "}" in label matching, expected string`,
+	},
 	// Test matrix selector.
 	{
 		input: "test[5s]",
@@ -4018,7 +4084,7 @@ func TestParseSeries(t *testing.T) {
 
 		if !test.fail {
 			require.NoError(t, err)
-			require.Equal(t, test.expectedMetric, metric, "error on input '%s'", test.input)
+			testutil.RequireEqual(t, test.expectedMetric, metric, "error on input '%s'", test.input)
 			require.Equal(t, test.expectedValues, vals, "error in input '%s'", test.input)
 		} else {
 			require.Error(t, err)

promql/testdata/functions.test

@@ -221,19 +221,19 @@ eval instant at 50m deriv(testcounter_reset_middle[100m])
 # intercept at t=0: 6.818181818181818
 # intercept at t=3000: 38.63636363636364
 # intercept at t=3000+3600: 76.81818181818181
-eval instant at 50m predict_linear(testcounter_reset_middle[100m], 3600)
+eval instant at 50m predict_linear(testcounter_reset_middle[50m], 3600)
 	{} 76.81818181818181
 
 # intercept at t = 3000+3600 = 6600
-eval instant at 50m predict_linear(testcounter_reset_middle[100m] @ 3000, 3600)
+eval instant at 50m predict_linear(testcounter_reset_middle[50m] @ 3000, 3600)
 	{} 76.81818181818181
 
 # intercept at t = 600+3600 = 4200
-eval instant at 10m predict_linear(testcounter_reset_middle[100m] @ 3000, 3600)
+eval instant at 10m predict_linear(testcounter_reset_middle[50m] @ 3000, 3600)
 	{} 51.36363636363637
 
 # intercept at t = 4200+3600 = 7800
-eval instant at 70m predict_linear(testcounter_reset_middle[100m] @ 3000, 3600)
+eval instant at 70m predict_linear(testcounter_reset_middle[50m] @ 3000, 3600)
 	{} 89.54545454545455
 
 # With http_requests, there is a sample value exactly at the end of

promql/testdata/native_histograms.test

@@ -11,6 +11,9 @@ eval instant at 5m histogram_count(empty_histogram)
 eval instant at 5m histogram_sum(empty_histogram)
 	{} 0
 
+eval instant at 5m histogram_avg(empty_histogram)
+	{} NaN
+
 eval instant at 5m histogram_fraction(-Inf, +Inf, empty_histogram)
 	{} NaN
 
@@ -31,6 +34,10 @@ eval instant at 5m histogram_count(single_histogram)
 eval instant at 5m histogram_sum(single_histogram)
 	{} 5
 
+# histogram_avg calculates the average from sum and count properties.
+eval instant at 5m histogram_avg(single_histogram)
+	{} 1.25
+
 # We expect half of the values to fall in the range 1 < x <= 2.
 eval instant at 5m histogram_fraction(1, 2, single_histogram)
 	{} 0.5
@@ -55,6 +62,9 @@ eval instant at 5m histogram_count(multi_histogram)
 eval instant at 5m histogram_sum(multi_histogram)
 	{} 5
 
+eval instant at 5m histogram_avg(multi_histogram)
+	{} 1.25
+
 eval instant at 5m histogram_fraction(1, 2, multi_histogram)
 	{} 0.5
 
@@ -69,6 +79,9 @@ eval instant at 50m histogram_count(multi_histogram)
 eval instant at 50m histogram_sum(multi_histogram)
 	{} 5
 
+eval instant at 50m histogram_avg(multi_histogram)
+	{} 1.25
+
 eval instant at 50m histogram_fraction(1, 2, multi_histogram)
 	{} 0.5
 
@@ -89,6 +102,9 @@ eval instant at 5m histogram_count(incr_histogram)
 eval instant at 5m histogram_sum(incr_histogram)
 	{} 6
 
+eval instant at 5m histogram_avg(incr_histogram)
+	{} 1.2
+
 # We expect 3/5ths of the values to fall in the range 1 < x <= 2.
 eval instant at 5m histogram_fraction(1, 2, incr_histogram)
 	{} 0.6
@@ -106,6 +122,9 @@ eval instant at 50m histogram_count(incr_histogram)
 eval instant at 50m histogram_sum(incr_histogram)
 	{} 24
 
+eval instant at 50m histogram_avg(incr_histogram)
+    {} 1.7142857142857142
+
 # We expect 12/14ths of the values to fall in the range 1 < x <= 2.
 eval instant at 50m histogram_fraction(1, 2, incr_histogram)
 	{} 0.8571428571428571
@@ -140,6 +159,9 @@ eval instant at 5m histogram_count(low_res_histogram)
 eval instant at 5m histogram_sum(low_res_histogram)
 	{} 8
 
+eval instant at 5m histogram_avg(low_res_histogram)
+	{} 1.6
+
 # We expect all values to fall into the lower-resolution bucket with the range 1 < x <= 4.
 eval instant at 5m histogram_fraction(1, 4, low_res_histogram)
 	{} 1
@@ -157,6 +179,9 @@ eval instant at 5m histogram_count(single_zero_histogram)
 eval instant at 5m histogram_sum(single_zero_histogram)
 	{} 0.25
 
+eval instant at 5m histogram_avg(single_zero_histogram)
+	{} 0.25
+
 # When only the zero bucket is populated, or there are negative buckets, the distribution is assumed to be equally
 # distributed around zero; i.e. that there are an equal number of positive and negative observations. Therefore the
 # entire distribution must lie within the full range of the zero bucket, in this case: -0.5 < x <= +0.5.
@@ -179,6 +204,9 @@ eval instant at 5m histogram_count(negative_histogram)
 eval instant at 5m histogram_sum(negative_histogram)
 	{} -5
 
+eval instant at 5m histogram_avg(negative_histogram)
+	{} -1.25
+
 # We expect half of the values to fall in the range -2 < x <= -1.
 eval instant at 5m histogram_fraction(-2, -1, negative_histogram)
 	{} 0.5
@@ -199,6 +227,9 @@ eval instant at 10m histogram_count(two_samples_histogram)
 eval instant at 10m histogram_sum(two_samples_histogram)
 	{} -4
 
+eval instant at 10m histogram_avg(two_samples_histogram)
+	{} -1
+
 eval instant at 10m histogram_fraction(-2, -1, two_samples_histogram)
 	{} 0.5
 
@@ -217,6 +248,9 @@ eval instant at 5m histogram_count(balanced_histogram)
 eval instant at 5m histogram_sum(balanced_histogram)
 	{} 0
 
+eval instant at 5m histogram_avg(balanced_histogram)
+	{} 0
+
 eval instant at 5m histogram_fraction(0, 4, balanced_histogram)
 	{} 0.5
 

rules/alerting.go

@@ -142,6 +142,9 @@ type AlertingRule struct {
 	active map[uint64]*Alert
 
 	logger log.Logger
+
+	noDependentRules  *atomic.Bool
+	noDependencyRules *atomic.Bool
 }
 
 // NewAlertingRule constructs a new AlertingRule.
@@ -168,6 +171,8 @@ func NewAlertingRule(
 		evaluationTimestamp: atomic.NewTime(time.Time{}),
 		evaluationDuration:  atomic.NewDuration(0),
 		lastError:           atomic.NewError(nil),
+		noDependentRules:    atomic.NewBool(false),
+		noDependencyRules:   atomic.NewBool(false),
 	}
 }
 
@@ -317,6 +322,22 @@ func (r *AlertingRule) Restored() bool {
 	return r.restored.Load()
 }
 
+func (r *AlertingRule) SetNoDependentRules(noDependentRules bool) {
+	r.noDependentRules.Store(noDependentRules)
+}
+
+func (r *AlertingRule) NoDependentRules() bool {
+	return r.noDependentRules.Load()
+}
+
+func (r *AlertingRule) SetNoDependencyRules(noDependencyRules bool) {
+	r.noDependencyRules.Store(noDependencyRules)
+}
+
+func (r *AlertingRule) NoDependencyRules() bool {
+	return r.noDependencyRules.Load()
+}
+
 // resolvedRetention is the duration for which a resolved alert instance
 // is kept in memory state and consequently repeatedly sent to the AlertManager.
 const resolvedRetention = 15 * time.Minute

rules/alerting_test.go

@@ -31,6 +31,7 @@ import (
 	"github.com/prometheus/prometheus/promql/parser"
 	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/util/teststorage"
+	"github.com/prometheus/prometheus/util/testutil"
 )
 
 var testEngine = promql.NewEngine(promql.EngineOpts{
@@ -180,7 +181,7 @@ func TestAlertingRuleLabelsUpdate(t *testing.T) {
 			}
 		}
 
-		require.Equal(t, result, filteredRes)
+		testutil.RequireEqual(t, result, filteredRes)
 	}
 	evalTime := baseTime.Add(time.Duration(len(results)) * time.Minute)
 	res, err := rule.Eval(context.TODO(), evalTime, EngineQueryFunc(testEngine, storage), nil, 0)
@@ -278,7 +279,7 @@ func TestAlertingRuleExternalLabelsInTemplate(t *testing.T) {
 		}
 	}
 
-	require.Equal(t, result, filteredRes)
+	testutil.RequireEqual(t, result, filteredRes)
 }
 
 func TestAlertingRuleExternalURLInTemplate(t *testing.T) {
@@ -371,7 +372,7 @@ func TestAlertingRuleExternalURLInTemplate(t *testing.T) {
 		}
 	}
 
-	require.Equal(t, result, filteredRes)
+	testutil.RequireEqual(t, result, filteredRes)
 }
 
 func TestAlertingRuleEmptyLabelFromTemplate(t *testing.T) {
@@ -425,7 +426,7 @@ func TestAlertingRuleEmptyLabelFromTemplate(t *testing.T) {
 			require.Equal(t, "ALERTS_FOR_STATE", smplName)
 		}
 	}
-	require.Equal(t, result, filteredRes)
+	testutil.RequireEqual(t, result, filteredRes)
 }
 
 func TestAlertingRuleQueryInTemplate(t *testing.T) {
@@ -718,7 +719,7 @@ func TestSendAlertsDontAffectActiveAlerts(t *testing.T) {
 
 	// The relabel rule changes a1=1 to a1=bug.
 	// But the labels with the AlertingRule should not be changed.
-	require.Equal(t, labels.FromStrings("a1", "1"), rule.active[h].Labels)
+	testutil.RequireEqual(t, labels.FromStrings("a1", "1"), rule.active[h].Labels)
 }
 
 func TestKeepFiringFor(t *testing.T) {
@@ -823,7 +824,7 @@ func TestKeepFiringFor(t *testing.T) {
 			}
 		}
 
-		require.Equal(t, result, filteredRes)
+		testutil.RequireEqual(t, result, filteredRes)
 	}
 	evalTime := baseTime.Add(time.Duration(len(results)) * time.Minute)
 	res, err := rule.Eval(context.TODO(), evalTime, EngineQueryFunc(testEngine, storage), nil, 0)
@@ -870,7 +871,7 @@ func TestPendingAndKeepFiringFor(t *testing.T) {
 	for _, smpl := range res {
 		smplName := smpl.Metric.Get("__name__")
 		if smplName == "ALERTS" {
-			require.Equal(t, result, smpl)
+			testutil.RequireEqual(t, result, smpl)
 		} else {
 			// If not 'ALERTS', it has to be 'ALERTS_FOR_STATE'.
 			require.Equal(t, "ALERTS_FOR_STATE", smplName)
@@ -920,3 +921,45 @@ func TestAlertingEvalWithOrigin(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, detail, NewRuleDetail(rule))
 }
+
+func TestAlertingRule_SetNoDependentRules(t *testing.T) {
+	rule := NewAlertingRule(
+		"test",
+		&parser.NumberLiteral{Val: 1},
+		time.Minute,
+		0,
+		labels.FromStrings("test", "test"),
+		labels.EmptyLabels(),
+		labels.EmptyLabels(),
+		"",
+		true, log.NewNopLogger(),
+	)
+	require.False(t, rule.NoDependentRules())
+
+	rule.SetNoDependentRules(false)
+	require.False(t, rule.NoDependentRules())
+
+	rule.SetNoDependentRules(true)
+	require.True(t, rule.NoDependentRules())
+}
+
+func TestAlertingRule_SetNoDependencyRules(t *testing.T) {
+	rule := NewAlertingRule(
+		"test",
+		&parser.NumberLiteral{Val: 1},
+		time.Minute,
+		0,
+		labels.FromStrings("test", "test"),
+		labels.EmptyLabels(),
+		labels.EmptyLabels(),
+		"",
+		true, log.NewNopLogger(),
+	)
+	require.False(t, rule.NoDependencyRules())
+
+	rule.SetNoDependencyRules(false)
+	require.False(t, rule.NoDependencyRules())
+
+	rule.SetNoDependencyRules(true)
+	require.True(t, rule.NoDependencyRules())
+}

rules/group.go

@@ -464,7 +464,7 @@ func (g *Group) Eval(ctx context.Context, ts time.Time) {
 			}(time.Now())
 
 			if sp.SpanContext().IsSampled() && sp.SpanContext().HasTraceID() {
-				logger = log.WithPrefix(logger, "traceID", sp.SpanContext().TraceID())
+				logger = log.WithPrefix(logger, "trace_id", sp.SpanContext().TraceID())
 			}
 
 			g.metrics.EvalTotal.WithLabelValues(GroupKey(g.File(), g.Name())).Inc()
@@ -579,7 +579,7 @@ func (g *Group) Eval(ctx context.Context, ts time.Time) {
 
 		// If the rule has no dependencies, it can run concurrently because no other rules in this group depend on its output.
 		// Try run concurrently if there are slots available.
-		if ctrl := g.concurrencyController; ctrl.RuleEligible(g, rule) && ctrl.Allow() {
+		if ctrl := g.concurrencyController; isRuleEligibleForConcurrentExecution(rule) && ctrl.Allow() {
 			wg.Add(1)
 
 			go eval(i, rule, func() {
@@ -1008,3 +1008,7 @@ func buildDependencyMap(rules []Rule) dependencyMap {
 
 	return dependencies
 }
+
+func isRuleEligibleForConcurrentExecution(rule Rule) bool {
+	return rule.NoDependentRules() && rule.NoDependencyRules()
+}

rules/manager.go

@@ -119,6 +119,7 @@ type ManagerOptions struct {
 	MaxConcurrentEvals        int64
 	ConcurrentEvalsEnabled    bool
 	RuleConcurrencyController RuleConcurrencyController
+	RuleDependencyController  RuleDependencyController
 
 	Metrics *Metrics
 }
@@ -142,6 +143,10 @@ func NewManager(o *ManagerOptions) *Manager {
 		}
 	}
 
+	if o.RuleDependencyController == nil {
+		o.RuleDependencyController = ruleDependencyController{}
+	}
+
 	m := &Manager{
 		groups: map[string]*Group{},
 		opts:   o,
@@ -188,8 +193,6 @@ func (m *Manager) Update(interval time.Duration, files []string, externalLabels
 	m.mtx.Lock()
 	defer m.mtx.Unlock()
 
-	m.opts.RuleConcurrencyController.Invalidate()
-
 	groups, errs := m.LoadGroups(interval, externalLabels, externalURL, groupEvalIterationFunc, files...)
 
 	if errs != nil {
@@ -322,6 +325,9 @@ func (m *Manager) LoadGroups(
 				))
 			}
 
+			// Check dependencies between rules and store it on the Rule itself.
+			m.opts.RuleDependencyController.AnalyseRules(rules)
+
 			groups[GroupKey(fn, rg.Name)] = NewGroup(GroupOptions{
 				Name:              rg.Name,
 				File:              fn,
@@ -418,24 +424,35 @@ func SendAlerts(s Sender, externalURL string) NotifyFunc {
 	}
 }
 
-// RuleConcurrencyController controls whether rules can be evaluated concurrently. Its purpose is to bound the amount
+// RuleDependencyController controls whether a set of rules have dependencies between each other.
-// of concurrency in rule evaluations to avoid overwhelming the Prometheus server with additional query load and ensure
+type RuleDependencyController interface {
-// the correctness of rules running concurrently. Concurrency is controlled globally, not on a per-group basis.
+	// AnalyseRules analyses dependencies between the input rules. For each rule that it's guaranteed
-type RuleConcurrencyController interface {
+	// not having any dependants and/or dependency, this function should call Rule.SetNoDependentRules(true)
-	// RuleEligible determines if the rule can guarantee correct results while running concurrently.
+	// and/or Rule.SetNoDependencyRules(true).
-	RuleEligible(g *Group, r Rule) bool
+	AnalyseRules(rules []Rule)
+}
 
+type ruleDependencyController struct{}
+
+// AnalyseRules implements RuleDependencyController.
+func (c ruleDependencyController) AnalyseRules(rules []Rule) {
+	depMap := buildDependencyMap(rules)
+	for _, r := range rules {
+		r.SetNoDependentRules(depMap.dependents(r) == 0)
+		r.SetNoDependencyRules(depMap.dependencies(r) == 0)
+	}
+}
+
+// RuleConcurrencyController controls concurrency for rules that are safe to be evaluated concurrently.
+// Its purpose is to bound the amount of concurrency in rule evaluations to avoid overwhelming the Prometheus
+// server with additional query load. Concurrency is controlled globally, not on a per-group basis.
+type RuleConcurrencyController interface {
 	// Allow determines whether any concurrent evaluation slots are available.
 	// If Allow() returns true, then Done() must be called to release the acquired slot.
 	Allow() bool
 
 	// Done releases a concurrent evaluation slot.
 	Done()
-
-	// Invalidate instructs the controller to invalidate its state.
-	// This should be called when groups are modified (during a reload, for instance), because the controller may
-	// store some state about each group in order to more efficiently determine rule eligibility.
-	Invalidate()
 }
 
 // concurrentRuleEvalController holds a weighted semaphore which controls the concurrent evaluation of rules.

rules/manager_test.go

@@ -42,6 +42,7 @@ import (
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
 	"github.com/prometheus/prometheus/tsdb/tsdbutil"
 	"github.com/prometheus/prometheus/util/teststorage"
+	prom_testutil "github.com/prometheus/prometheus/util/testutil"
 )
 
 func TestMain(m *testing.M) {
@@ -180,7 +181,7 @@ func TestAlertingRule(t *testing.T) {
 		sort.Slice(filteredRes, func(i, j int) bool {
 			return labels.Compare(filteredRes[i].Metric, filteredRes[j].Metric) < 0
 		})
-		require.Equal(t, test.result, filteredRes)
+		prom_testutil.RequireEqual(t, test.result, filteredRes)
 
 		for _, aa := range rule.ActiveAlerts() {
 			require.Zero(t, aa.Labels.Get(model.MetricNameLabel), "%s label set on active alert: %s", model.MetricNameLabel, aa.Labels)
@@ -330,7 +331,7 @@ func TestForStateAddSamples(t *testing.T) {
 		sort.Slice(filteredRes, func(i, j int) bool {
 			return labels.Compare(filteredRes[i].Metric, filteredRes[j].Metric) < 0
 		})
-		require.Equal(t, test.result, filteredRes)
+		prom_testutil.RequireEqual(t, test.result, filteredRes)
 
 		for _, aa := range rule.ActiveAlerts() {
 			require.Zero(t, aa.Labels.Get(model.MetricNameLabel), "%s label set on active alert: %s", model.MetricNameLabel, aa.Labels)
@@ -1314,6 +1315,8 @@ func TestRuleGroupEvalIterationFunc(t *testing.T) {
 			evaluationTimestamp: atomic.NewTime(time.Time{}),
 			evaluationDuration:  atomic.NewDuration(0),
 			lastError:           atomic.NewError(nil),
+			noDependentRules:    atomic.NewBool(false),
+			noDependencyRules:   atomic.NewBool(false),
 		}
 
 		group := NewGroup(GroupOptions{
@@ -1407,6 +1410,66 @@ func TestNativeHistogramsInRecordingRules(t *testing.T) {
 	require.Equal(t, chunkenc.ValNone, it.Next())
 }
 
+func TestManager_LoadGroups_ShouldCheckWhetherEachRuleHasDependentsAndDependencies(t *testing.T) {
+	storage := teststorage.New(t)
+	t.Cleanup(func() {
+		require.NoError(t, storage.Close())
+	})
+
+	ruleManager := NewManager(&ManagerOptions{
+		Context:    context.Background(),
+		Logger:     log.NewNopLogger(),
+		Appendable: storage,
+		QueryFunc:  func(ctx context.Context, q string, ts time.Time) (promql.Vector, error) { return nil, nil },
+	})
+
+	t.Run("load a mix of dependent and independent rules", func(t *testing.T) {
+		groups, errs := ruleManager.LoadGroups(time.Second, labels.EmptyLabels(), "", nil, []string{"fixtures/rules_multiple.yaml"}...)
+		require.Empty(t, errs)
+		require.Len(t, groups, 1)
+
+		expected := map[string]struct {
+			noDependentRules  bool
+			noDependencyRules bool
+		}{
+			"job:http_requests:rate1m": {
+				noDependentRules:  true,
+				noDependencyRules: true,
+			},
+			"job:http_requests:rate5m": {
+				noDependentRules:  true,
+				noDependencyRules: true,
+			},
+			"job:http_requests:rate15m": {
+				noDependentRules:  true,
+				noDependencyRules: false,
+			},
+			"TooManyRequests": {
+				noDependentRules:  false,
+				noDependencyRules: true,
+			},
+		}
+
+		for _, r := range ruleManager.Rules() {
+			exp, ok := expected[r.Name()]
+			require.Truef(t, ok, "rule: %s", r.String())
+			require.Equalf(t, exp.noDependentRules, r.NoDependentRules(), "rule: %s", r.String())
+			require.Equalf(t, exp.noDependencyRules, r.NoDependencyRules(), "rule: %s", r.String())
+		}
+	})
+
+	t.Run("load only independent rules", func(t *testing.T) {
+		groups, errs := ruleManager.LoadGroups(time.Second, labels.EmptyLabels(), "", nil, []string{"fixtures/rules_multiple_independent.yaml"}...)
+		require.Empty(t, errs)
+		require.Len(t, groups, 1)
+
+		for _, r := range ruleManager.Rules() {
+			require.Truef(t, r.NoDependentRules(), "rule: %s", r.String())
+			require.Truef(t, r.NoDependencyRules(), "rule: %s", r.String())
+		}
+	})
+}
+
 func TestDependencyMap(t *testing.T) {
 	ctx := context.Background()
 	opts := &ManagerOptions{

rules/origin.go

@@ -28,6 +28,14 @@ type RuleDetail struct {
 	Query  string
 	Labels labels.Labels
 	Kind   string
+
+	// NoDependentRules is set to true if it's guaranteed that in the rule group there's no other rule
+	// which depends on this one.
+	NoDependentRules bool
+
+	// NoDependencyRules is set to true if it's guaranteed that this rule doesn't depend on any other
+	// rule within the rule group.
+	NoDependencyRules bool
 }
 
 const (
@@ -48,10 +56,12 @@ func NewRuleDetail(r Rule) RuleDetail {
 	}
 
 	return RuleDetail{
-		Name:   r.Name(),
+		Name:              r.Name(),
-		Query:  r.Query().String(),
+		Query:             r.Query().String(),
-		Labels: r.Labels(),
+		Labels:            r.Labels(),
-		Kind:   kind,
+		Kind:              kind,
+		NoDependentRules:  r.NoDependentRules(),
+		NoDependencyRules: r.NoDependencyRules(),
 	}
 }
 

rules/origin_test.go

@@ -19,6 +19,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/go-kit/log"
 	"github.com/stretchr/testify/require"
 
 	"github.com/prometheus/prometheus/model/labels"
@@ -43,9 +44,73 @@ func (u unknownRule) SetEvaluationDuration(time.Duration)  {}
 func (u unknownRule) GetEvaluationDuration() time.Duration { return 0 }
 func (u unknownRule) SetEvaluationTimestamp(time.Time)     {}
 func (u unknownRule) GetEvaluationTimestamp() time.Time    { return time.Time{} }
+func (u unknownRule) SetNoDependentRules(bool)             {}
+func (u unknownRule) NoDependentRules() bool               { return false }
+func (u unknownRule) SetNoDependencyRules(bool)            {}
+func (u unknownRule) NoDependencyRules() bool              { return false }
 
 func TestNewRuleDetailPanics(t *testing.T) {
 	require.PanicsWithValue(t, `unknown rule type "rules.unknownRule"`, func() {
 		NewRuleDetail(unknownRule{})
 	})
 }
+
+func TestFromOriginContext(t *testing.T) {
+	t.Run("should return zero value if RuleDetail is missing in the context", func(t *testing.T) {
+		detail := FromOriginContext(context.Background())
+		require.Zero(t, detail)
+
+		// The zero value for NoDependentRules must be the most conservative option.
+		require.False(t, detail.NoDependentRules)
+
+		// The zero value for NoDependencyRules must be the most conservative option.
+		require.False(t, detail.NoDependencyRules)
+	})
+}
+
+func TestNewRuleDetail(t *testing.T) {
+	t.Run("should populate NoDependentRules and NoDependencyRules for a RecordingRule", func(t *testing.T) {
+		rule := NewRecordingRule("test", &parser.NumberLiteral{Val: 1}, labels.EmptyLabels())
+		detail := NewRuleDetail(rule)
+		require.False(t, detail.NoDependentRules)
+		require.False(t, detail.NoDependencyRules)
+
+		rule.SetNoDependentRules(true)
+		detail = NewRuleDetail(rule)
+		require.True(t, detail.NoDependentRules)
+		require.False(t, detail.NoDependencyRules)
+
+		rule.SetNoDependencyRules(true)
+		detail = NewRuleDetail(rule)
+		require.True(t, detail.NoDependentRules)
+		require.True(t, detail.NoDependencyRules)
+	})
+
+	t.Run("should populate NoDependentRules and NoDependencyRules for a AlertingRule", func(t *testing.T) {
+		rule := NewAlertingRule(
+			"test",
+			&parser.NumberLiteral{Val: 1},
+			time.Minute,
+			0,
+			labels.FromStrings("test", "test"),
+			labels.EmptyLabels(),
+			labels.EmptyLabels(),
+			"",
+			true, log.NewNopLogger(),
+		)
+
+		detail := NewRuleDetail(rule)
+		require.False(t, detail.NoDependentRules)
+		require.False(t, detail.NoDependencyRules)
+
+		rule.SetNoDependentRules(true)
+		detail = NewRuleDetail(rule)
+		require.True(t, detail.NoDependentRules)
+		require.False(t, detail.NoDependencyRules)
+
+		rule.SetNoDependencyRules(true)
+		detail = NewRuleDetail(rule)
+		require.True(t, detail.NoDependentRules)
+		require.True(t, detail.NoDependencyRules)
+	})
+}

rules/recording.go

@@ -41,6 +41,9 @@ type RecordingRule struct {
 	lastError *atomic.Error
 	// Duration of how long it took to evaluate the recording rule.
 	evaluationDuration *atomic.Duration
+
+	noDependentRules  *atomic.Bool
+	noDependencyRules *atomic.Bool
 }
 
 // NewRecordingRule returns a new recording rule.
@@ -53,6 +56,8 @@ func NewRecordingRule(name string, vector parser.Expr, lset labels.Labels) *Reco
 		evaluationTimestamp: atomic.NewTime(time.Time{}),
 		evaluationDuration:  atomic.NewDuration(0),
 		lastError:           atomic.NewError(nil),
+		noDependentRules:    atomic.NewBool(false),
+		noDependencyRules:   atomic.NewBool(false),
 	}
 }
 
@@ -166,3 +171,19 @@ func (rule *RecordingRule) SetEvaluationTimestamp(ts time.Time) {
 func (rule *RecordingRule) GetEvaluationTimestamp() time.Time {
 	return rule.evaluationTimestamp.Load()
 }
+
+func (rule *RecordingRule) SetNoDependentRules(noDependentRules bool) {
+	rule.noDependentRules.Store(noDependentRules)
+}
+
+func (rule *RecordingRule) NoDependentRules() bool {
+	return rule.noDependentRules.Load()
+}
+
+func (rule *RecordingRule) SetNoDependencyRules(noDependencyRules bool) {
+	rule.noDependencyRules.Store(noDependencyRules)
+}
+
+func (rule *RecordingRule) NoDependencyRules() bool {
+	return rule.noDependencyRules.Load()
+}

rules/recording_test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/promql/parser"
 	"github.com/prometheus/prometheus/util/teststorage"
+	"github.com/prometheus/prometheus/util/testutil"
 )
 
 var (
@@ -126,7 +127,7 @@ func TestRuleEval(t *testing.T) {
 			rule := NewRecordingRule("test_rule", scenario.expr, scenario.ruleLabels)
 			result, err := rule.Eval(context.TODO(), ruleEvaluationTime, EngineQueryFunc(testEngine, storage), nil, 0)
 			require.NoError(t, err)
-			require.Equal(t, scenario.expected, result)
+			testutil.RequireEqual(t, scenario.expected, result)
 		})
 	}
 }
@@ -249,3 +250,25 @@ func TestRecordingEvalWithOrigin(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, detail, NewRuleDetail(rule))
 }
+
+func TestRecordingRule_SetNoDependentRules(t *testing.T) {
+	rule := NewRecordingRule("1", &parser.NumberLiteral{Val: 1}, labels.EmptyLabels())
+	require.False(t, rule.NoDependentRules())
+
+	rule.SetNoDependentRules(false)
+	require.False(t, rule.NoDependentRules())
+
+	rule.SetNoDependentRules(true)
+	require.True(t, rule.NoDependentRules())
+}
+
+func TestRecordingRule_SetNoDependencyRules(t *testing.T) {
+	rule := NewRecordingRule("1", &parser.NumberLiteral{Val: 1}, labels.EmptyLabels())
+	require.False(t, rule.NoDependencyRules())
+
+	rule.SetNoDependencyRules(false)
+	require.False(t, rule.NoDependencyRules())
+
+	rule.SetNoDependencyRules(true)
+	require.True(t, rule.NoDependencyRules())
+}

rules/rule.go

@@ -61,4 +61,20 @@ type Rule interface {
 	// GetEvaluationTimestamp returns last evaluation timestamp.
 	// NOTE: Used dynamically by rules.html template.
 	GetEvaluationTimestamp() time.Time
+
+	// SetNoDependentRules sets whether there's no other rule in the rule group that depends on this rule.
+	SetNoDependentRules(bool)
+
+	// NoDependentRules returns true if it's guaranteed that in the rule group there's no other rule
+	// which depends on this one. In case this function returns false there's no such guarantee, which
+	// means there may or may not be other rules depending on this one.
+	NoDependentRules() bool
+
+	// SetNoDependencyRules sets whether this rule doesn't depend on the output of any rule in the rule group.
+	SetNoDependencyRules(bool)
+
+	// NoDependencyRules returns true if it's guaranteed that this rule doesn't depend on the output of
+	// any other rule in the group. In case this function returns false there's no such guarantee, which
+	// means the rule may or may not depend on other rules.
+	NoDependencyRules() bool
 }

scrape/helpers_test.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"encoding/binary"
 	"fmt"
+	"math"
 	"math/rand"
 	"strings"
 	"sync"
@@ -71,6 +72,11 @@ type floatSample struct {
 	f      float64
 }
 
+func equalFloatSamples(a, b floatSample) bool {
+	// Compare Float64bits so NaN values which are exactly the same will compare equal.
+	return labels.Equal(a.metric, b.metric) && a.t == b.t && math.Float64bits(a.f) == math.Float64bits(b.f)
+}
+
 type histogramSample struct {
 	t  int64
 	h  *histogram.Histogram

scrape/manager_test.go

@@ -40,6 +40,7 @@ import (
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/relabel"
 	"github.com/prometheus/prometheus/util/runutil"
+	"github.com/prometheus/prometheus/util/testutil"
 )
 
 func TestPopulateLabels(t *testing.T) {
@@ -449,8 +450,8 @@ func TestPopulateLabels(t *testing.T) {
 			require.NoError(t, err)
 		}
 		require.Equal(t, c.in, in)
-		require.Equal(t, c.res, res)
+		testutil.RequireEqual(t, c.res, res)
-		require.Equal(t, c.resOrig, orig)
+		testutil.RequireEqual(t, c.resOrig, orig)
 	}
 }
 
@@ -458,9 +459,9 @@ func loadConfiguration(t testing.TB, c string) *config.Config {
 	t.Helper()
 
 	cfg := &config.Config{}
-	if err := yaml.UnmarshalStrict([]byte(c), cfg); err != nil {
+	err := yaml.UnmarshalStrict([]byte(c), cfg)
-		t.Fatalf("Unable to load YAML config: %s", err)
+	require.NoError(t, err, "Unable to load YAML config.")
-	}
+
 	return cfg
 }
 
@@ -533,42 +534,38 @@ scrape_configs:
 	}
 
 	// Apply the initial configuration.
-	if err := scrapeManager.ApplyConfig(cfg1); err != nil {
+	err = scrapeManager.ApplyConfig(cfg1)
-		t.Fatalf("unable to apply configuration: %s", err)
+	require.NoError(t, err, "Unable to apply configuration.")
-	}
 	select {
 	case <-ch:
-		t.Fatal("reload happened")
+		require.FailNow(t, "Reload happened.")
 	default:
 	}
 
 	// Apply a configuration for which the reload fails.
-	if err := scrapeManager.ApplyConfig(cfg2); err == nil {
+	err = scrapeManager.ApplyConfig(cfg2)
-		t.Fatalf("expecting error but got none")
+	require.Error(t, err, "Expecting error but got none.")
-	}
 	select {
 	case <-ch:
-		t.Fatal("reload happened")
+		require.FailNow(t, "Reload happened.")
 	default:
 	}
 
 	// Apply a configuration for which the reload succeeds.
-	if err := scrapeManager.ApplyConfig(cfg3); err != nil {
+	err = scrapeManager.ApplyConfig(cfg3)
-		t.Fatalf("unable to apply configuration: %s", err)
+	require.NoError(t, err, "Unable to apply configuration.")
-	}
 	select {
 	case <-ch:
 	default:
-		t.Fatal("reload didn't happen")
+		require.FailNow(t, "Reload didn't happen.")
 	}
 
 	// Re-applying the same configuration shouldn't trigger a reload.
-	if err := scrapeManager.ApplyConfig(cfg3); err != nil {
+	err = scrapeManager.ApplyConfig(cfg3)
-		t.Fatalf("unable to apply configuration: %s", err)
+	require.NoError(t, err, "Unable to apply configuration.")
-	}
 	select {
 	case <-ch:
-		t.Fatal("reload happened")
+		require.FailNow(t, "Reload happened.")
 	default:
 	}
 }
@@ -595,7 +592,7 @@ func TestManagerTargetsUpdates(t *testing.T) {
 		select {
 		case ts <- tgSent:
 		case <-time.After(10 * time.Millisecond):
-			t.Error("Scrape manager's channel remained blocked after the set threshold.")
+			require.Fail(t, "Scrape manager's channel remained blocked after the set threshold.")
 		}
 	}
 
@@ -609,7 +606,7 @@ func TestManagerTargetsUpdates(t *testing.T) {
 	select {
 	case <-m.triggerReload:
 	default:
-		t.Error("No scrape loops reload was triggered after targets update.")
+		require.Fail(t, "No scrape loops reload was triggered after targets update.")
 	}
 }
 
@@ -622,9 +619,8 @@ global:
 `
 
 		cfg := &config.Config{}
-		if err := yaml.UnmarshalStrict([]byte(cfgText), cfg); err != nil {
+		err := yaml.UnmarshalStrict([]byte(cfgText), cfg)
-			t.Fatalf("Unable to load YAML config cfgYaml: %s", err)
+		require.NoError(t, err, "Unable to load YAML config cfgYaml.")
-		}
 
 		return cfg
 	}
@@ -636,25 +632,18 @@ global:
 
 	// Load the first config.
 	cfg1 := getConfig("ha1")
-	if err := scrapeManager.setOffsetSeed(cfg1.GlobalConfig.ExternalLabels); err != nil {
+	err = scrapeManager.setOffsetSeed(cfg1.GlobalConfig.ExternalLabels)
-		t.Error(err)
+	require.NoError(t, err)
-	}
 	offsetSeed1 := scrapeManager.offsetSeed
 
-	if offsetSeed1 == 0 {
+	require.NotZero(t, offsetSeed1, "Offset seed has to be a hash of uint64.")
-		t.Error("Offset seed has to be a hash of uint64")
-	}
 
 	// Load the first config.
 	cfg2 := getConfig("ha2")
-	if err := scrapeManager.setOffsetSeed(cfg2.GlobalConfig.ExternalLabels); err != nil {
+	require.NoError(t, scrapeManager.setOffsetSeed(cfg2.GlobalConfig.ExternalLabels))
-		t.Error(err)
-	}
 	offsetSeed2 := scrapeManager.offsetSeed
 
-	if offsetSeed1 == offsetSeed2 {
+	require.NotEqual(t, offsetSeed1, offsetSeed2, "Offset seed should not be the same on different set of external labels.")
-		t.Error("Offset seed should not be the same on different set of external labels")
-	}
 }
 
 func TestManagerScrapePools(t *testing.T) {

scrape/scrape_test.go

@@ -32,6 +32,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/gogo/protobuf/proto"
+	"github.com/google/go-cmp/cmp"
 	"github.com/prometheus/client_golang/prometheus"
 	dto "github.com/prometheus/client_model/go"
 	config_util "github.com/prometheus/common/config"
@@ -72,15 +73,11 @@ func TestNewScrapePool(t *testing.T) {
 		sp, _ = newScrapePool(cfg, app, 0, nil, nil, &Options{}, newTestScrapeMetrics(t))
 	)
 
-	if a, ok := sp.appendable.(*nopAppendable); !ok || a != app {
+	a, ok := sp.appendable.(*nopAppendable)
-		t.Fatalf("Wrong sample appender")
+	require.True(t, ok, "Failure to append.")
-	}
+	require.Equal(t, app, a, "Wrong sample appender.")
-	if sp.config != cfg {
+	require.Equal(t, cfg, sp.config, "Wrong scrape config.")
-		t.Fatalf("Wrong scrape config")
+	require.NotNil(t, sp.newLoop, "newLoop function not initialized.")
-	}
-	if sp.newLoop == nil {
-		t.Fatalf("newLoop function not initialized")
-	}
 }
 
 func TestDroppedTargetsList(t *testing.T) {
@@ -233,12 +230,10 @@ func TestScrapePoolStop(t *testing.T) {
 
 	select {
 	case <-time.After(5 * time.Second):
-		t.Fatalf("scrapeLoop.stop() did not return as expected")
+		require.Fail(t, "scrapeLoop.stop() did not return as expected")
 	case <-done:
 		// This should have taken at least as long as the last target slept.
-		if time.Since(stopTime) < time.Duration(numTargets*20)*time.Millisecond {
+		require.GreaterOrEqual(t, time.Since(stopTime), time.Duration(numTargets*20)*time.Millisecond, "scrapeLoop.stop() exited before all targets stopped")
-			t.Fatalf("scrapeLoop.stop() exited before all targets stopped")
-		}
 	}
 
 	mtx.Lock()
@@ -324,12 +319,10 @@ func TestScrapePoolReload(t *testing.T) {
 
 	select {
 	case <-time.After(5 * time.Second):
-		t.Fatalf("scrapeLoop.reload() did not return as expected")
+		require.FailNow(t, "scrapeLoop.reload() did not return as expected")
 	case <-done:
 		// This should have taken at least as long as the last target slept.
-		if time.Since(reloadTime) < time.Duration(numTargets*20)*time.Millisecond {
+		require.GreaterOrEqual(t, time.Since(reloadTime), time.Duration(numTargets*20)*time.Millisecond, "scrapeLoop.stop() exited before all targets stopped")
-			t.Fatalf("scrapeLoop.stop() exited before all targets stopped")
-		}
 	}
 
 	mtx.Lock()
@@ -703,13 +696,13 @@ func TestScrapeLoopStopBeforeRun(t *testing.T) {
 
 	select {
 	case <-stopDone:
-		t.Fatalf("Stopping terminated before run exited successfully")
+		require.FailNow(t, "Stopping terminated before run exited successfully.")
 	case <-time.After(500 * time.Millisecond):
 	}
 
 	// Running the scrape loop must exit before calling the scraper even once.
 	scraper.scrapeFunc = func(context.Context, io.Writer) error {
-		t.Fatalf("scraper was called for terminated scrape loop")
+		require.FailNow(t, "Scraper was called for terminated scrape loop.")
 		return nil
 	}
 
@@ -722,13 +715,13 @@ func TestScrapeLoopStopBeforeRun(t *testing.T) {
 	select {
 	case <-runDone:
 	case <-time.After(1 * time.Second):
-		t.Fatalf("Running terminated scrape loop did not exit")
+		require.FailNow(t, "Running terminated scrape loop did not exit.")
 	}
 
 	select {
 	case <-stopDone:
 	case <-time.After(1 * time.Second):
-		t.Fatalf("Stopping did not terminate after running exited")
+		require.FailNow(t, "Stopping did not terminate after running exited.")
 	}
 }
 
@@ -765,14 +758,13 @@ func TestScrapeLoopStop(t *testing.T) {
 	select {
 	case <-signal:
 	case <-time.After(5 * time.Second):
-		t.Fatalf("Scrape wasn't stopped.")
+		require.FailNow(t, "Scrape wasn't stopped.")
 	}
 
 	// We expected 1 actual sample for each scrape plus 5 for report samples.
 	// At least 2 scrapes were made, plus the final stale markers.
-	if len(appender.resultFloats) < 6*3 || len(appender.resultFloats)%6 != 0 {
+	require.GreaterOrEqual(t, len(appender.resultFloats), 6*3, "Expected at least 3 scrapes with 6 samples each.")
-		t.Fatalf("Expected at least 3 scrapes with 6 samples each, got %d samples", len(appender.resultFloats))
+	require.Zero(t, len(appender.resultFloats)%6, "There is a scrape with missing samples.")
-	}
 	// All samples in a scrape must have the same timestamp.
 	var ts int64
 	for i, s := range appender.resultFloats {
@@ -785,9 +777,7 @@ func TestScrapeLoopStop(t *testing.T) {
 	}
 	// All samples from the last scrape must be stale markers.
 	for _, s := range appender.resultFloats[len(appender.resultFloats)-5:] {
-		if !value.IsStaleNaN(s.f) {
+		require.True(t, value.IsStaleNaN(s.f), "Appended last sample not as expected. Wanted: stale NaN Got: %x", math.Float64bits(s.f))
-			t.Fatalf("Appended last sample not as expected. Wanted: stale NaN Got: %x", math.Float64bits(s.f))
-		}
 	}
 }
 
@@ -843,9 +833,9 @@ func TestScrapeLoopRun(t *testing.T) {
 	select {
 	case <-signal:
 	case <-time.After(5 * time.Second):
-		t.Fatalf("Cancellation during initial offset failed")
+		require.FailNow(t, "Cancellation during initial offset failed.")
 	case err := <-errc:
-		t.Fatalf("Unexpected error: %s", err)
+		require.FailNow(t, "Unexpected error: %s", err)
 	}
 
 	// The provided timeout must cause cancellation of the context passed down to the
@@ -873,11 +863,9 @@ func TestScrapeLoopRun(t *testing.T) {
 
 	select {
 	case err := <-errc:
-		if !errors.Is(err, context.DeadlineExceeded) {
+		require.ErrorIs(t, err, context.DeadlineExceeded)
-			t.Fatalf("Expected timeout error but got: %s", err)
-		}
 	case <-time.After(3 * time.Second):
-		t.Fatalf("Expected timeout error but got none")
+		require.FailNow(t, "Expected timeout error but got none.")
 	}
 
 	// We already caught the timeout error and are certainly in the loop.
@@ -890,9 +878,9 @@ func TestScrapeLoopRun(t *testing.T) {
 	case <-signal:
 		// Loop terminated as expected.
 	case err := <-errc:
-		t.Fatalf("Unexpected error: %s", err)
+		require.FailNow(t, "Unexpected error: %s", err)
 	case <-time.After(3 * time.Second):
-		t.Fatalf("Loop did not terminate on context cancellation")
+		require.FailNow(t, "Loop did not terminate on context cancellation")
 	}
 }
 
@@ -912,7 +900,7 @@ func TestScrapeLoopForcedErr(t *testing.T) {
 	sl.setForcedError(forcedErr)
 
 	scraper.scrapeFunc = func(context.Context, io.Writer) error {
-		t.Fatalf("should not be scraped")
+		require.FailNow(t, "Should not be scraped.")
 		return nil
 	}
 
@@ -923,18 +911,16 @@ func TestScrapeLoopForcedErr(t *testing.T) {
 
 	select {
 	case err := <-errc:
-		if !errors.Is(err, forcedErr) {
+		require.ErrorIs(t, err, forcedErr)
-			t.Fatalf("Expected forced error but got: %s", err)
-		}
 	case <-time.After(3 * time.Second):
-		t.Fatalf("Expected forced error but got none")
+		require.FailNow(t, "Expected forced error but got none.")
 	}
 	cancel()
 
 	select {
 	case <-signal:
 	case <-time.After(5 * time.Second):
-		t.Fatalf("Scrape not stopped")
+		require.FailNow(t, "Scrape not stopped.")
 	}
 }
 
@@ -1141,7 +1127,7 @@ func TestScrapeLoopRunCreatesStaleMarkersOnFailedScrape(t *testing.T) {
 	select {
 	case <-signal:
 	case <-time.After(5 * time.Second):
-		t.Fatalf("Scrape wasn't stopped.")
+		require.FailNow(t, "Scrape wasn't stopped.")
 	}
 
 	// 1 successfully scraped sample, 1 stale marker after first fail, 5 report samples for
@@ -1188,7 +1174,7 @@ func TestScrapeLoopRunCreatesStaleMarkersOnParseFailure(t *testing.T) {
 	select {
 	case <-signal:
 	case <-time.After(5 * time.Second):
-		t.Fatalf("Scrape wasn't stopped.")
+		require.FailNow(t, "Scrape wasn't stopped.")
 	}
 
 	// 1 successfully scraped sample, 1 stale marker after first fail, 5 report samples for
@@ -1220,19 +1206,15 @@ func TestScrapeLoopCache(t *testing.T) {
 	scraper.scrapeFunc = func(ctx context.Context, w io.Writer) error {
 		switch numScrapes {
 		case 1, 2:
-			if _, ok := sl.cache.series["metric_a"]; !ok {
+			_, ok := sl.cache.series["metric_a"]
-				t.Errorf("metric_a missing from cache after scrape %d", numScrapes)
+			require.True(t, ok, "metric_a missing from cache after scrape %d", numScrapes)
-			}
+			_, ok = sl.cache.series["metric_b"]
-			if _, ok := sl.cache.series["metric_b"]; !ok {
+			require.True(t, ok, "metric_b missing from cache after scrape %d", numScrapes)
-				t.Errorf("metric_b missing from cache after scrape %d", numScrapes)
-			}
 		case 3:
-			if _, ok := sl.cache.series["metric_a"]; !ok {
+			_, ok := sl.cache.series["metric_a"]
-				t.Errorf("metric_a missing from cache after scrape %d", numScrapes)
+			require.True(t, ok, "metric_a missing from cache after scrape %d", numScrapes)
-			}
+			_, ok = sl.cache.series["metric_b"]
-			if _, ok := sl.cache.series["metric_b"]; ok {
+			require.False(t, ok, "metric_b present in cache after scrape %d", numScrapes)
-				t.Errorf("metric_b present in cache after scrape %d", numScrapes)
-			}
 		}
 
 		numScrapes++
@@ -1257,7 +1239,7 @@ func TestScrapeLoopCache(t *testing.T) {
 	select {
 	case <-signal:
 	case <-time.After(5 * time.Second):
-		t.Fatalf("Scrape wasn't stopped.")
+		require.FailNow(t, "Scrape wasn't stopped.")
 	}
 
 	// 1 successfully scraped sample, 1 stale marker after first fail, 5 report samples for
@@ -1305,12 +1287,10 @@ func TestScrapeLoopCacheMemoryExhaustionProtection(t *testing.T) {
 	select {
 	case <-signal:
 	case <-time.After(5 * time.Second):
-		t.Fatalf("Scrape wasn't stopped.")
+		require.FailNow(t, "Scrape wasn't stopped.")
 	}
 
-	if len(sl.cache.series) > 2000 {
+	require.LessOrEqual(t, len(sl.cache.series), 2000, "More than 2000 series cached.")
-		t.Fatalf("More than 2000 series cached. Got: %d", len(sl.cache.series))
-	}
 }
 
 func TestScrapeLoopAppend(t *testing.T) {
@@ -1352,7 +1332,7 @@ func TestScrapeLoopAppend(t *testing.T) {
 			// Honor Labels should ignore labels with the same name.
 			title:           "Honor Labels",
 			honorLabels:     true,
-			scrapeLabels:    `metric{n1="1" n2="2"} 0`,
+			scrapeLabels:    `metric{n1="1", n2="2"} 0`,
 			discoveryLabels: []string{"n1", "0"},
 			expLset:         labels.FromStrings("__name__", "metric", "n1", "1", "n2", "2"),
 			expValue:        0,
@@ -1362,7 +1342,7 @@ func TestScrapeLoopAppend(t *testing.T) {
 			scrapeLabels:    `metric NaN`,
 			discoveryLabels: nil,
 			expLset:         labels.FromStrings("__name__", "metric"),
-			expValue:        float64(value.NormalNaN),
+			expValue:        math.Float64frombits(value.NormalNaN),
 		},
 	}
 
@@ -1396,18 +1376,17 @@ func TestScrapeLoopAppend(t *testing.T) {
 			},
 		}
 
-		// When the expected value is NaN
-		// DeepEqual will report NaNs as being different,
-		// so replace it with the expected one.
-		if test.expValue == float64(value.NormalNaN) {
-			app.resultFloats[0].f = expected[0].f
-		}
-
 		t.Logf("Test:%s", test.title)
-		require.Equal(t, expected, app.resultFloats)
+		requireEqual(t, expected, app.resultFloats)
 	}
 }
 
+func requireEqual(t *testing.T, expected, actual interface{}, msgAndArgs ...interface{}) {
+	testutil.RequireEqualWithOptions(t, expected, actual,
+		[]cmp.Option{cmp.Comparer(equalFloatSamples), cmp.AllowUnexported(histogramSample{})},
+		msgAndArgs...)
+}
+
 func TestScrapeLoopAppendForConflictingPrefixedLabels(t *testing.T) {
 	testcases := map[string]struct {
 		targetLabels  []string
@@ -1427,7 +1406,7 @@ func TestScrapeLoopAppendForConflictingPrefixedLabels(t *testing.T) {
 		},
 		"One target label collides with existing label, plus existing label already with prefix 'exported": {
 			targetLabels:  []string{"foo", "3"},
-			exposedLabels: `metric{foo="1" exported_foo="2"} 0`,
+			exposedLabels: `metric{foo="1", exported_foo="2"} 0`,
 			expected:      []string{"__name__", "metric", "exported_exported_foo", "1", "exported_foo", "2", "foo", "3"},
 		},
 		"One target label collides with existing label, both already with prefix 'exported'": {
@@ -1437,7 +1416,7 @@ func TestScrapeLoopAppendForConflictingPrefixedLabels(t *testing.T) {
 		},
 		"Two target labels collide with existing labels, both with and without prefix 'exported'": {
 			targetLabels:  []string{"foo", "3", "exported_foo", "4"},
-			exposedLabels: `metric{foo="1" exported_foo="2"} 0`,
+			exposedLabels: `metric{foo="1", exported_foo="2"} 0`,
 			expected: []string{
 				"__name__", "metric", "exported_exported_foo", "1", "exported_exported_exported_foo",
 				"2", "exported_foo", "4", "foo", "3",
@@ -1445,7 +1424,7 @@ func TestScrapeLoopAppendForConflictingPrefixedLabels(t *testing.T) {
 		},
 		"Extreme example": {
 			targetLabels:  []string{"foo", "0", "exported_exported_foo", "1", "exported_exported_exported_foo", "2"},
-			exposedLabels: `metric{foo="3" exported_foo="4" exported_exported_exported_foo="5"} 0`,
+			exposedLabels: `metric{foo="3", exported_foo="4", exported_exported_exported_foo="5"} 0`,
 			expected: []string{
 				"__name__", "metric",
 				"exported_exported_exported_exported_exported_foo", "5",
@@ -1471,7 +1450,7 @@ func TestScrapeLoopAppendForConflictingPrefixedLabels(t *testing.T) {
 
 			require.NoError(t, slApp.Commit())
 
-			require.Equal(t, []floatSample{
+			requireEqual(t, []floatSample{
 				{
 					metric: labels.FromStrings(tc.expected...),
 					t:      timestamp.FromTime(time.Date(2000, 1, 1, 1, 0, 0, 0, time.UTC)),
@@ -1541,9 +1520,7 @@ func TestScrapeLoopAppendSampleLimit(t *testing.T) {
 	now := time.Now()
 	slApp := sl.appender(context.Background())
 	total, added, seriesAdded, err := sl.append(app, []byte("metric_a 1\nmetric_b 1\nmetric_c 1\n"), "", now)
-	if !errors.Is(err, errSampleLimit) {
+	require.ErrorIs(t, err, errSampleLimit)
-		t.Fatalf("Did not see expected sample limit error: %s", err)
-	}
 	require.NoError(t, slApp.Rollback())
 	require.Equal(t, 3, total)
 	require.Equal(t, 3, added)
@@ -1567,14 +1544,12 @@ func TestScrapeLoopAppendSampleLimit(t *testing.T) {
 			f:      1,
 		},
 	}
-	require.Equal(t, want, resApp.rolledbackFloats, "Appended samples not as expected:\n%s", appender)
+	requireEqual(t, want, resApp.rolledbackFloats, "Appended samples not as expected:\n%s", appender)
 
 	now = time.Now()
 	slApp = sl.appender(context.Background())
 	total, added, seriesAdded, err = sl.append(slApp, []byte("metric_a 1\nmetric_b 1\nmetric_c{deleteme=\"yes\"} 1\nmetric_d 1\nmetric_e 1\nmetric_f 1\nmetric_g 1\nmetric_h{deleteme=\"yes\"} 1\nmetric_i{deleteme=\"yes\"} 1\n"), "", now)
-	if !errors.Is(err, errSampleLimit) {
+	require.ErrorIs(t, err, errSampleLimit)
-		t.Fatalf("Did not see expected sample limit error: %s", err)
-	}
 	require.NoError(t, slApp.Rollback())
 	require.Equal(t, 9, total)
 	require.Equal(t, 6, added)
@@ -1709,7 +1684,6 @@ func TestScrapeLoop_ChangingMetricString(t *testing.T) {
 	require.NoError(t, err)
 	require.NoError(t, slApp.Commit())
 
-	// DeepEqual will report NaNs as being different, so replace with a different value.
 	want := []floatSample{
 		{
 			metric: labels.FromStrings("__name__", "metric_a", "a", "1", "b", "1"),
@@ -1741,11 +1715,6 @@ func TestScrapeLoopAppendStaleness(t *testing.T) {
 	require.NoError(t, err)
 	require.NoError(t, slApp.Commit())
 
-	ingestedNaN := math.Float64bits(app.resultFloats[1].f)
-	require.Equal(t, value.StaleNaN, ingestedNaN, "Appended stale sample wasn't as expected")
-
-	// DeepEqual will report NaNs as being different, so replace with a different value.
-	app.resultFloats[1].f = 42
 	want := []floatSample{
 		{
 			metric: labels.FromStrings(model.MetricNameLabel, "metric_a"),
@@ -1755,10 +1724,10 @@ func TestScrapeLoopAppendStaleness(t *testing.T) {
 		{
 			metric: labels.FromStrings(model.MetricNameLabel, "metric_a"),
 			t:      timestamp.FromTime(now.Add(time.Second)),
-			f:      42,
+			f:      math.Float64frombits(value.StaleNaN),
 		},
 	}
-	require.Equal(t, want, app.resultFloats, "Appended samples not as expected:\n%s", appender)
+	requireEqual(t, want, app.resultFloats, "Appended samples not as expected:\n%s", appender)
 }
 
 func TestScrapeLoopAppendNoStalenessIfTimestamp(t *testing.T) {
@@ -1801,8 +1770,6 @@ func TestScrapeLoopAppendStalenessIfTrackTimestampStaleness(t *testing.T) {
 	require.NoError(t, err)
 	require.NoError(t, slApp.Commit())
 
-	// DeepEqual will report NaNs as being different, so replace with a different value.
-	app.resultFloats[1].f = 42
 	want := []floatSample{
 		{
 			metric: labels.FromStrings(model.MetricNameLabel, "metric_a"),
@@ -1812,10 +1779,10 @@ func TestScrapeLoopAppendStalenessIfTrackTimestampStaleness(t *testing.T) {
 		{
 			metric: labels.FromStrings(model.MetricNameLabel, "metric_a"),
 			t:      timestamp.FromTime(now.Add(time.Second)),
-			f:      42,
+			f:      math.Float64frombits(value.StaleNaN),
 		},
 	}
-	require.Equal(t, want, app.resultFloats, "Appended samples not as expected:\n%s", appender)
+	requireEqual(t, want, app.resultFloats, "Appended samples not as expected:\n%s", appender)
 }
 
 func TestScrapeLoopAppendExemplar(t *testing.T) {
@@ -2183,9 +2150,9 @@ metric: <
 			_, _, _, err := sl.append(app, buf.Bytes(), test.contentType, now)
 			require.NoError(t, err)
 			require.NoError(t, app.Commit())
-			require.Equal(t, test.floats, app.resultFloats)
+			requireEqual(t, test.floats, app.resultFloats)
-			require.Equal(t, test.histograms, app.resultHistograms)
+			requireEqual(t, test.histograms, app.resultHistograms)
-			require.Equal(t, test.exemplars, app.resultExemplars)
+			requireEqual(t, test.exemplars, app.resultExemplars)
 		})
 	}
 }
@@ -2240,8 +2207,8 @@ func TestScrapeLoopAppendExemplarSeries(t *testing.T) {
 		require.NoError(t, app.Commit())
 	}
 
-	require.Equal(t, samples, app.resultFloats)
+	requireEqual(t, samples, app.resultFloats)
-	require.Equal(t, exemplars, app.resultExemplars)
+	requireEqual(t, exemplars, app.resultExemplars)
 }
 
 func TestScrapeLoopRunReportsTargetDownOnScrapeError(t *testing.T) {
@@ -2317,7 +2284,7 @@ func TestScrapeLoopAppendGracefullyIfAmendOrOutOfOrderOrOutOfBounds(t *testing.T
 			f:      1,
 		},
 	}
-	require.Equal(t, want, app.resultFloats, "Appended samples not as expected:\n%s", appender)
+	requireEqual(t, want, app.resultFloats, "Appended samples not as expected:\n%s", appender)
 	require.Equal(t, 4, total)
 	require.Equal(t, 4, added)
 	require.Equal(t, 1, seriesAdded)
@@ -2357,15 +2324,12 @@ func TestTargetScraperScrapeOK(t *testing.T) {
 		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			if protobufParsing {
 				accept := r.Header.Get("Accept")
-				if !strings.HasPrefix(accept, "application/vnd.google.protobuf;") {
+				require.True(t, strings.HasPrefix(accept, "application/vnd.google.protobuf;"),
-					t.Errorf("Expected Accept header to prefer application/vnd.google.protobuf, got %q", accept)
+					"Expected Accept header to prefer application/vnd.google.protobuf.")
-				}
 			}
 
 			timeout := r.Header.Get("X-Prometheus-Scrape-Timeout-Seconds")
-			if timeout != expectedTimeout {
+			require.Equal(t, expectedTimeout, timeout, "Expected scrape timeout header.")
-				t.Errorf("Expected scrape timeout header %q, got %q", expectedTimeout, timeout)
-			}
 
 			w.Header().Set("Content-Type", `text/plain; version=0.0.4`)
 			w.Write([]byte("metric_a 1\nmetric_b 2\n"))
@@ -2453,7 +2417,7 @@ func TestTargetScrapeScrapeCancel(t *testing.T) {
 
 	select {
 	case <-time.After(5 * time.Second):
-		t.Fatalf("Scrape function did not return unexpectedly")
+		require.FailNow(t, "Scrape function did not return unexpectedly.")
 	case err := <-errc:
 		require.NoError(t, err)
 	}
@@ -3053,7 +3017,7 @@ func TestScrapeReportSingleAppender(t *testing.T) {
 	select {
 	case <-signal:
 	case <-time.After(5 * time.Second):
-		t.Fatalf("Scrape wasn't stopped.")
+		require.FailNow(t, "Scrape wasn't stopped.")
 	}
 }
 

scrape/target_test.go

@@ -77,9 +77,7 @@ func TestTargetOffset(t *testing.T) {
 	buckets := make([]int, interval/bucketSize)
 
 	for _, offset := range offsets {
-		if offset < 0 || offset >= interval {
+		require.InDelta(t, time.Duration(0), offset, float64(interval), "Offset %v out of bounds.", offset)
-			t.Fatalf("Offset %v out of bounds", offset)
-		}
 
 		bucket := offset / bucketSize
 		buckets[bucket]++
@@ -98,9 +96,7 @@ func TestTargetOffset(t *testing.T) {
 			diff = -diff
 		}
 
-		if float64(diff)/float64(avg) > tolerance {
+		require.LessOrEqual(t, float64(diff)/float64(avg), tolerance, "Bucket out of tolerance bounds.")
-			t.Fatalf("Bucket out of tolerance bounds")
-		}
 	}
 }
 
@@ -150,9 +146,7 @@ func TestNewHTTPBearerToken(t *testing.T) {
 			func(w http.ResponseWriter, r *http.Request) {
 				expected := "Bearer 1234"
 				received := r.Header.Get("Authorization")
-				if expected != received {
+				require.Equal(t, expected, received, "Authorization header was not set correctly.")
-					t.Fatalf("Authorization header was not set correctly: expected '%v', got '%v'", expected, received)
-				}
 			},
 		),
 	)
@@ -162,13 +156,9 @@ func TestNewHTTPBearerToken(t *testing.T) {
 		BearerToken: "1234",
 	}
 	c, err := config_util.NewClientFromConfig(cfg, "test")
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
-	}
 	_, err = c.Get(server.URL)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
-	}
 }
 
 func TestNewHTTPBearerTokenFile(t *testing.T) {
@@ -177,9 +167,7 @@ func TestNewHTTPBearerTokenFile(t *testing.T) {
 			func(w http.ResponseWriter, r *http.Request) {
 				expected := "Bearer 12345"
 				received := r.Header.Get("Authorization")
-				if expected != received {
+				require.Equal(t, expected, received, "Authorization header was not set correctly.")
-					t.Fatalf("Authorization header was not set correctly: expected '%v', got '%v'", expected, received)
-				}
 			},
 		),
 	)
@@ -189,13 +177,9 @@ func TestNewHTTPBearerTokenFile(t *testing.T) {
 		BearerTokenFile: "testdata/bearertoken.txt",
 	}
 	c, err := config_util.NewClientFromConfig(cfg, "test")
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
-	}
 	_, err = c.Get(server.URL)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
-	}
 }
 
 func TestNewHTTPBasicAuth(t *testing.T) {
@@ -203,9 +187,9 @@ func TestNewHTTPBasicAuth(t *testing.T) {
 		http.HandlerFunc(
 			func(w http.ResponseWriter, r *http.Request) {
 				username, password, ok := r.BasicAuth()
-				if !(ok && username == "user" && password == "password123") {
+				require.True(t, ok, "Basic authorization header was not set correctly.")
-					t.Fatalf("Basic authorization header was not set correctly: expected '%v:%v', got '%v:%v'", "user", "password123", username, password)
+				require.Equal(t, "user", username)
-				}
+				require.Equal(t, "password123", password)
 			},
 		),
 	)
@@ -218,13 +202,9 @@ func TestNewHTTPBasicAuth(t *testing.T) {
 		},
 	}
 	c, err := config_util.NewClientFromConfig(cfg, "test")
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
-	}
 	_, err = c.Get(server.URL)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
-	}
 }
 
 func TestNewHTTPCACert(t *testing.T) {
@@ -246,13 +226,9 @@ func TestNewHTTPCACert(t *testing.T) {
 		},
 	}
 	c, err := config_util.NewClientFromConfig(cfg, "test")
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
-	}
 	_, err = c.Get(server.URL)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
-	}
 }
 
 func TestNewHTTPClientCert(t *testing.T) {
@@ -279,13 +255,9 @@ func TestNewHTTPClientCert(t *testing.T) {
 		},
 	}
 	c, err := config_util.NewClientFromConfig(cfg, "test")
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
-	}
 	_, err = c.Get(server.URL)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
-	}
 }
 
 func TestNewHTTPWithServerName(t *testing.T) {
@@ -308,13 +280,9 @@ func TestNewHTTPWithServerName(t *testing.T) {
 		},
 	}
 	c, err := config_util.NewClientFromConfig(cfg, "test")
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
-	}
 	_, err = c.Get(server.URL)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
-	}
 }
 
 func TestNewHTTPWithBadServerName(t *testing.T) {
@@ -337,31 +305,23 @@ func TestNewHTTPWithBadServerName(t *testing.T) {
 		},
 	}
 	c, err := config_util.NewClientFromConfig(cfg, "test")
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
-	}
 	_, err = c.Get(server.URL)
-	if err == nil {
+	require.Error(t, err)
-		t.Fatal("Expected error, got nil.")
-	}
 }
 
 func newTLSConfig(certName string, t *testing.T) *tls.Config {
 	tlsConfig := &tls.Config{}
 	caCertPool := x509.NewCertPool()
 	caCert, err := os.ReadFile(caCertPath)
-	if err != nil {
+	require.NoError(t, err, "Couldn't read CA cert.")
-		t.Fatalf("Couldn't set up TLS server: %v", err)
-	}
 	caCertPool.AppendCertsFromPEM(caCert)
 	tlsConfig.RootCAs = caCertPool
 	tlsConfig.ServerName = "127.0.0.1"
 	certPath := fmt.Sprintf("testdata/%s.cer", certName)
 	keyPath := fmt.Sprintf("testdata/%s.key", certName)
 	cert, err := tls.LoadX509KeyPair(certPath, keyPath)
-	if err != nil {
+	require.NoError(t, err, "Unable to use specified server cert (%s) & key (%v).", certPath, keyPath)
-		t.Errorf("Unable to use specified server cert (%s) & key (%v): %s", certPath, keyPath, err)
-	}
 	tlsConfig.Certificates = []tls.Certificate{cert}
 	return tlsConfig
 }
@@ -375,9 +335,7 @@ func TestNewClientWithBadTLSConfig(t *testing.T) {
 		},
 	}
 	_, err := config_util.NewClientFromConfig(cfg, "test")
-	if err == nil {
+	require.Error(t, err)
-		t.Fatalf("Expected error, got nil.")
-	}
 }
 
 func TestTargetsFromGroup(t *testing.T) {
@@ -389,15 +347,9 @@ func TestTargetsFromGroup(t *testing.T) {
 	}
 	lb := labels.NewBuilder(labels.EmptyLabels())
 	targets, failures := TargetsFromGroup(&targetgroup.Group{Targets: []model.LabelSet{{}, {model.AddressLabel: "localhost:9090"}}}, &cfg, false, nil, lb)
-	if len(targets) != 1 {
+	require.Len(t, targets, 1)
-		t.Fatalf("Expected 1 target, got %v", len(targets))
+	require.Len(t, failures, 1)
-	}
+	require.EqualError(t, failures[0], expectedError)
-	if len(failures) != 1 {
-		t.Fatalf("Expected 1 failure, got %v", len(failures))
-	}
-	if failures[0].Error() != expectedError {
-		t.Fatalf("Expected error %s, got %s", expectedError, failures[0])
-	}
 }
 
 func BenchmarkTargetsFromGroup(b *testing.B) {

scripts/genproto.sh

@@ -46,11 +46,11 @@ for dir in ${DIRS}; do
 		protoc --gogofast_out=Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types,paths=source_relative:. -I=. \
             -I="${GOGOPROTO_PATH}" \
             ./io/prometheus/client/*.proto
-		sed -i.bak -E 's/import _ \"github.com\/gogo\/protobuf\/gogoproto\"//g' -- *.pb.go
+		sed -i.bak -E 's/import _ \"github.com\/gogo\/protobuf\/gogoproto\"//g' *.pb.go
-		sed -i.bak -E 's/import _ \"google\/protobuf\"//g' -- *.pb.go
+		sed -i.bak -E 's/import _ \"google\/protobuf\"//g' *.pb.go
-		sed -i.bak -E 's/\t_ \"google\/protobuf\"//g' -- *.pb.go
+		sed -i.bak -E 's/\t_ \"google\/protobuf\"//g' *.pb.go
-		sed -i.bak -E 's/golang\/protobuf\/descriptor/gogo\/protobuf\/protoc-gen-gogo\/descriptor/g' -- *.go
+		sed -i.bak -E 's/golang\/protobuf\/descriptor/gogo\/protobuf\/protoc-gen-gogo\/descriptor/g' *.go
-		sed -i.bak -E 's/golang\/protobuf/gogo\/protobuf/g' -- *.go
+		sed -i.bak -E 's/golang\/protobuf/gogo\/protobuf/g' *.go
 		rm -f -- *.bak
 		goimports -w ./*.go ./io/prometheus/client/*.go
 	popd

storage/remote/client.go

@@ -146,24 +146,24 @@ func NewWriteClient(name string, conf *ClientConfig) (WriteClient, error) {
 	}
 	t := httpClient.Transport
 
+	if len(conf.Headers) > 0 {
+		t = newInjectHeadersRoundTripper(conf.Headers, t)
+	}
+
 	if conf.SigV4Config != nil {
-		t, err = sigv4.NewSigV4RoundTripper(conf.SigV4Config, httpClient.Transport)
+		t, err = sigv4.NewSigV4RoundTripper(conf.SigV4Config, t)
 		if err != nil {
 			return nil, err
 		}
 	}
 
 	if conf.AzureADConfig != nil {
-		t, err = azuread.NewAzureADRoundTripper(conf.AzureADConfig, httpClient.Transport)
+		t, err = azuread.NewAzureADRoundTripper(conf.AzureADConfig, t)
 		if err != nil {
 			return nil, err
 		}
 	}
 
-	if len(conf.Headers) > 0 {
-		t = newInjectHeadersRoundTripper(conf.Headers, t)
-	}
-
 	httpClient.Transport = otelhttp.NewTransport(t)
 
 	return &Client{

storage/remote/otlptranslator/prometheus/normalize_label.go

@@ -27,6 +27,7 @@ var dropSanitizationGate = featuregate.GlobalRegistry().MustRegister(
 //
 // Exception is made for double-underscores which are allowed
 func NormalizeLabel(label string) string {
+
 	// Trivial case
 	if len(label) == 0 {
 		return label

storage/remote/queue_manager.go

@@ -422,10 +422,11 @@ type QueueManager struct {
 	clientMtx   sync.RWMutex
 	storeClient WriteClient
 
-	seriesMtx      sync.Mutex // Covers seriesLabels, seriesMetadata and droppedSeries.
+	seriesMtx      sync.Mutex // Covers seriesLabels, seriesMetadata, droppedSeries and builder.
 	seriesLabels   map[chunks.HeadSeriesRef]labels.Labels
 	seriesMetadata map[chunks.HeadSeriesRef]*metadata.Metadata
 	droppedSeries  map[chunks.HeadSeriesRef]struct{}
+	builder        *labels.Builder
 
 	seriesSegmentMtx     sync.Mutex // Covers seriesSegmentIndexes - if you also lock seriesMtx, take seriesMtx first.
 	seriesSegmentIndexes map[chunks.HeadSeriesRef]int
@@ -497,6 +498,7 @@ func NewQueueManager(
 		seriesMetadata:       make(map[chunks.HeadSeriesRef]*metadata.Metadata),
 		seriesSegmentIndexes: make(map[chunks.HeadSeriesRef]int),
 		droppedSeries:        make(map[chunks.HeadSeriesRef]struct{}),
+		builder:              labels.NewBuilder(labels.EmptyLabels()),
 
 		numShards:   cfg.MinShards,
 		reshardChan: make(chan int),
@@ -971,12 +973,14 @@ func (t *QueueManager) StoreSeries(series []record.RefSeries, index int) {
 		// Just make sure all the Refs of Series will insert into seriesSegmentIndexes map for tracking.
 		t.seriesSegmentIndexes[s.Ref] = index
 
-		ls := processExternalLabels(s.Labels, t.externalLabels)
+		t.builder.Reset(s.Labels)
-		lbls, keep := relabel.Process(ls, t.relabelConfigs...)
+		processExternalLabels(t.builder, t.externalLabels)
-		if !keep || lbls.IsEmpty() {
+		keep := relabel.ProcessBuilder(t.builder, t.relabelConfigs...)
+		if !keep {
 			t.droppedSeries[s.Ref] = struct{}{}
 			continue
 		}
+		lbls := t.builder.Labels()
 		t.internLabels(lbls)
 
 		// We should not ever be replacing a series labels in the map, but just
@@ -1059,30 +1063,14 @@ func (t *QueueManager) releaseLabels(ls labels.Labels) {
 	ls.ReleaseStrings(t.interner.release)
 }
 
-// processExternalLabels merges externalLabels into ls. If ls contains
+// processExternalLabels merges externalLabels into b. If b contains
-// a label in externalLabels, the value in ls wins.
+// a label in externalLabels, the value in b wins.
-func processExternalLabels(ls labels.Labels, externalLabels []labels.Label) labels.Labels {
+func processExternalLabels(b *labels.Builder, externalLabels []labels.Label) {
-	if len(externalLabels) == 0 {
+	for _, el := range externalLabels {
-		return ls
+		if b.Get(el.Name) == "" {
-	}
+			b.Set(el.Name, el.Value)
-
-	b := labels.NewScratchBuilder(ls.Len() + len(externalLabels))
-	j := 0
-	ls.Range(func(l labels.Label) {
-		for j < len(externalLabels) && l.Name > externalLabels[j].Name {
-			b.Add(externalLabels[j].Name, externalLabels[j].Value)
-			j++
 		}
-		if j < len(externalLabels) && l.Name == externalLabels[j].Name {
-			j++
-		}
-		b.Add(l.Name, l.Value)
-	})
-	for ; j < len(externalLabels); j++ {
-		b.Add(externalLabels[j].Name, externalLabels[j].Value)
 	}
-
-	return b.Labels()
 }
 
 func (t *QueueManager) updateShardsLoop() {

storage/remote/queue_manager_test.go

@@ -39,6 +39,7 @@ import (
 	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/model/relabel"
 	"github.com/prometheus/prometheus/model/timestamp"
 	"github.com/prometheus/prometheus/prompb"
 	writev2 "github.com/prometheus/prometheus/prompb/write/v2"
@@ -46,6 +47,7 @@ import (
 	"github.com/prometheus/prometheus/tsdb/chunks"
 	"github.com/prometheus/prometheus/tsdb/record"
 	"github.com/prometheus/prometheus/util/runutil"
+	"github.com/prometheus/prometheus/util/testutil"
 )
 
 const defaultFlushDeadline = 1 * time.Minute
@@ -843,7 +845,7 @@ func createExemplars(numExemplars, numSeries int) ([]record.RefExemplar, []recor
 				Ref:    chunks.HeadSeriesRef(i),
 				T:      int64(j),
 				V:      float64(i),
-				Labels: labels.FromStrings("traceID", fmt.Sprintf("trace-%d", i)),
+				Labels: labels.FromStrings("trace_id", fmt.Sprintf("trace-%d", i)),
 			}
 			exemplars = append(exemplars, e)
 		}
@@ -1186,29 +1188,30 @@ func (c *NopWriteClient) Store(context.Context, []byte, int) error { return nil
 func (c *NopWriteClient) Name() string                             { return "nopwriteclient" }
 func (c *NopWriteClient) Endpoint() string                         { return "http://test-remote.com/1234" }
 
+// Extra labels to make a more realistic workload - taken from Kubernetes' embedded cAdvisor metrics.
+var extraLabels []labels.Label = []labels.Label{
+	{Name: "kubernetes_io_arch", Value: "amd64"},
+	{Name: "kubernetes_io_instance_type", Value: "c3.somesize"},
+	{Name: "kubernetes_io_os", Value: "linux"},
+	{Name: "container_name", Value: "some-name"},
+	{Name: "failure_domain_kubernetes_io_region", Value: "somewhere-1"},
+	{Name: "failure_domain_kubernetes_io_zone", Value: "somewhere-1b"},
+	{Name: "id", Value: "/kubepods/burstable/pod6e91c467-e4c5-11e7-ace3-0a97ed59c75e/a3c8498918bd6866349fed5a6f8c643b77c91836427fb6327913276ebc6bde28"},
+	{Name: "image", Value: "registry/organisation/name@sha256:dca3d877a80008b45d71d7edc4fd2e44c0c8c8e7102ba5cbabec63a374d1d506"},
+	{Name: "instance", Value: "ip-111-11-1-11.ec2.internal"},
+	{Name: "job", Value: "kubernetes-cadvisor"},
+	{Name: "kubernetes_io_hostname", Value: "ip-111-11-1-11"},
+	{Name: "monitor", Value: "prod"},
+	{Name: "name", Value: "k8s_some-name_some-other-name-5j8s8_kube-system_6e91c467-e4c5-11e7-ace3-0a97ed59c75e_0"},
+	{Name: "namespace", Value: "kube-system"},
+	{Name: "pod_name", Value: "some-other-name-5j8s8"},
+}
+
 func BenchmarkSampleSend(b *testing.B) {
 	// Send one sample per series, which is the typical remote_write case
 	const numSamples = 1
 	const numSeries = 10000
 
-	// Extra labels to make a more realistic workload - taken from Kubernetes' embedded cAdvisor metrics.
-	extraLabels := []labels.Label{
-		{Name: "kubernetes_io_arch", Value: "amd64"},
-		{Name: "kubernetes_io_instance_type", Value: "c3.somesize"},
-		{Name: "kubernetes_io_os", Value: "linux"},
-		{Name: "container_name", Value: "some-name"},
-		{Name: "failure_domain_kubernetes_io_region", Value: "somewhere-1"},
-		{Name: "failure_domain_kubernetes_io_zone", Value: "somewhere-1b"},
-		{Name: "id", Value: "/kubepods/burstable/pod6e91c467-e4c5-11e7-ace3-0a97ed59c75e/a3c8498918bd6866349fed5a6f8c643b77c91836427fb6327913276ebc6bde28"},
-		{Name: "image", Value: "registry/organisation/name@sha256:dca3d877a80008b45d71d7edc4fd2e44c0c8c8e7102ba5cbabec63a374d1d506"},
-		{Name: "instance", Value: "ip-111-11-1-11.ec2.internal"},
-		{Name: "job", Value: "kubernetes-cadvisor"},
-		{Name: "kubernetes_io_hostname", Value: "ip-111-11-1-11"},
-		{Name: "monitor", Value: "prod"},
-		{Name: "name", Value: "k8s_some-name_some-other-name-5j8s8_kube-system_6e91c467-e4c5-11e7-ace3-0a97ed59c75e_0"},
-		{Name: "namespace", Value: "kube-system"},
-		{Name: "pod_name", Value: "some-other-name-5j8s8"},
-	}
 	samples, series := createTimeseries(numSamples, numSeries, extraLabels...)
 
 	c := NewNopWriteClient()
@@ -1240,6 +1243,58 @@ func BenchmarkSampleSend(b *testing.B) {
 	b.StopTimer()
 }
 
+// Check how long it takes to add N series, including external labels processing.
+func BenchmarkStoreSeries(b *testing.B) {
+	externalLabels := []labels.Label{
+		{Name: "cluster", Value: "mycluster"},
+		{Name: "replica", Value: "1"},
+	}
+	relabelConfigs := []*relabel.Config{{
+		SourceLabels: model.LabelNames{"namespace"},
+		Separator:    ";",
+		Regex:        relabel.MustNewRegexp("kube.*"),
+		TargetLabel:  "job",
+		Replacement:  "$1",
+		Action:       relabel.Replace,
+	}}
+	testCases := []struct {
+		name           string
+		externalLabels []labels.Label
+		ts             []prompb.TimeSeries
+		relabelConfigs []*relabel.Config
+	}{
+		{name: "plain"},
+		{name: "externalLabels", externalLabels: externalLabels},
+		{name: "relabel", relabelConfigs: relabelConfigs},
+		{
+			name:           "externalLabels+relabel",
+			externalLabels: externalLabels,
+			relabelConfigs: relabelConfigs,
+		},
+	}
+
+	// numSeries chosen to be big enough that StoreSeries dominates creating a new queue manager.
+	const numSeries = 1000
+	_, series := createTimeseries(0, numSeries, extraLabels...)
+
+	for _, tc := range testCases {
+		b.Run(tc.name, func(b *testing.B) {
+			for i := 0; i < b.N; i++ {
+				c := NewTestWriteClient(Version1)
+				dir := b.TempDir()
+				cfg := config.DefaultQueueConfig
+				mcfg := config.DefaultMetadataConfig
+				metrics := newQueueManagerMetrics(nil, "", "")
+				m := NewQueueManager(metrics, nil, nil, nil, dir, newEWMARate(ewmaWeight, shardUpdateDuration), cfg, mcfg, labels.EmptyLabels(), nil, c, defaultFlushDeadline, newPool(), newHighestTimestampMetric(), nil, false, false, Version1)
+				m.externalLabels = tc.externalLabels
+				m.relabelConfigs = tc.relabelConfigs
+
+				m.StoreSeries(series, 0)
+			}
+		})
+	}
+}
+
 func BenchmarkStartup(b *testing.B) {
 	dir := os.Getenv("WALDIR")
 	if dir == "" {
@@ -1279,7 +1334,8 @@ func BenchmarkStartup(b *testing.B) {
 }
 
 func TestProcessExternalLabels(t *testing.T) {
-	for _, tc := range []struct {
+	b := labels.NewBuilder(labels.EmptyLabels())
+	for i, tc := range []struct {
 		labels         labels.Labels
 		externalLabels []labels.Label
 		expected       labels.Labels
@@ -1340,7 +1396,9 @@ func TestProcessExternalLabels(t *testing.T) {
 			expected:       labels.FromStrings("a", "b", "c", "d", "e", "f"),
 		},
 	} {
-		require.Equal(t, tc.expected, processExternalLabels(tc.labels, tc.externalLabels))
+		b.Reset(tc.labels)
+		processExternalLabels(b, tc.externalLabels)
+		testutil.RequireEqual(t, tc.expected, b.Labels(), "test %d", i)
 	}
 }
 

storage/remote/read_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/prompb"
 	"github.com/prometheus/prometheus/util/annotations"
+	"github.com/prometheus/prometheus/util/testutil"
 )
 
 func TestNoDuplicateReadConfigs(t *testing.T) {
@@ -485,7 +486,7 @@ func TestSampleAndChunkQueryableClient(t *testing.T) {
 				got = append(got, ss.At().Labels())
 			}
 			require.NoError(t, ss.Err())
-			require.Equal(t, tc.expectedSeries, got)
+			testutil.RequireEqual(t, tc.expectedSeries, got)
 		})
 	}
 }

storage/remote/write_handler.go

@@ -113,7 +113,7 @@ func (h *writeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusNoContent)
 }
 
-// checkAppendExemplarError modifies the AppendExamplar's returned error based on the error cause.
+// checkAppendExemplarError modifies the AppendExemplar's returned error based on the error cause.
 func (h *writeHandler) checkAppendExemplarError(err error, e exemplar.Exemplar, outOfOrderErrs *int) error {
 	unwrappedErr := errors.Unwrap(err)
 	if unwrappedErr == nil {

storage/remote/write_handler_test.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/require"
 
 	"github.com/prometheus/prometheus/model/exemplar"
@@ -35,6 +36,7 @@ import (
 	"github.com/prometheus/prometheus/prompb"
 	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/tsdb"
+	"github.com/prometheus/prometheus/util/testutil"
 )
 
 func TestRemoteWriteHandler(t *testing.T) {
@@ -58,25 +60,25 @@ func TestRemoteWriteHandler(t *testing.T) {
 	j := 0
 	k := 0
 	for _, ts := range writeRequestFixture.Timeseries {
-		ls := labelProtosToLabels(ts.Labels)
+		labels := labelProtosToLabels(ts.Labels)
 		for _, s := range ts.Samples {
-			require.Equal(t, mockSample{ls, s.Timestamp, s.Value}, appendable.samples[i])
+			requireEqual(t, mockSample{labels, s.Timestamp, s.Value}, appendable.samples[i])
 			i++
 		}
 
 		for _, e := range ts.Exemplars {
 			exemplarLabels := labelProtosToLabels(e.Labels)
-			require.Equal(t, mockExemplar{ls, exemplarLabels, e.Timestamp, e.Value}, appendable.exemplars[j])
+			requireEqual(t, mockExemplar{labels, exemplarLabels, e.Timestamp, e.Value}, appendable.exemplars[j])
 			j++
 		}
 
 		for _, hp := range ts.Histograms {
 			if hp.IsFloatHistogram() {
 				fh := FloatHistogramProtoToFloatHistogram(hp)
-				require.Equal(t, mockHistogram{ls, hp.Timestamp, nil, fh}, appendable.histograms[k])
+				requireEqual(t, mockHistogram{labels, hp.Timestamp, nil, fh}, appendable.histograms[k])
 			} else {
 				h := HistogramProtoToHistogram(hp)
-				require.Equal(t, mockHistogram{ls, hp.Timestamp, h, nil}, appendable.histograms[k])
+				requireEqual(t, mockHistogram{labels, hp.Timestamp, h, nil}, appendable.histograms[k])
 			}
 
 			k++
@@ -351,6 +353,13 @@ type mockHistogram struct {
 	fh *histogram.FloatHistogram
 }
 
+// Wrapper to instruct go-cmp package to compare a list of structs with unexported fields.
+func requireEqual(t *testing.T, expected, actual interface{}, msgAndArgs ...interface{}) {
+	testutil.RequireEqualWithOptions(t, expected, actual,
+		[]cmp.Option{cmp.AllowUnexported(mockSample{}), cmp.AllowUnexported(mockExemplar{}), cmp.AllowUnexported(mockHistogram{})},
+		msgAndArgs...)
+}
+
 func (m *mockAppendable) Appender(_ context.Context) storage.Appender {
 	return m
 }

tsdb/db_test.go

@@ -27,7 +27,6 @@ import (
 	"path/filepath"
 	"sort"
 	"strconv"
-	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -2327,9 +2326,7 @@ func TestBlockRanges(t *testing.T) {
 	app := db.Appender(ctx)
 	lbl := labels.FromStrings("a", "b")
 	_, err = app.Append(0, lbl, firstBlockMaxT-1, rand.Float64())
-	if err == nil {
+	require.Error(t, err, "appending a sample with a timestamp covered by a previous block shouldn't be possible")
-		t.Fatalf("appending a sample with a timestamp covered by a previous block shouldn't be possible")
-	}
 	_, err = app.Append(0, lbl, firstBlockMaxT+1, rand.Float64())
 	require.NoError(t, err)
 	_, err = app.Append(0, lbl, firstBlockMaxT+2, rand.Float64())
@@ -2347,9 +2344,8 @@ func TestBlockRanges(t *testing.T) {
 	}
 	require.Len(t, db.Blocks(), 2, "no new block created after the set timeout")
 
-	if db.Blocks()[0].Meta().MaxTime > db.Blocks()[1].Meta().MinTime {
+	require.LessOrEqual(t, db.Blocks()[1].Meta().MinTime, db.Blocks()[0].Meta().MaxTime,
-		t.Fatalf("new block overlaps  old:%v,new:%v", db.Blocks()[0].Meta(), db.Blocks()[1].Meta())
+		"new block overlaps  old:%v,new:%v", db.Blocks()[0].Meta(), db.Blocks()[1].Meta())
-	}
 
 	// Test that wal records are skipped when an existing block covers the same time ranges
 	// and compaction doesn't create an overlapping block.
@@ -2389,9 +2385,8 @@ func TestBlockRanges(t *testing.T) {
 
 	require.Len(t, db.Blocks(), 4, "no new block created after the set timeout")
 
-	if db.Blocks()[2].Meta().MaxTime > db.Blocks()[3].Meta().MinTime {
+	require.LessOrEqual(t, db.Blocks()[3].Meta().MinTime, db.Blocks()[2].Meta().MaxTime,
-		t.Fatalf("new block overlaps  old:%v,new:%v", db.Blocks()[2].Meta(), db.Blocks()[3].Meta())
+		"new block overlaps  old:%v,new:%v", db.Blocks()[2].Meta(), db.Blocks()[3].Meta())
-	}
 }
 
 // TestDBReadOnly ensures that opening a DB in readonly mode doesn't modify any files on the disk.
@@ -3180,9 +3175,8 @@ func TestOpen_VariousBlockStates(t *testing.T) {
 
 	var loaded int
 	for _, l := range loadedBlocks {
-		if _, ok := expectedLoadedDirs[filepath.Join(tmpDir, l.meta.ULID.String())]; !ok {
+		_, ok := expectedLoadedDirs[filepath.Join(tmpDir, l.meta.ULID.String())]
-			t.Fatal("unexpected block", l.meta.ULID, "was loaded")
+		require.True(t, ok, "unexpected block", l.meta.ULID, "was loaded")
-		}
 		loaded++
 	}
 	require.Len(t, expectedLoadedDirs, loaded)
@@ -3193,9 +3187,8 @@ func TestOpen_VariousBlockStates(t *testing.T) {
 
 	var ignored int
 	for _, f := range files {
-		if _, ok := expectedRemovedDirs[filepath.Join(tmpDir, f.Name())]; ok {
+		_, ok := expectedRemovedDirs[filepath.Join(tmpDir, f.Name())]
-			t.Fatal("expected", filepath.Join(tmpDir, f.Name()), "to be removed, but still exists")
+		require.False(t, ok, "expected", filepath.Join(tmpDir, f.Name()), "to be removed, but still exists")
-		}
 		if _, ok := expectedIgnoredDirs[filepath.Join(tmpDir, f.Name())]; ok {
 			ignored++
 		}
@@ -3486,8 +3479,8 @@ func testQuerierShouldNotPanicIfHeadChunkIsTruncatedWhileReadingQueriedChunks(t
 	// the "cannot populate chunk XXX: not found" error occurred. This error can occur
 	// when the iterator tries to fetch an head chunk which has been offloaded because
 	// of the head compaction in the meanwhile.
-	if firstErr != nil && !strings.Contains(firstErr.Error(), "cannot populate chunk") {
+	if firstErr != nil {
-		t.Fatalf("unexpected error: %s", firstErr.Error())
+		require.ErrorContains(t, firstErr, "cannot populate chunk")
 	}
 }
 
@@ -4065,11 +4058,11 @@ func TestOOOWALWrite(t *testing.T) {
 
 	// The normal WAL.
 	actRecs := getRecords(path.Join(dir, "wal"))
-	require.Equal(t, inOrderRecords, actRecs)
+	testutil.RequireEqual(t, inOrderRecords, actRecs)
 
 	// The WBL.
 	actRecs = getRecords(path.Join(dir, wlog.WblDirName))
-	require.Equal(t, oooRecords, actRecs)
+	testutil.RequireEqual(t, oooRecords, actRecs)
 }
 
 // Tests https://github.com/prometheus/prometheus/issues/10291#issuecomment-1044373110.

tsdb/docs/format/index.md

@@ -82,6 +82,10 @@ Each series section is aligned to 16 bytes. The ID for a series is the `offset/1
 Every series entry first holds its number of labels, followed by tuples of symbol table references that contain the label name and value. The label pairs are lexicographically sorted.  
 After the labels, the number of indexed chunks is encoded, followed by a sequence of metadata entries containing the chunks minimum (`mint`) and maximum (`maxt`) timestamp and a reference to its position in the chunk file. The `mint` is the time of the first sample and `maxt` is the time of the last sample in the chunk. Holding the time range data in the index allows dropping chunks irrelevant to queried time ranges without accessing them directly.
 
+Chunk references within single series must be increasing, and chunk references for `series_(N+1)` must be higher than chunk references for `series_N`.
+This property guarantees that chunks that belong to the same series are grouped together in the segment files.
+Furthermore chunk `mint` must be less or equal than `maxt`, and subsequent chunks within single series must have increasing `mint` and `maxt` and not overlap.
+
 `mint` of the first chunk is stored, it's `maxt` is stored as a delta and the `mint` and `maxt` are encoded as deltas to the previous time for subsequent chunks. Similarly, the reference of the first chunk is stored and the next ref is stored as a delta to the previous one.
 
 ```

tsdb/exemplar_test.go

@@ -40,7 +40,7 @@ func TestValidateExemplar(t *testing.T) {
 
 	l := labels.FromStrings("service", "asdf")
 	e := exemplar.Exemplar{
-		Labels: labels.FromStrings("traceID", "qwerty"),
+		Labels: labels.FromStrings("trace_id", "qwerty"),
 		Value:  0.1,
 		Ts:     1,
 	}
@@ -49,7 +49,7 @@ func TestValidateExemplar(t *testing.T) {
 	require.NoError(t, es.AddExemplar(l, e))
 
 	e2 := exemplar.Exemplar{
-		Labels: labels.FromStrings("traceID", "zxcvb"),
+		Labels: labels.FromStrings("trace_id", "zxcvb"),
 		Value:  0.1,
 		Ts:     2,
 	}
@@ -82,7 +82,7 @@ func TestAddExemplar(t *testing.T) {
 
 	l := labels.FromStrings("service", "asdf")
 	e := exemplar.Exemplar{
-		Labels: labels.FromStrings("traceID", "qwerty"),
+		Labels: labels.FromStrings("trace_id", "qwerty"),
 		Value:  0.1,
 		Ts:     1,
 	}
@@ -91,7 +91,7 @@ func TestAddExemplar(t *testing.T) {
 	require.Equal(t, 0, es.index[string(l.Bytes(nil))].newest, "exemplar was not stored correctly")
 
 	e2 := exemplar.Exemplar{
-		Labels: labels.FromStrings("traceID", "zxcvb"),
+		Labels: labels.FromStrings("trace_id", "zxcvb"),
 		Value:  0.1,
 		Ts:     2,
 	}
@@ -132,7 +132,7 @@ func TestStorageOverflow(t *testing.T) {
 	var eList []exemplar.Exemplar
 	for i := 0; i < len(es.exemplars)+1; i++ {
 		e := exemplar.Exemplar{
-			Labels: labels.FromStrings("traceID", "a"),
+			Labels: labels.FromStrings("trace_id", "a"),
 			Value:  float64(i+1) / 10,
 			Ts:     int64(101 + i),
 		}
@@ -158,7 +158,7 @@ func TestSelectExemplar(t *testing.T) {
 	lName, lValue := "service", "asdf"
 	l := labels.FromStrings(lName, lValue)
 	e := exemplar.Exemplar{
-		Labels: labels.FromStrings("traceID", "querty"),
+		Labels: labels.FromStrings("trace_id", "querty"),
 		Value:  0.1,
 		Ts:     12,
 	}
@@ -189,7 +189,7 @@ func TestSelectExemplar_MultiSeries(t *testing.T) {
 
 	for i := 0; i < len(es.exemplars); i++ {
 		e1 := exemplar.Exemplar{
-			Labels: labels.FromStrings("traceID", "a"),
+			Labels: labels.FromStrings("trace_id", "a"),
 			Value:  float64(i+1) / 10,
 			Ts:     int64(101 + i),
 		}
@@ -197,7 +197,7 @@ func TestSelectExemplar_MultiSeries(t *testing.T) {
 		require.NoError(t, err)
 
 		e2 := exemplar.Exemplar{
-			Labels: labels.FromStrings("traceID", "b"),
+			Labels: labels.FromStrings("trace_id", "b"),
 			Value:  float64(i+1) / 10,
 			Ts:     int64(101 + i),
 		}
@@ -231,7 +231,7 @@ func TestSelectExemplar_TimeRange(t *testing.T) {
 
 	for i := 0; int64(i) < lenEs; i++ {
 		err := es.AddExemplar(l, exemplar.Exemplar{
-			Labels: labels.FromStrings("traceID", strconv.Itoa(i)),
+			Labels: labels.FromStrings("trace_id", strconv.Itoa(i)),
 			Value:  0.1,
 			Ts:     int64(101 + i),
 		})
@@ -255,7 +255,7 @@ func TestSelectExemplar_DuplicateSeries(t *testing.T) {
 	es := exs.(*CircularExemplarStorage)
 
 	e := exemplar.Exemplar{
-		Labels: labels.FromStrings("traceID", "qwerty"),
+		Labels: labels.FromStrings("trace_id", "qwerty"),
 		Value:  0.1,
 		Ts:     12,
 	}
@@ -413,7 +413,7 @@ func TestResize(t *testing.T) {
 func BenchmarkAddExemplar(b *testing.B) {
 	// We need to include these labels since we do length calculation
 	// before adding.
-	exLabels := labels.FromStrings("traceID", "89620921")
+	exLabels := labels.FromStrings("trace_id", "89620921")
 
 	for _, n := range []int{10000, 100000, 1000000} {
 		b.Run(fmt.Sprintf("%d", n), func(b *testing.B) {

tsdb/fileutil/flock_test.go

@@ -18,6 +18,8 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/prometheus/prometheus/util/testutil"
 )
 
@@ -27,54 +29,35 @@ func TestLocking(t *testing.T) {
 
 	fileName := filepath.Join(dir.Path(), "LOCK")
 
-	if _, err := os.Stat(fileName); err == nil {
+	_, err := os.Stat(fileName)
-		t.Fatalf("File %q unexpectedly exists.", fileName)
+	require.Error(t, err, "File %q unexpectedly exists.", fileName)
-	}
 
 	lock, existed, err := Flock(fileName)
-	if err != nil {
+	require.NoError(t, err, "Error locking file %q", fileName)
-		t.Fatalf("Error locking file %q: %s", fileName, err)
+	require.False(t, existed, "File %q reported as existing during locking.", fileName)
-	}
-	if existed {
-		t.Errorf("File %q reported as existing during locking.", fileName)
-	}
 
 	// File must now exist.
-	if _, err = os.Stat(fileName); err != nil {
+	_, err = os.Stat(fileName)
-		t.Errorf("Could not stat file %q expected to exist: %s", fileName, err)
+	require.NoError(t, err, "Could not stat file %q expected to exist", fileName)
-	}
 
 	// Try to lock again.
 	lockedAgain, existed, err := Flock(fileName)
-	if err == nil {
+	require.Error(t, err, "File %q locked twice.", fileName)
-		t.Fatalf("File %q locked twice.", fileName)
+	require.Nil(t, lockedAgain, "Unsuccessful locking did not return nil.")
-	}
+	require.True(t, existed, "Existing file %q not recognized.", fileName)
-	if lockedAgain != nil {
-		t.Error("Unsuccessful locking did not return nil.")
-	}
-	if !existed {
-		t.Errorf("Existing file %q not recognized.", fileName)
-	}
 
-	if err := lock.Release(); err != nil {
+	err = lock.Release()
-		t.Errorf("Error releasing lock for file %q: %s", fileName, err)
+	require.NoError(t, err, "Error releasing lock for file %q", fileName)
-	}
 
 	// File must still exist.
-	if _, err = os.Stat(fileName); err != nil {
+	_, err = os.Stat(fileName)
-		t.Errorf("Could not stat file %q expected to exist: %s", fileName, err)
+	require.NoError(t, err, "Could not stat file %q expected to exist", fileName)
-	}
 
 	// Lock existing file.
 	lock, existed, err = Flock(fileName)
-	if err != nil {
+	require.NoError(t, err, "Error locking file %q", fileName)
-		t.Fatalf("Error locking file %q: %s", fileName, err)
+	require.True(t, existed, "Existing file %q not recognized.", fileName)
-	}
-	if !existed {
-		t.Errorf("Existing file %q not recognized.", fileName)
-	}
 
-	if err := lock.Release(); err != nil {
+	err = lock.Release()
-		t.Errorf("Error releasing lock for file %q: %s", fileName, err)
+	require.NoError(t, err, "Error releasing lock for file %q", fileName)
-	}
 }

tsdb/head_test.go

@@ -30,6 +30,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
 	"github.com/prometheus/client_golang/prometheus"
 	prom_testutil "github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/prometheus/common/model"
@@ -50,6 +51,7 @@ import (
 	"github.com/prometheus/prometheus/tsdb/tombstones"
 	"github.com/prometheus/prometheus/tsdb/tsdbutil"
 	"github.com/prometheus/prometheus/tsdb/wlog"
+	"github.com/prometheus/prometheus/util/testutil"
 )
 
 // newTestHeadDefaultOptions returns the HeadOptions that should be used by default in unit tests.
@@ -206,7 +208,7 @@ func readTestWAL(t testing.TB, dir string) (recs []interface{}) {
 			require.NoError(t, err)
 			recs = append(recs, exemplars)
 		default:
-			t.Fatalf("unknown record type")
+			require.Fail(t, "unknown record type")
 		}
 	}
 	require.NoError(t, r.Err())
@@ -373,7 +375,7 @@ func BenchmarkLoadWLs(b *testing.B) {
 									Ref:    chunks.HeadSeriesRef(k) * 101,
 									T:      int64(i) * 10,
 									V:      float64(i) * 100,
-									Labels: labels.FromStrings("traceID", fmt.Sprintf("trace-%d", i)),
+									Labels: labels.FromStrings("trace_id", fmt.Sprintf("trace-%d", i)),
 								})
 							}
 							populateTestWL(b, wal, []interface{}{refExemplars})
@@ -658,7 +660,7 @@ func TestHead_ReadWAL(t *testing.T) {
 					{Ref: 0, Intervals: []tombstones.Interval{{Mint: 99, Maxt: 101}}},
 				},
 				[]record.RefExemplar{
-					{Ref: 10, T: 100, V: 1, Labels: labels.FromStrings("traceID", "asdf")},
+					{Ref: 10, T: 100, V: 1, Labels: labels.FromStrings("trace_id", "asdf")},
 				},
 			}
 
@@ -677,10 +679,10 @@ func TestHead_ReadWAL(t *testing.T) {
 			s50 := head.series.getByID(50)
 			s100 := head.series.getByID(100)
 
-			require.Equal(t, labels.FromStrings("a", "1"), s10.lset)
+			testutil.RequireEqual(t, labels.FromStrings("a", "1"), s10.lset)
-			require.Equal(t, (*memSeries)(nil), s11) // Series without samples should be garbage collected at head.Init().
+			require.Nil(t, s11) // Series without samples should be garbage collected at head.Init().
-			require.Equal(t, labels.FromStrings("a", "4"), s50.lset)
+			testutil.RequireEqual(t, labels.FromStrings("a", "4"), s50.lset)
-			require.Equal(t, labels.FromStrings("a", "3"), s100.lset)
+			testutil.RequireEqual(t, labels.FromStrings("a", "3"), s100.lset)
 
 			expandChunk := func(c chunkenc.Iterator) (x []sample) {
 				for c.Next() == chunkenc.ValFloat {
@@ -707,7 +709,7 @@ func TestHead_ReadWAL(t *testing.T) {
 			require.NoError(t, err)
 			e, err := q.Select(0, 1000, []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "a", "1")})
 			require.NoError(t, err)
-			require.Equal(t, exemplar.Exemplar{Ts: 100, Value: 1, Labels: labels.FromStrings("traceID", "asdf")}, e[0].Exemplars[0])
+			require.True(t, exemplar.Exemplar{Ts: 100, Value: 1, Labels: labels.FromStrings("trace_id", "asdf")}.Equals(e[0].Exemplars[0]))
 		})
 	}
 }
@@ -1371,7 +1373,7 @@ func TestDeletedSamplesAndSeriesStillInWALAfterCheckpoint(t *testing.T) {
 		case []record.RefMetadata:
 			metadata++
 		default:
-			t.Fatalf("unknown record type")
+			require.Fail(t, "unknown record type")
 		}
 	}
 	require.Equal(t, 1, series)
@@ -1620,9 +1622,7 @@ func TestComputeChunkEndTime(t *testing.T) {
 	for testName, tc := range cases {
 		t.Run(testName, func(t *testing.T) {
 			got := computeChunkEndTime(tc.start, tc.cur, tc.max, tc.ratioToFull)
-			if got != tc.res {
+			require.Equal(t, tc.res, got, "(start: %d, cur: %d, max: %d)", tc.start, tc.cur, tc.max)
-				t.Errorf("expected %d for (start: %d, cur: %d, max: %d, ratioToFull: %f), got %d", tc.res, tc.start, tc.cur, tc.max, tc.ratioToFull, got)
-			}
 		})
 	}
 }
@@ -3049,7 +3049,7 @@ func TestHeadExemplars(t *testing.T) {
 	head, _ := newTestHead(t, chunkRange, wlog.CompressionNone, false)
 	app := head.Appender(context.Background())
 
-	l := labels.FromStrings("traceId", "123")
+	l := labels.FromStrings("trace_id", "123")
 	// It is perfectly valid to add Exemplars before the current start time -
 	// histogram buckets that haven't been update in a while could still be
 	// exported exemplars from an hour ago.
@@ -3694,7 +3694,7 @@ func TestChunkSnapshot(t *testing.T) {
 		e := ex{
 			seriesLabels: lbls,
 			e: exemplar.Exemplar{
-				Labels: labels.FromStrings("traceID", fmt.Sprintf("%d", rand.Int())),
+				Labels: labels.FromStrings("trace_id", fmt.Sprintf("%d", rand.Int())),
 				Value:  rand.Float64(),
 				Ts:     ts,
 			},
@@ -3745,7 +3745,7 @@ func TestChunkSnapshot(t *testing.T) {
 		})
 		require.NoError(t, err)
 		// Verifies both existence of right exemplars and order of exemplars in the buffer.
-		require.Equal(t, expExemplars, actExemplars)
+		testutil.RequireEqualWithOptions(t, expExemplars, actExemplars, []cmp.Option{cmp.AllowUnexported(ex{})})
 	}
 
 	var (

tsdb/index/index.go

@@ -146,8 +146,11 @@ type Writer struct {
 	labelNames   map[string]uint64     // Label names, and their usage.
 
 	// Hold last series to validate that clients insert new series in order.
-	lastSeries labels.Labels
+	lastSeries    labels.Labels
-	lastRef    storage.SeriesRef
+	lastSeriesRef storage.SeriesRef
+
+	// Hold last added chunk reference to make sure that chunks are ordered properly.
+	lastChunkRef chunks.ChunkRef
 
 	crc32 hash.Hash
 
@@ -433,9 +436,27 @@ func (w *Writer) AddSeries(ref storage.SeriesRef, lset labels.Labels, chunks ...
 		return fmt.Errorf("out-of-order series added with label set %q", lset)
 	}
 
-	if ref < w.lastRef && !w.lastSeries.IsEmpty() {
+	if ref < w.lastSeriesRef && !w.lastSeries.IsEmpty() {
 		return fmt.Errorf("series with reference greater than %d already added", ref)
 	}
+
+	lastChunkRef := w.lastChunkRef
+	lastMaxT := int64(0)
+	for ix, c := range chunks {
+		if c.Ref < lastChunkRef {
+			return fmt.Errorf("unsorted chunk reference: %d, previous: %d", c.Ref, lastChunkRef)
+		}
+		lastChunkRef = c.Ref
+
+		if ix > 0 && c.MinTime <= lastMaxT {
+			return fmt.Errorf("chunk minT %d is not higher than previous chunk maxT %d", c.MinTime, lastMaxT)
+		}
+		if c.MaxTime < c.MinTime {
+			return fmt.Errorf("chunk maxT %d is less than minT %d", c.MaxTime, c.MinTime)
+		}
+		lastMaxT = c.MaxTime
+	}
+
 	// We add padding to 16 bytes to increase the addressable space we get through 4 byte
 	// series references.
 	if err := w.addPadding(seriesByteAlign); err != nil {
@@ -510,7 +531,8 @@ func (w *Writer) AddSeries(ref storage.SeriesRef, lset labels.Labels, chunks ...
 	}
 
 	w.lastSeries.CopyFrom(lset)
-	w.lastRef = ref
+	w.lastSeriesRef = ref
+	w.lastChunkRef = lastChunkRef
 
 	return nil
 }
@@ -715,17 +737,11 @@ func (w *Writer) writeLabelIndexesOffsetTable() error {
 	}
 
 	// Write out the length.
-	w.buf1.Reset()
+	err := w.writeLengthAndHash(startPos)
-	l := w.f.pos - startPos - 4
+	if err != nil {
-	if l > math.MaxUint32 {
+		return fmt.Errorf("label indexes offset table length/crc32 write error: %w", err)
-		return fmt.Errorf("label indexes offset table size exceeds 4 bytes: %d", l)
 	}
-	w.buf1.PutBE32int(int(l))
+	return nil
-	if err := w.writeAt(w.buf1.Get(), startPos); err != nil {
-		return err
-	}
-
-	return w.writeLenghtAndHash(startPos)
 }
 
 // writePostingsOffsetTable writes the postings offset table.
@@ -793,25 +809,31 @@ func (w *Writer) writePostingsOffsetTable() error {
 	}
 	w.fPO = nil
 
-	return w.writeLenghtAndHash(startPos)
+	err = w.writeLengthAndHash(startPos)
+	if err != nil {
+		return fmt.Errorf("postings offset table length/crc32 write error: %w", err)
+	}
+	return nil
 }
 
-func (w *Writer) writeLenghtAndHash(startPos uint64) error {
+func (w *Writer) writeLengthAndHash(startPos uint64) error {
-	// Write out the length.
 	w.buf1.Reset()
 	l := w.f.pos - startPos - 4
 	if l > math.MaxUint32 {
-		return fmt.Errorf("postings offset table size exceeds 4 bytes: %d", l)
+		return fmt.Errorf("length size exceeds 4 bytes: %d", l)
 	}
 	w.buf1.PutBE32int(int(l))
 	if err := w.writeAt(w.buf1.Get(), startPos); err != nil {
-		return err
+		return fmt.Errorf("write length from buffer error: %w", err)
 	}
 
 	// Write out the hash.
 	w.buf1.Reset()
 	w.buf1.PutHashSum(w.crc32)
-	return w.write(w.buf1.Get())
+	if err := w.write(w.buf1.Get()); err != nil {
+		return fmt.Errorf("write buffer's crc32 error: %w", err)
+	}
+	return nil
 }
 
 const indexTOCLen = 6*8 + crc32.Size

tsdb/index/index_test.go

@@ -18,7 +18,6 @@ import (
 	"errors"
 	"fmt"
 	"hash/crc32"
-	"math/rand"
 	"os"
 	"path/filepath"
 	"sort"
@@ -206,7 +205,7 @@ func TestIndexRW_Postings(t *testing.T) {
 
 		require.NoError(t, err)
 		require.Empty(t, c)
-		require.Equal(t, series[i], builder.Labels())
+		testutil.RequireEqual(t, series[i], builder.Labels())
 	}
 	require.NoError(t, p.Err())
 
@@ -407,15 +406,17 @@ func TestPersistence_index_e2e(t *testing.T) {
 
 	var input indexWriterSeriesSlice
 
+	ref := uint64(0)
 	// Generate ChunkMetas for every label set.
 	for i, lset := range lbls {
 		var metas []chunks.Meta
 
 		for j := 0; j <= (i % 20); j++ {
+			ref++
 			metas = append(metas, chunks.Meta{
 				MinTime: int64(j * 10000),
-				MaxTime: int64((j + 1) * 10000),
+				MaxTime: int64((j+1)*10000) - 1,
-				Ref:     chunks.ChunkRef(rand.Uint64()),
+				Ref:     chunks.ChunkRef(ref),
 				Chunk:   chunkenc.NewXORChunk(),
 			})
 		}
@@ -487,7 +488,7 @@ func TestPersistence_index_e2e(t *testing.T) {
 
 			err = mi.Series(expp.At(), &eBuilder, &expchks)
 			require.NoError(t, err)
-			require.Equal(t, eBuilder.Labels(), builder.Labels())
+			testutil.RequireEqual(t, eBuilder.Labels(), builder.Labels())
 			require.Equal(t, expchks, chks)
 		}
 		require.False(t, expp.Next(), "Expected no more postings for %q=%q", p.Name, p.Value)
@@ -670,3 +671,51 @@ func TestDecoder_Postings_WrongInput(t *testing.T) {
 	_, _, err := (&Decoder{}).Postings([]byte("the cake is a lie"))
 	require.Error(t, err)
 }
+
+func TestChunksRefOrdering(t *testing.T) {
+	dir := t.TempDir()
+
+	idx, err := NewWriter(context.Background(), filepath.Join(dir, "index"))
+	require.NoError(t, err)
+
+	require.NoError(t, idx.AddSymbol("1"))
+	require.NoError(t, idx.AddSymbol("2"))
+	require.NoError(t, idx.AddSymbol("__name__"))
+
+	c50 := chunks.Meta{Ref: 50}
+	c100 := chunks.Meta{Ref: 100}
+	c200 := chunks.Meta{Ref: 200}
+
+	require.NoError(t, idx.AddSeries(1, labels.FromStrings("__name__", "1"), c100))
+	require.EqualError(t, idx.AddSeries(2, labels.FromStrings("__name__", "2"), c50), "unsorted chunk reference: 50, previous: 100")
+	require.NoError(t, idx.AddSeries(2, labels.FromStrings("__name__", "2"), c200))
+	require.NoError(t, idx.Close())
+}
+
+func TestChunksTimeOrdering(t *testing.T) {
+	dir := t.TempDir()
+
+	idx, err := NewWriter(context.Background(), filepath.Join(dir, "index"))
+	require.NoError(t, err)
+
+	require.NoError(t, idx.AddSymbol("1"))
+	require.NoError(t, idx.AddSymbol("2"))
+	require.NoError(t, idx.AddSymbol("__name__"))
+
+	require.NoError(t, idx.AddSeries(1, labels.FromStrings("__name__", "1"),
+		chunks.Meta{Ref: 1, MinTime: 0, MaxTime: 10}, // Also checks that first chunk can have MinTime: 0.
+		chunks.Meta{Ref: 2, MinTime: 11, MaxTime: 20},
+		chunks.Meta{Ref: 3, MinTime: 21, MaxTime: 30},
+	))
+
+	require.EqualError(t, idx.AddSeries(1, labels.FromStrings("__name__", "2"),
+		chunks.Meta{Ref: 10, MinTime: 0, MaxTime: 10},
+		chunks.Meta{Ref: 20, MinTime: 10, MaxTime: 20},
+	), "chunk minT 10 is not higher than previous chunk maxT 10")
+
+	require.EqualError(t, idx.AddSeries(1, labels.FromStrings("__name__", "2"),
+		chunks.Meta{Ref: 10, MinTime: 100, MaxTime: 30},
+	), "chunk maxT 30 is less than minT 100")
+
+	require.NoError(t, idx.Close())
+}

tsdb/index/postings_test.go

@@ -61,9 +61,7 @@ func TestMemPostings_ensureOrder(t *testing.T) {
 			ok := sort.SliceIsSorted(l, func(i, j int) bool {
 				return l[i] < l[j]
 			})
-			if !ok {
+			require.True(t, ok, "postings list %v is not sorted", l)
-				t.Fatalf("postings list %v is not sorted", l)
-			}
 		}
 	}
 }
@@ -214,9 +212,7 @@ func TestIntersect(t *testing.T) {
 
 	for _, c := range cases {
 		t.Run("", func(t *testing.T) {
-			if c.res == nil {
+			require.NotNil(t, c.res, "intersect result expectancy cannot be nil")
-				t.Fatal("intersect result expectancy cannot be nil")
-			}
 
 			expected, err := ExpandPostings(c.res)
 			require.NoError(t, err)
@@ -228,9 +224,7 @@ func TestIntersect(t *testing.T) {
 				return
 			}
 
-			if i == EmptyPostings() {
+			require.NotEqual(t, EmptyPostings(), i, "intersect unexpected result: EmptyPostings sentinel")
-				t.Fatal("intersect unexpected result: EmptyPostings sentinel")
-			}
 
 			res, err := ExpandPostings(i)
 			require.NoError(t, err)
@@ -501,9 +495,7 @@ func TestMergedPostings(t *testing.T) {
 
 	for _, c := range cases {
 		t.Run("", func(t *testing.T) {
-			if c.res == nil {
+			require.NotNil(t, c.res, "merge result expectancy cannot be nil")
-				t.Fatal("merge result expectancy cannot be nil")
-			}
 
 			ctx := context.Background()
 
@@ -517,9 +509,7 @@ func TestMergedPostings(t *testing.T) {
 				return
 			}
 
-			if m == EmptyPostings() {
+			require.NotEqual(t, EmptyPostings(), m, "merge unexpected result: EmptyPostings sentinel")
-				t.Fatal("merge unexpected result: EmptyPostings sentinel")
-			}
 
 			res, err := ExpandPostings(m)
 			require.NoError(t, err)
@@ -897,9 +887,7 @@ func TestWithoutPostings(t *testing.T) {
 
 	for _, c := range cases {
 		t.Run("", func(t *testing.T) {
-			if c.res == nil {
+			require.NotNil(t, c.res, "without result expectancy cannot be nil")
-				t.Fatal("without result expectancy cannot be nil")
-			}
 
 			expected, err := ExpandPostings(c.res)
 			require.NoError(t, err)
@@ -911,9 +899,7 @@ func TestWithoutPostings(t *testing.T) {
 				return
 			}
 
-			if w == EmptyPostings() {
+			require.NotEqual(t, EmptyPostings(), w, "without unexpected result: EmptyPostings sentinel")
-				t.Fatal("without unexpected result: EmptyPostings sentinel")
-			}
 
 			res, err := ExpandPostings(w)
 			require.NoError(t, err)

tsdb/querier_test.go

@@ -2702,22 +2702,7 @@ func TestFindSetMatches(t *testing.T) {
 	}
 
 	for _, c := range cases {
-		matches := findSetMatches(c.pattern)
+		require.Equal(t, c.exp, findSetMatches(c.pattern), "Evaluating %s, unexpected result.", c.pattern)
-		if len(c.exp) == 0 {
-			if len(matches) != 0 {
-				t.Errorf("Evaluating %s, unexpected result %v", c.pattern, matches)
-			}
-		} else {
-			if len(matches) != len(c.exp) {
-				t.Errorf("Evaluating %s, length of result not equal to exp", c.pattern)
-			} else {
-				for i := 0; i < len(c.exp); i++ {
-					if c.exp[i] != matches[i] {
-						t.Errorf("Evaluating %s, unexpected result %s", c.pattern, matches[i])
-					}
-				}
-			}
-		}
 	}
 }
 
@@ -3016,9 +3001,7 @@ func TestPostingsForMatchers(t *testing.T) {
 				}
 			}
 			require.NoError(t, p.Err())
-			if len(exp) != 0 {
+			require.Empty(t, exp, "Evaluating %v", c.matchers)
-				t.Errorf("Evaluating %v, missing results %+v", c.matchers, exp)
-			}
 		})
 	}
 }
@@ -3101,9 +3084,7 @@ func TestClose(t *testing.T) {
 	createBlock(t, dir, genSeries(1, 1, 10, 20))
 
 	db, err := Open(dir, nil, nil, DefaultOptions(), nil)
-	if err != nil {
+	require.NoError(t, err, "Opening test storage failed: %s")
-		t.Fatalf("Opening test storage failed: %s", err)
-	}
 	defer func() {
 		require.NoError(t, db.Close())
 	}()

tsdb/record/record_test.go

@@ -24,6 +24,7 @@ import (
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/tsdb/encoding"
 	"github.com/prometheus/prometheus/tsdb/tombstones"
+	"github.com/prometheus/prometheus/util/testutil"
 )
 
 func TestRecord_EncodeDecode(t *testing.T) {
@@ -44,7 +45,7 @@ func TestRecord_EncodeDecode(t *testing.T) {
 	}
 	decSeries, err := dec.Series(enc.Series(series, nil), nil)
 	require.NoError(t, err)
-	require.Equal(t, series, decSeries)
+	testutil.RequireEqual(t, series, decSeries)
 
 	metadata := []RefMetadata{
 		{
@@ -101,13 +102,13 @@ func TestRecord_EncodeDecode(t *testing.T) {
 	}, decTstones)
 
 	exemplars := []RefExemplar{
-		{Ref: 0, T: 12423423, V: 1.2345, Labels: labels.FromStrings("traceID", "qwerty")},
+		{Ref: 0, T: 12423423, V: 1.2345, Labels: labels.FromStrings("trace_id", "qwerty")},
-		{Ref: 123, T: -1231, V: -123, Labels: labels.FromStrings("traceID", "asdf")},
+		{Ref: 123, T: -1231, V: -123, Labels: labels.FromStrings("trace_id", "asdf")},
-		{Ref: 2, T: 0, V: 99999, Labels: labels.FromStrings("traceID", "zxcv")},
+		{Ref: 2, T: 0, V: 99999, Labels: labels.FromStrings("trace_id", "zxcv")},
 	}
 	decExemplars, err := dec.Exemplars(enc.Exemplars(exemplars, nil), nil)
 	require.NoError(t, err)
-	require.Equal(t, exemplars, decExemplars)
+	testutil.RequireEqual(t, exemplars, decExemplars)
 
 	histograms := []RefHistogramSample{
 		{
@@ -226,7 +227,7 @@ func TestRecord_Corrupted(t *testing.T) {
 
 	t.Run("Test corrupted exemplar record", func(t *testing.T) {
 		exemplars := []RefExemplar{
-			{Ref: 0, T: 12423423, V: 1.2345, Labels: labels.FromStrings("traceID", "asdf")},
+			{Ref: 0, T: 12423423, V: 1.2345, Labels: labels.FromStrings("trace_id", "asdf")},
 		}
 
 		corrupted := enc.Exemplars(exemplars, nil)[:8]

tsdb/repair_test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/prometheus/prometheus/tsdb/chunks"
 	"github.com/prometheus/prometheus/tsdb/fileutil"
 	"github.com/prometheus/prometheus/tsdb/index"
+	"github.com/prometheus/prometheus/util/testutil"
 )
 
 func TestRepairBadIndexVersion(t *testing.T) {
@@ -112,7 +113,7 @@ func TestRepairBadIndexVersion(t *testing.T) {
 	}
 
 	require.NoError(t, p.Err())
-	require.Equal(t, []labels.Labels{
+	testutil.RequireEqual(t, []labels.Labels{
 		labels.FromStrings("a", "1", "b", "1"),
 		labels.FromStrings("a", "2", "b", "1"),
 	}, res)

tsdb/wal_test.go

@@ -34,6 +34,7 @@ import (
 	"github.com/prometheus/prometheus/tsdb/record"
 	"github.com/prometheus/prometheus/tsdb/tombstones"
 	"github.com/prometheus/prometheus/tsdb/wlog"
+	"github.com/prometheus/prometheus/util/testutil"
 )
 
 func TestSegmentWAL_cut(t *testing.T) {
@@ -147,7 +148,7 @@ func TestSegmentWAL_Truncate(t *testing.T) {
 		readSeries = append(readSeries, s...)
 	}, nil, nil))
 
-	require.Equal(t, expected, readSeries)
+	testutil.RequireEqual(t, expected, readSeries)
 }
 
 // Symmetrical test of reading and writing to the WAL via its main interface.
@@ -213,9 +214,9 @@ func TestSegmentWAL_Log_Restore(t *testing.T) {
 
 		require.NoError(t, r.Read(serf, smplf, delf))
 
-		require.Equal(t, recordedSamples, resultSamples)
+		testutil.RequireEqual(t, recordedSamples, resultSamples)
-		require.Equal(t, recordedSeries, resultSeries)
+		testutil.RequireEqual(t, recordedSeries, resultSeries)
-		require.Equal(t, recordedDeletes, resultDeletes)
+		testutil.RequireEqual(t, recordedDeletes, resultDeletes)
 
 		series := series[k : k+(numMetrics/iterations)]
 
@@ -528,12 +529,12 @@ func TestMigrateWAL_Fuzz(t *testing.T) {
 			require.NoError(t, err)
 			res = append(res, s)
 		default:
-			t.Fatalf("unknown record type %d", dec.Type(rec))
+			require.Fail(t, "unknown record type %d", dec.Type(rec))
 		}
 	}
 	require.NoError(t, r.Err())
 
-	require.Equal(t, []interface{}{
+	testutil.RequireEqual(t, []interface{}{
 		[]record.RefSeries{
 			{Ref: 100, Labels: labels.FromStrings("abc", "def", "123", "456")},
 			{Ref: 1, Labels: labels.FromStrings("abc", "def2", "1234", "4567")},

tsdb/wlog/checkpoint_test.go

@@ -29,6 +29,7 @@ import (
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/tsdb/chunks"
 	"github.com/prometheus/prometheus/tsdb/record"
+	"github.com/prometheus/prometheus/util/testutil"
 )
 
 func TestLastCheckpoint(t *testing.T) {
@@ -201,7 +202,7 @@ func TestCheckpoint(t *testing.T) {
 				histogramsInWAL += 4
 
 				b = enc.Exemplars([]record.RefExemplar{
-					{Ref: 1, T: last, V: float64(i), Labels: labels.FromStrings("traceID", fmt.Sprintf("trace-%d", i))},
+					{Ref: 1, T: last, V: float64(i), Labels: labels.FromStrings("trace_id", fmt.Sprintf("trace-%d", i))},
 				}, nil)
 				require.NoError(t, w.Log(b))
 
@@ -286,7 +287,7 @@ func TestCheckpoint(t *testing.T) {
 				{Ref: 2, Labels: labels.FromStrings("a", "b", "c", "2")},
 				{Ref: 4, Labels: labels.FromStrings("a", "b", "c", "4")},
 			}
-			require.Equal(t, expectedRefSeries, series)
+			testutil.RequireEqual(t, expectedRefSeries, series)
 
 			expectedRefMetadata := []record.RefMetadata{
 				{Ref: 0, Unit: fmt.Sprintf("%d", last-100), Help: fmt.Sprintf("%d", last-100)},

tsdb/wlog/reader_test.go

@@ -182,16 +182,13 @@ func TestReader(t *testing.T) {
 					t.Logf("record %d", j)
 					rec := r.Record()
 
-					if j >= len(c.exp) {
+					require.Less(t, j, len(c.exp), "received more records than expected")
-						t.Fatal("received more records than expected")
-					}
 					require.Equal(t, c.exp[j], rec, "Bytes within record did not match expected Bytes")
 				}
-				if !c.fail && r.Err() != nil {
+				if !c.fail {
-					t.Fatalf("unexpected error: %s", r.Err())
+					require.NoError(t, r.Err())
-				}
+				} else {
-				if c.fail && r.Err() == nil {
+					require.Error(t, r.Err())
-					t.Fatalf("expected error but got none")
 				}
 			})
 		}

tsdb/wlog/watcher_test.go

@@ -171,7 +171,7 @@ func TestTailSamples(t *testing.T) {
 							Ref:    chunks.HeadSeriesRef(inner),
 							T:      now.UnixNano() + 1,
 							V:      float64(i),
-							Labels: labels.FromStrings("traceID", fmt.Sprintf("trace-%d", inner)),
+							Labels: labels.FromStrings("trace_id", fmt.Sprintf("trace-%d", inner)),
 						},
 					}, nil)
 					require.NoError(t, w.Log(exemplar))

tsdb/wlog/wlog_test.go

@@ -192,9 +192,7 @@ func TestWALRepair_ReadingError(t *testing.T) {
 			require.Len(t, result, test.intactRecs, "Wrong number of intact records")
 
 			for i, r := range result {
-				if !bytes.Equal(records[i], r) {
+				require.True(t, bytes.Equal(records[i], r), "record %d diverges: want %x, got %x", i, records[i][:10], r[:10])
-					t.Fatalf("record %d diverges: want %x, got %x", i, records[i][:10], r[:10])
-				}
 			}
 
 			// Make sure there is a new 0 size Segment after the corrupted Segment.

util/runtime/statfs_default.go

@@ -71,7 +71,8 @@ func Statfs(path string) string {
 
 	var fs syscall.Statfs_t
 	err := syscall.Statfs(path, &fs)
-	//nolint:unconvert // This ensure Type format on all Platforms
+	// nolintlint might cry out depending on the architecture (e.g. ARM64), so ignore it.
+	//nolint:unconvert,nolintlint // This ensures Type format on all Platforms.
 	localType := int64(fs.Type)
 	if err != nil {
 		return strconv.FormatInt(localType, 16)

util/testutil/cmp.go

@@ -0,0 +1,43 @@
+// Copyright 2023 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package testutil
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/prometheus/model/labels"
+)
+
+// Replacement for require.Equal using go-cmp adapted for Prometheus data structures, instead of DeepEqual.
+func RequireEqual(t testing.TB, expected, actual interface{}, msgAndArgs ...interface{}) {
+	t.Helper()
+	RequireEqualWithOptions(t, expected, actual, nil, msgAndArgs...)
+}
+
+// As RequireEqual but allows extra cmp.Options.
+func RequireEqualWithOptions(t testing.TB, expected, actual interface{}, extra []cmp.Option, msgAndArgs ...interface{}) {
+	t.Helper()
+	options := append([]cmp.Option{cmp.Comparer(labels.Equal)}, extra...)
+	if cmp.Equal(expected, actual, options...) {
+		return
+	}
+	diff := cmp.Diff(expected, actual, options...)
+	require.Fail(t, fmt.Sprintf("Not equal: \n"+
+		"expected: %s\n"+
+		"actual  : %s%s", expected, actual, diff), msgAndArgs...)
+}

web/api/v1/api_test.go

@@ -32,6 +32,7 @@ import (
 
 	"github.com/prometheus/prometheus/prompb"
 	"github.com/prometheus/prometheus/util/stats"
+	"github.com/prometheus/prometheus/util/testutil"
 
 	"github.com/go-kit/log"
 	"github.com/prometheus/client_golang/prometheus"
@@ -216,18 +217,11 @@ type rulesRetrieverMock struct {
 
 func (m *rulesRetrieverMock) CreateAlertingRules() {
 	expr1, err := parser.ParseExpr(`absent(test_metric3) != 1`)
-	if err != nil {
+	require.NoError(m.testing, err)
-		m.testing.Fatalf("unable to parse alert expression: %s", err)
-	}
 	expr2, err := parser.ParseExpr(`up == 1`)
-	if err != nil {
+	require.NoError(m.testing, err)
-		m.testing.Fatalf("Unable to parse alert expression: %s", err)
-	}
-
 	expr3, err := parser.ParseExpr(`vector(1)`)
-	if err != nil {
+	require.NoError(m.testing, err)
-		m.testing.Fatalf("Unable to parse alert expression: %s", err)
-	}
 
 	rule1 := rules.NewAlertingRule(
 		"test_metric3",
@@ -302,9 +296,7 @@ func (m *rulesRetrieverMock) CreateRuleGroups() {
 	}
 
 	recordingExpr, err := parser.ParseExpr(`vector(1)`)
-	if err != nil {
+	require.NoError(m.testing, err, "unable to parse alert expression")
-		m.testing.Fatalf("unable to parse alert expression: %s", err)
-	}
 	recordingRule := rules.NewRecordingRule("recording-rule-1", recordingExpr, labels.Labels{})
 	r = append(r, recordingRule)
 
@@ -607,7 +599,7 @@ func TestGetSeries(t *testing.T) {
 				r := res.data.([]labels.Labels)
 				sort.Sort(byLabels(tc.expected))
 				sort.Sort(byLabels(r))
-				require.Equal(t, tc.expected, r)
+				testutil.RequireEqual(t, tc.expected, r)
 			}
 		})
 	}
@@ -715,9 +707,7 @@ func TestQueryExemplars(t *testing.T) {
 			for _, te := range tc.exemplars {
 				for _, e := range te.Exemplars {
 					_, err := es.AppendExemplar(0, te.SeriesLabels, e)
-					if err != nil {
+					require.NoError(t, err)
-						t.Fatal(err)
-					}
 				}
 			}
 
@@ -2833,9 +2823,7 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 					}
 
 					req, err := request(method, test.query)
-					if err != nil {
+					require.NoError(t, err)
-						t.Fatal(err)
-					}
 
 					tr.ResetMetadataStore()
 					for _, tm := range test.metadata {
@@ -2845,9 +2833,7 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 					for _, te := range test.exemplars {
 						for _, e := range te.Exemplars {
 							_, err := es.AppendExemplar(0, te.SeriesLabels, e)
-							if err != nil {
+							require.NoError(t, err)
-								t.Fatal(err)
-							}
 						}
 					}
 
@@ -2883,37 +2869,25 @@ func describeAPIFunc(f apiFunc) string {
 func assertAPIError(t *testing.T, got *apiError, exp errorType) {
 	t.Helper()
 
-	if got != nil {
+	if exp == errorNone {
-		if exp == errorNone {
+		require.Nil(t, got)
-			t.Fatalf("Unexpected error: %s", got)
+	} else {
-		}
+		require.NotNil(t, got)
-		if exp != got.typ {
+		require.Equal(t, exp, got.typ, "(%q)", got)
-			t.Fatalf("Expected error of type %q but got type %q (%q)", exp, got.typ, got)
-		}
-		return
-	}
-	if exp != errorNone {
-		t.Fatalf("Expected error of type %q but got none", exp)
 	}
 }
 
 func assertAPIResponse(t *testing.T, got, exp interface{}) {
 	t.Helper()
 
-	require.Equal(t, exp, got)
+	testutil.RequireEqual(t, exp, got)
 }
 
 func assertAPIResponseLength(t *testing.T, got interface{}, expLen int) {
 	t.Helper()
 
 	gotLen := reflect.ValueOf(got).Len()
-	if gotLen != expLen {
+	require.Equal(t, expLen, gotLen, "Response length does not match")
-		t.Fatalf(
-			"Response length does not match, expected:\n%d\ngot:\n%d",
-			expLen,
-			gotLen,
-		)
-	}
 }
 
 func assertAPIResponseMetadataLen(t *testing.T, got interface{}, expLen int) {
@@ -2925,13 +2899,7 @@ func assertAPIResponseMetadataLen(t *testing.T, got interface{}, expLen int) {
 		gotLen += len(m)
 	}
 
-	if gotLen != expLen {
+	require.Equal(t, expLen, gotLen, "Amount of metadata in the response does not match")
-		t.Fatalf(
-			"Amount of metadata in the response does not match, expected:\n%d\ngot:\n%d",
-			expLen,
-			gotLen,
-		)
-	}
 }
 
 type fakeDB struct {
@@ -3272,26 +3240,18 @@ func TestRespondError(t *testing.T) {
 	defer s.Close()
 
 	resp, err := http.Get(s.URL)
-	if err != nil {
+	require.NoError(t, err, "Error on test request")
-		t.Fatalf("Error on test request: %s", err)
-	}
 	body, err := io.ReadAll(resp.Body)
 	defer resp.Body.Close()
-	if err != nil {
+	require.NoError(t, err, "Error reading response body")
-		t.Fatalf("Error reading response body: %s", err)
+	want, have := http.StatusServiceUnavailable, resp.StatusCode
-	}
+	require.Equal(t, want, have, "Return code %d expected in error response but got %d", want, have)
-
+	h := resp.Header.Get("Content-Type")
-	if want, have := http.StatusServiceUnavailable, resp.StatusCode; want != have {
+	require.Equal(t, "application/json", h, "Expected Content-Type %q but got %q", "application/json", h)
-		t.Fatalf("Return code %d expected in error response but got %d", want, have)
-	}
-	if h := resp.Header.Get("Content-Type"); h != "application/json" {
-		t.Fatalf("Expected Content-Type %q but got %q", "application/json", h)
-	}
 
 	var res Response
-	if err = json.Unmarshal(body, &res); err != nil {
+	err = json.Unmarshal(body, &res)
-		t.Fatalf("Error unmarshaling JSON body: %s", err)
+	require.NoError(t, err, "Error unmarshaling JSON body")
-	}
 
 	exp := &Response{
 		Status:    statusError,
@@ -3420,17 +3380,13 @@ func TestParseTime(t *testing.T) {
 
 	for _, test := range tests {
 		ts, err := parseTime(test.input)
-		if err != nil && !test.fail {
+		if !test.fail {
-			t.Errorf("Unexpected error for %q: %s", test.input, err)
+			require.NoError(t, err, "Unexpected error for %q", test.input)
+			require.NotNil(t, ts)
+			require.True(t, ts.Equal(test.result), "Expected time %v for input %q but got %v", test.result, test.input, ts)
 			continue
 		}
-		if err == nil && test.fail {
+		require.Error(t, err, "Expected error for %q but got none", test.input)
-			t.Errorf("Expected error for %q but got none", test.input)
-			continue
-		}
-		if !test.fail && !ts.Equal(test.result) {
-			t.Errorf("Expected time %v for input %q but got %v", test.result, test.input, ts)
-		}
 	}
 }
 
@@ -3474,17 +3430,12 @@ func TestParseDuration(t *testing.T) {
 
 	for _, test := range tests {
 		d, err := parseDuration(test.input)
-		if err != nil && !test.fail {
+		if !test.fail {
-			t.Errorf("Unexpected error for %q: %s", test.input, err)
+			require.NoError(t, err, "Unexpected error for %q", test.input)
+			require.Equal(t, test.result, d, "Expected duration %v for input %q but got %v", test.result, test.input, d)
 			continue
 		}
-		if err == nil && test.fail {
+		require.Error(t, err, "Expected error for %q but got none", test.input)
-			t.Errorf("Expected error for %q but got none", test.input)
-			continue
-		}
-		if !test.fail && d != test.result {
-			t.Errorf("Expected duration %v for input %q but got %v", test.result, test.input, d)
-		}
 	}
 }
 
@@ -3497,18 +3448,11 @@ func TestOptionsMethod(t *testing.T) {
 	defer s.Close()
 
 	req, err := http.NewRequest("OPTIONS", s.URL+"/any_path", nil)
-	if err != nil {
+	require.NoError(t, err, "Error creating OPTIONS request")
-		t.Fatalf("Error creating OPTIONS request: %s", err)
-	}
 	client := &http.Client{}
 	resp, err := client.Do(req)
-	if err != nil {
+	require.NoError(t, err, "Error executing OPTIONS request")
-		t.Fatalf("Error executing OPTIONS request: %s", err)
+	require.Equal(t, http.StatusNoContent, resp.StatusCode)
-	}
-
-	if resp.StatusCode != http.StatusNoContent {
-		t.Fatalf("Expected status %d, got %d", http.StatusNoContent, resp.StatusCode)
-	}
 }
 
 func TestTSDBStatus(t *testing.T) {
@@ -3547,9 +3491,7 @@ func TestTSDBStatus(t *testing.T) {
 			api := &API{db: tc.db, gatherer: prometheus.DefaultGatherer}
 			endpoint := tc.endpoint(api)
 			req, err := http.NewRequest(tc.method, fmt.Sprintf("?%s", tc.values.Encode()), nil)
-			if err != nil {
+			require.NoError(t, err, "Error when creating test request")
-				t.Fatalf("Error when creating test request: %s", err)
-			}
 			res := endpoint(req)
 			assertAPIError(t, res.err, tc.errType)
 		})

web/api/v1/json_codec_test.go

@@ -134,14 +134,14 @@ func TestJsonCodec_Encode(t *testing.T) {
 					SeriesLabels: labels.FromStrings("foo", "bar"),
 					Exemplars: []exemplar.Exemplar{
 						{
-							Labels: labels.FromStrings("traceID", "abc"),
+							Labels: labels.FromStrings("trace_id", "abc"),
 							Value:  100.123,
 							Ts:     1234,
 						},
 					},
 				},
 			},
-			expected: `{"status":"success","data":[{"seriesLabels":{"foo":"bar"},"exemplars":[{"labels":{"traceID":"abc"},"value":"100.123","timestamp":1.234}]}]}`,
+			expected: `{"status":"success","data":[{"seriesLabels":{"foo":"bar"},"exemplars":[{"labels":{"trace_id":"abc"},"value":"100.123","timestamp":1.234}]}]}`,
 		},
 		{
 			response: []exemplar.QueryResult{
@@ -149,14 +149,14 @@ func TestJsonCodec_Encode(t *testing.T) {
 					SeriesLabels: labels.FromStrings("foo", "bar"),
 					Exemplars: []exemplar.Exemplar{
 						{
-							Labels: labels.FromStrings("traceID", "abc"),
+							Labels: labels.FromStrings("trace_id", "abc"),
 							Value:  math.Inf(1),
 							Ts:     1234,
 						},
 					},
 				},
 			},
-			expected: `{"status":"success","data":[{"seriesLabels":{"foo":"bar"},"exemplars":[{"labels":{"traceID":"abc"},"value":"+Inf","timestamp":1.234}]}]}`,
+			expected: `{"status":"success","data":[{"seriesLabels":{"foo":"bar"},"exemplars":[{"labels":{"trace_id":"abc"},"value":"+Inf","timestamp":1.234}]}]}`,
 		},
 	}
 

web/federate.go

@@ -189,8 +189,9 @@ Loop:
 	)
 	for _, s := range vec {
 		isHistogram := s.H != nil
+		formatType := format.FormatType()
 		if isHistogram &&
-			format != expfmt.FmtProtoDelim && format != expfmt.FmtProtoText && format != expfmt.FmtProtoCompact {
+			formatType != expfmt.TypeProtoDelim && formatType != expfmt.TypeProtoText && formatType != expfmt.TypeProtoCompact {
 			// Can't serve the native histogram.
 			// TODO(codesome): Serve them when other protocols get the native histogram support.
 			continue

web/federate_test.go

@@ -37,6 +37,7 @@ import (
 	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/tsdb"
 	"github.com/prometheus/prometheus/util/teststorage"
+	"github.com/prometheus/prometheus/util/testutil"
 )
 
 var scenarios = map[string]struct {
@@ -427,5 +428,5 @@ func TestFederationWithNativeHistograms(t *testing.T) {
 	// TODO(codesome): Once PromQL is able to set the CounterResetHint on histograms,
 	// test it with switching histogram types for metric families.
 	require.Equal(t, 4, metricFamilies)
-	require.Equal(t, expVec, actVec)
+	testutil.RequireEqual(t, expVec, actVec)
 }

web/ui/module/codemirror-promql/src/complete/promql.terms.ts

@@ -215,6 +215,12 @@ export const functionIdentifierTerms = [
     info: 'Round down values of input series to nearest integer',
     type: 'function',
   },
+  {
+    label: 'histogram_avg',
+    detail: 'function',
+    info: 'Return the average of observations from a native histogram (experimental feature)',
+    type: 'function',
+  },
   {
     label: 'histogram_count',
     detail: 'function',

web/ui/module/codemirror-promql/src/parser/parser.test.ts

@@ -757,6 +757,18 @@ describe('promql operations', () => {
       expectedValueType: ValueType.vector,
       expectedDiag: [],
     },
+    {
+      expr:
+        'histogram_avg(                                           # Root of the query, final result, returns the average of observations.\n' +
+        '  sum by(method, path) (                                 # Argument to histogram_avg(), an aggregated histogram.\n' +
+        '    rate(                                                # Argument to sum(), the per-second increase of a histogram over 5m.\n' +
+        '      demo_api_request_duration_seconds{job="demo"}[5m]  # Argument to rate(), a vector of sparse histogram series over the last 5m.\n' +
+        '    )\n' +
+        '  )\n' +
+        ')',
+      expectedValueType: ValueType.vector,
+      expectedDiag: [],
+    },
     {
       expr:
         'histogram_stddev(                                           # Root of the query, final result, returns the standard deviation of observations.\n' +

web/ui/module/codemirror-promql/src/types/function.ts

@@ -39,6 +39,7 @@ import {
   Deriv,
   Exp,
   Floor,
+  HistogramAvg,
   HistogramCount,
   HistogramFraction,
   HistogramQuantile,
@@ -269,6 +270,12 @@ const promqlFunctions: { [key: number]: PromQLFunction } = {
     variadic: 0,
     returnType: ValueType.vector,
   },
+  [HistogramAvg]: {
+    name: 'histogram_avg',
+    argTypes: [ValueType.vector],
+    variadic: 0,
+    returnType: ValueType.vector,
+  },
   [HistogramCount]: {
     name: 'histogram_count',
     argTypes: [ValueType.vector],

web/ui/module/lezer-promql/src/highlight.js

@@ -20,7 +20,7 @@ export const promQLHighLight = styleTags({
     NumberLiteral: tags.number,
     Duration: tags.number,
     Identifier: tags.variableName,
-    'Abs Absent AbsentOverTime Acos Acosh Asin Asinh Atan Atanh AvgOverTime Ceil Changes Clamp ClampMax ClampMin Cos Cosh CountOverTime DaysInMonth DayOfMonth DayOfWeek DayOfYear Deg Delta Deriv Exp Floor HistogramCount HistogramFraction HistogramQuantile HistogramSum HoltWinters Hour Idelta Increase Irate LabelReplace LabelJoin LastOverTime Ln Log10 Log2 MaxOverTime MinOverTime Minute Month Pi PredictLinear PresentOverTime QuantileOverTime Rad Rate Resets Round Scalar Sgn Sin Sinh Sort SortDesc SortByLabel SortByLabelDesc Sqrt StddevOverTime StdvarOverTime SumOverTime Tan Tanh Time Timestamp Vector Year':
+    'Abs Absent AbsentOverTime Acos Acosh Asin Asinh Atan Atanh AvgOverTime Ceil Changes Clamp ClampMax ClampMin Cos Cosh CountOverTime DaysInMonth DayOfMonth DayOfWeek DayOfYear Deg Delta Deriv Exp Floor HistogramAvg HistogramCount HistogramFraction HistogramQuantile HistogramSum HoltWinters Hour Idelta Increase Irate LabelReplace LabelJoin LastOverTime Ln Log10 Log2 MaxOverTime MinOverTime Minute Month Pi PredictLinear PresentOverTime QuantileOverTime Rad Rate Resets Round Scalar Sgn Sin Sinh Sort SortDesc SortByLabel SortByLabelDesc Sqrt StddevOverTime StdvarOverTime SumOverTime Tan Tanh Time Timestamp Vector Year':
         tags.function(tags.variableName),
     'Avg Bottomk Count Count_values Group Max Min Quantile Stddev Stdvar Sum Topk': tags.operatorKeyword,
     'By Without Bool On Ignoring GroupLeft GroupRight Offset Start End': tags.modifier,

web/ui/module/lezer-promql/src/promql.grammar

@@ -138,6 +138,7 @@ FunctionIdentifier {
   HistogramStdDev |
   HistogramStdVar |
   HistogramSum |
+  HistogramAvg |
   HoltWinters |
   Hour |
   Idelta |
@@ -364,6 +365,7 @@ NumberLiteral  {
   Deriv { condFn<"deriv"> }
   Exp { condFn<"exp"> }
   Floor { condFn<"floor"> }
+  HistogramAvg { condFn<"histogram_avg"> }
   HistogramCount { condFn<"histogram_count"> }
   HistogramFraction { condFn<"histogram_fraction"> }
   HistogramQuantile { condFn<"histogram_quantile"> }

web/ui/react-app/src/pages/graph/Graph.test.tsx

@@ -80,7 +80,7 @@ describe('Graph', () => {
             exemplars: [
               {
                 labels: {
-                  traceID: '12345',
+                  trace_id: '12345',
                 },
                 timestamp: 1572130580,
                 value: '9',

web/web_test.go

@@ -440,7 +440,7 @@ func TestShutdownWithStaleConnection(t *testing.T) {
 	select {
 	case <-closed:
 	case <-time.After(timeout + 5*time.Second):
-		t.Fatalf("Server still running after read timeout.")
+		require.FailNow(t, "Server still running after read timeout.")
 	}
 }
 
@@ -502,7 +502,7 @@ func TestHandleMultipleQuitRequests(t *testing.T) {
 	select {
 	case <-closed:
 	case <-time.After(5 * time.Second):
-		t.Fatalf("Server still running after 5 seconds.")
+		require.FailNow(t, "Server still running after 5 seconds.")
 	}
 }