mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2024-11-14 17:44:06 -08:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

Merge pull request #13623 from prometheus/njpm/rw2-sync-main

Browse source 
remote write 2.0: sync with `main` branch
This commit is contained in:
Nico Pazos 2024-02-21 11:36:10 -03:00 committed by GitHub
parent 552620a8ec 5644522b7c
commit 6d86554cf3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
100 changed files with 3201 additions and 2229 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
Makefile.common
View file

@ -62,10 +62,10 @@ SKIP_GOLANGCI_LINT :=
GOLANGCI_LINT :=
GOLANGCI_LINT_OPTS ?=
GOLANGCI_LINT_VERSION ?= v1.55.2
# golangci-lint only supports linux, darwin and windows platforms on i386/amd64.
# golangci-lint only supports linux, darwin and windows platforms on i386/amd64/arm64.
# windows isn't included here because of the path separator being different.
ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin))
ifeq ($(GOHOSTARCH),$(filter $(GOHOSTARCH),amd64 i386))
ifeq ($(GOHOSTARCH),$(filter $(GOHOSTARCH),amd64 i386 arm64))
# If we're in CI and there is an Actions file, that means the linter
# is being run in Actions, so we don't need to run it here.
ifneq (,$(SKIP_GOLANGCI_LINT))

2
README.md
View file

@ -149,7 +149,7 @@ We are publishing our Remote Write protobuf independently at
You can use that as a library:
```shell
go get go.buf.build/protocolbuffers/go/prometheus/prometheus
go get buf.build/gen/go/prometheus/prometheus/protocolbuffers/go@latest
```
This is experimental.

41
cmd/prometheus/main_test.go
View file

@ -126,12 +126,9 @@ func TestFailedStartupExitCode(t *testing.T) {
require.Error(t, err)
var exitError *exec.ExitError
if errors.As(err, &exitError) {
status := exitError.Sys().(syscall.WaitStatus)
require.Equal(t, expectedExitStatus, status.ExitStatus())
} else {
t.Errorf("unable to retrieve the exit status for prometheus: %v", err)
}
require.ErrorAs(t, err, &exitError)
status := exitError.Sys().(syscall.WaitStatus)
require.Equal(t, expectedExitStatus, status.ExitStatus())
}
type senderFunc func(alerts ...*notifier.Alert)
@ -194,9 +191,7 @@ func TestSendAlerts(t *testing.T) {
tc := tc
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
senderFunc := senderFunc(func(alerts ...*notifier.Alert) {
if len(tc.in) == 0 {
t.Fatalf("sender called with 0 alert")
}
require.NotEmpty(t, tc.in, "sender called with 0 alert")
require.Equal(t, tc.exp, alerts)
})
rules.SendAlerts(senderFunc, "http://localhost:9090")(context.TODO(), "up", tc.in...)
@ -228,7 +223,7 @@ func TestWALSegmentSizeBounds(t *testing.T) {
go func() { done <- prom.Wait() }()
select {
case err := <-done:
t.Errorf("prometheus should be still running: %v", err)
require.Fail(t, "prometheus should be still running: %v", err)
case <-time.After(startupTime):
prom.Process.Kill()
<-done
@ -239,12 +234,9 @@ func TestWALSegmentSizeBounds(t *testing.T) {
err = prom.Wait()
require.Error(t, err)
var exitError *exec.ExitError
if errors.As(err, &exitError) {
status := exitError.Sys().(syscall.WaitStatus)
require.Equal(t, expectedExitStatus, status.ExitStatus())
} else {
t.Errorf("unable to retrieve the exit status for prometheus: %v", err)
}
require.ErrorAs(t, err, &exitError)
status := exitError.Sys().(syscall.WaitStatus)
require.Equal(t, expectedExitStatus, status.ExitStatus())
}
}
@ -274,7 +266,7 @@ func TestMaxBlockChunkSegmentSizeBounds(t *testing.T) {
go func() { done <- prom.Wait() }()
select {
case err := <-done:
t.Errorf("prometheus should be still running: %v", err)
require.Fail(t, "prometheus should be still running: %v", err)
case <-time.After(startupTime):
prom.Process.Kill()
<-done
@ -285,12 +277,9 @@ func TestMaxBlockChunkSegmentSizeBounds(t *testing.T) {
err = prom.Wait()
require.Error(t, err)
var exitError *exec.ExitError
if errors.As(err, &exitError) {
status := exitError.Sys().(syscall.WaitStatus)
require.Equal(t, expectedExitStatus, status.ExitStatus())
} else {
t.Errorf("unable to retrieve the exit status for prometheus: %v", err)
}
require.ErrorAs(t, err, &exitError)
status := exitError.Sys().(syscall.WaitStatus)
require.Equal(t, expectedExitStatus, status.ExitStatus())
}
}
@ -347,10 +336,8 @@ func getCurrentGaugeValuesFor(t *testing.T, reg prometheus.Gatherer, metricNames
}
require.Len(t, g.GetMetric(), 1)
if _, ok := res[m]; ok {
t.Error("expected only one metric family for", m)
t.FailNow()
}
_, ok := res[m]
require.False(t, ok, "expected only one metric family for", m)
res[m] = *g.GetMetric()[0].GetGauge().Value
}
}

21
cmd/prometheus/main_unix_test.go
View file

@ -23,6 +23,8 @@ import (
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/util/testutil"
)
@ -37,9 +39,7 @@ func TestStartupInterrupt(t *testing.T) {
prom := exec.Command(promPath, "-test.main", "--config.file="+promConfig, "--storage.tsdb.path="+t.TempDir(), "--web.listen-address=0.0.0.0"+port)
err := prom.Start()
if err != nil {
t.Fatalf("execution error: %v", err)
}
require.NoError(t, err)
done := make(chan error, 1)
go func() {
@ -68,14 +68,11 @@ Loop:
time.Sleep(500 * time.Millisecond)
}
if !startedOk {
t.Fatal("prometheus didn't start in the specified timeout")
}
switch err := prom.Process.Kill(); {
case err == nil:
t.Errorf("prometheus didn't shutdown gracefully after sending the Interrupt signal")
case stoppedErr != nil && stoppedErr.Error() != "signal: interrupt":
// TODO: find a better way to detect when the process didn't exit as expected!
t.Errorf("prometheus exited with an unexpected error: %v", stoppedErr)
require.True(t, startedOk, "prometheus didn't start in the specified timeout")
err = prom.Process.Kill()
require.Error(t, err, "prometheus didn't shutdown gracefully after sending the Interrupt signal")
// TODO - find a better way to detect when the process didn't exit as expected!
if stoppedErr != nil {
require.EqualError(t, stoppedErr, "signal: interrupt", "prometheus exit")
}
}

3
cmd/promtool/backfill_test.go
View file

@ -26,6 +26,7 @@ import (
"github.com/prometheus/prometheus/storage"
"github.com/prometheus/prometheus/tsdb"
"github.com/prometheus/prometheus/tsdb/chunkenc"
"github.com/prometheus/prometheus/util/testutil"
)
type backfillSample struct {
@ -76,7 +77,7 @@ func testBlocks(t *testing.T, db *tsdb.DB, expectedMinTime, expectedMaxTime, exp
allSamples := queryAllSeries(t, q, expectedMinTime, expectedMaxTime)
sortSamples(allSamples)
sortSamples(expectedSamples)
require.Equal(t, expectedSamples, allSamples, "did not create correct samples")
testutil.RequireEqual(t, expectedSamples, allSamples, "did not create correct samples")
if len(allSamples) > 0 {
require.Equal(t, expectedMinTime, allSamples[0].Timestamp, "timestamp of first sample is not the expected minimum time")

4
cmd/promtool/sd.go
View file

@ -18,10 +18,10 @@ import (
"encoding/json"
"fmt"
"os"
"reflect"
"time"
"github.com/go-kit/log"
"github.com/google/go-cmp/cmp"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/prometheus/config"
@ -153,7 +153,7 @@ func getSDCheckResult(targetGroups []*targetgroup.Group, scrapeConfig *config.Sc
duplicateRes := false
for _, sdCheckRes := range sdCheckResults {
if reflect.DeepEqual(sdCheckRes, result) {
if cmp.Equal(sdCheckRes, result, cmp.Comparer(labels.Equal)) {
duplicateRes = true
break
}

3
cmd/promtool/sd_test.go
View file

@ -23,6 +23,7 @@ import (
"github.com/prometheus/prometheus/discovery/targetgroup"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/model/relabel"
"github.com/prometheus/prometheus/util/testutil"
"github.com/stretchr/testify/require"
)
@ -69,5 +70,5 @@ func TestSDCheckResult(t *testing.T) {
},
}
require.Equal(t, expectedSDCheckResult, getSDCheckResult(targetGroups, scrapeConfig, true))
testutil.RequireEqual(t, expectedSDCheckResult, getSDCheckResult(targetGroups, scrapeConfig, true))
}

6
cmd/promtool/unittest.go
View file

@ -20,13 +20,13 @@ import (
"fmt"
"os"
"path/filepath"
"reflect"
"sort"
"strconv"
"strings"
"time"
"github.com/go-kit/log"
"github.com/google/go-cmp/cmp"
"github.com/grafana/regexp"
"github.com/nsf/jsondiff"
"github.com/prometheus/common/model"
@ -340,7 +340,7 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
sort.Sort(gotAlerts)
sort.Sort(expAlerts)
if !reflect.DeepEqual(expAlerts, gotAlerts) {
if !cmp.Equal(expAlerts, gotAlerts, cmp.Comparer(labels.Equal)) {
var testName string
if tg.TestGroupName != "" {
testName = fmt.Sprintf(" name: %s,\n", tg.TestGroupName)
@ -448,7 +448,7 @@ Outer:
sort.Slice(gotSamples, func(i, j int) bool {
return labels.Compare(gotSamples[i].Labels, gotSamples[j].Labels) <= 0
})
if !reflect.DeepEqual(expSamples, gotSamples) {
if !cmp.Equal(expSamples, gotSamples, cmp.Comparer(labels.Equal)) {
errs = append(errs, fmt.Errorf(" expr: %q, time: %s,\n exp: %v\n got: %v", testCase.Expr,
testCase.EvalTime.String(), parsedSamplesString(expSamples), parsedSamplesString(gotSamples)))
}

7
cmd/promtool/unittest_test.go
View file

@ -16,6 +16,8 @@ package main
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/promql"
)
@ -178,9 +180,8 @@ func TestRulesUnitTestRun(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := RulesUnitTest(tt.queryOpts, tt.args.run, false, tt.args.files...); got != tt.want {
t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want)
}
got := RulesUnitTest(tt.queryOpts, tt.args.run, false, tt.args.files...)
require.Equal(t, tt.want, got)
})
}
}

7
config/config_test.go
View file

@ -58,6 +58,7 @@ import (
"github.com/prometheus/prometheus/discovery/zookeeper"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/model/relabel"
"github.com/prometheus/prometheus/util/testutil"
)
func mustParseURL(u string) *config.URL {
@ -2037,16 +2038,16 @@ func TestExpandExternalLabels(t *testing.T) {
c, err := LoadFile("testdata/external_labels.good.yml", false, false, log.NewNopLogger())
require.NoError(t, err)
require.Equal(t, labels.FromStrings("bar", "foo", "baz", "foo${TEST}bar", "foo", "${TEST}", "qux", "foo$${TEST}", "xyz", "foo$$bar"), c.GlobalConfig.ExternalLabels)
testutil.RequireEqual(t, labels.FromStrings("bar", "foo", "baz", "foo${TEST}bar", "foo", "${TEST}", "qux", "foo$${TEST}", "xyz", "foo$$bar"), c.GlobalConfig.ExternalLabels)
c, err = LoadFile("testdata/external_labels.good.yml", false, true, log.NewNopLogger())
require.NoError(t, err)
require.Equal(t, labels.FromStrings("bar", "foo", "baz", "foobar", "foo", "", "qux", "foo${TEST}", "xyz", "foo$bar"), c.GlobalConfig.ExternalLabels)
testutil.RequireEqual(t, labels.FromStrings("bar", "foo", "baz", "foobar", "foo", "", "qux", "foo${TEST}", "xyz", "foo$bar"), c.GlobalConfig.ExternalLabels)
os.Setenv("TEST", "TestValue")
c, err = LoadFile("testdata/external_labels.good.yml", false, true, log.NewNopLogger())
require.NoError(t, err)
require.Equal(t, labels.FromStrings("bar", "foo", "baz", "fooTestValuebar", "foo", "TestValue", "qux", "foo${TEST}", "xyz", "foo$bar"), c.GlobalConfig.ExternalLabels)
testutil.RequireEqual(t, labels.FromStrings("bar", "foo", "baz", "fooTestValuebar", "foo", "TestValue", "qux", "foo${TEST}", "xyz", "foo$bar"), c.GlobalConfig.ExternalLabels)
}
func TestAgentMode(t *testing.T) {

65
discovery/consul/consul_test.go
View file

@ -56,15 +56,11 @@ func TestConfiguredService(t *testing.T) {
metrics := NewTestMetrics(t, conf, prometheus.NewRegistry())
consulDiscovery, err := NewDiscovery(conf, nil, metrics)
if err != nil {
t.Errorf("Unexpected error when initializing discovery %v", err)
}
if !consulDiscovery.shouldWatch("configuredServiceName", []string{""}) {
t.Errorf("Expected service %s to be watched", "configuredServiceName")
}
if consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{""}) {
t.Errorf("Expected service %s to not be watched", "nonConfiguredServiceName")
}
require.NoError(t, err, "when initializing discovery")
require.True(t, consulDiscovery.shouldWatch("configuredServiceName", []string{""}),
"Expected service %s to be watched", "configuredServiceName")
require.False(t, consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{""}),
"Expected service %s to not be watched", "nonConfiguredServiceName")
}
func TestConfiguredServiceWithTag(t *testing.T) {
@ -76,21 +72,18 @@ func TestConfiguredServiceWithTag(t *testing.T) {
metrics := NewTestMetrics(t, conf, prometheus.NewRegistry())
consulDiscovery, err := NewDiscovery(conf, nil, metrics)
if err != nil {
t.Errorf("Unexpected error when initializing discovery %v", err)
}
if consulDiscovery.shouldWatch("configuredServiceName", []string{""}) {
t.Errorf("Expected service %s to not be watched without tag", "configuredServiceName")
}
if !consulDiscovery.shouldWatch("configuredServiceName", []string{"http"}) {
t.Errorf("Expected service %s to be watched with tag %s", "configuredServiceName", "http")
}
if consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{""}) {
t.Errorf("Expected service %s to not be watched without tag", "nonConfiguredServiceName")
}
if consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{"http"}) {
t.Errorf("Expected service %s to not be watched with tag %s", "nonConfiguredServiceName", "http")
}
require.NoError(t, err, "when initializing discovery")
require.False(t, consulDiscovery.shouldWatch("configuredServiceName", []string{""}),
"Expected service %s to not be watched without tag", "configuredServiceName")
require.True(t, consulDiscovery.shouldWatch("configuredServiceName", []string{"http"}),
"Expected service %s to be watched with tag %s", "configuredServiceName", "http")
require.False(t, consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{""}),
"Expected service %s to not be watched without tag", "nonConfiguredServiceName")
require.False(t, consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{"http"}),
"Expected service %s to not be watched with tag %s", "nonConfiguredServiceName", "http")
}
func TestConfiguredServiceWithTags(t *testing.T) {
@ -173,13 +166,10 @@ func TestConfiguredServiceWithTags(t *testing.T) {
metrics := NewTestMetrics(t, tc.conf, prometheus.NewRegistry())
consulDiscovery, err := NewDiscovery(tc.conf, nil, metrics)
if err != nil {
t.Errorf("Unexpected error when initializing discovery %v", err)
}
require.NoError(t, err, "when initializing discovery")
ret := consulDiscovery.shouldWatch(tc.serviceName, tc.serviceTags)
if ret != tc.shouldWatch {
t.Errorf("Expected should watch? %t, got %t. Watched service and tags: %s %+v, input was %s %+v", tc.shouldWatch, ret, tc.conf.Services, tc.conf.ServiceTags, tc.serviceName, tc.serviceTags)
}
require.Equal(t, tc.shouldWatch, ret, "Watched service and tags: %s %+v, input was %s %+v",
tc.conf.Services, tc.conf.ServiceTags, tc.serviceName, tc.serviceTags)
}
}
@ -189,12 +179,8 @@ func TestNonConfiguredService(t *testing.T) {
metrics := NewTestMetrics(t, conf, prometheus.NewRegistry())
consulDiscovery, err := NewDiscovery(conf, nil, metrics)
if err != nil {
t.Errorf("Unexpected error when initializing discovery %v", err)
}
if !consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{""}) {
t.Errorf("Expected service %s to be watched", "nonConfiguredServiceName")
}
require.NoError(t, err, "when initializing discovery")
require.True(t, consulDiscovery.shouldWatch("nonConfiguredServiceName", []string{""}), "Expected service %s to be watched", "nonConfiguredServiceName")
}
const (
@ -502,13 +488,10 @@ oauth2:
var config SDConfig
err := config.UnmarshalYAML(unmarshal([]byte(test.config)))
if err != nil {
require.Equalf(t, err.Error(), test.errMessage, "Expected error '%s', got '%v'", test.errMessage, err)
return
}
if test.errMessage != "" {
t.Errorf("Expected error %s, got none", test.errMessage)
require.EqualError(t, err, test.errMessage)
return
}
require.Empty(t, test.errMessage, "Expected error.")
require.Equal(t, test.expected, config)
})

4
discovery/file/file_test.go
View file

@ -358,9 +358,7 @@ func TestInvalidFile(t *testing.T) {
// Verify that we've received nothing.
time.Sleep(defaultWait)
if runner.lastReceive().After(now) {
t.Fatalf("unexpected targets received: %v", runner.targets())
}
require.False(t, runner.lastReceive().After(now), "unexpected targets received: %v", runner.targets())
})
}
}

10
discovery/kubernetes/kubernetes_test.go
View file

@ -131,14 +131,8 @@ func (d k8sDiscoveryTest) Run(t *testing.T) {
go readResultWithTimeout(t, ch, d.expectedMaxItems, time.Second, resChan)
dd, ok := d.discovery.(hasSynced)
if !ok {
t.Errorf("discoverer does not implement hasSynced interface")
return
}
if !cache.WaitForCacheSync(ctx.Done(), dd.hasSynced) {
t.Errorf("discoverer failed to sync: %v", dd)
return
}
require.True(t, ok, "discoverer does not implement hasSynced interface")
require.True(t, cache.WaitForCacheSync(ctx.Done(), dd.hasSynced), "discoverer failed to sync: %v", dd)
if d.afterStart != nil {
d.afterStart()

55
discovery/manager_test.go
View file

@ -694,7 +694,7 @@ func TestTargetUpdatesOrder(t *testing.T) {
for x := 0; x < totalUpdatesCount; x++ {
select {
case <-ctx.Done():
t.Fatalf("%d: no update arrived within the timeout limit", x)
require.FailNow(t, "%d: no update arrived within the timeout limit", x)
case tgs := <-provUpdates:
discoveryManager.updateGroup(poolKey{setName: strconv.Itoa(i), provider: tc.title}, tgs)
for _, got := range discoveryManager.allGroups() {
@ -756,10 +756,8 @@ func verifySyncedPresence(t *testing.T, tGroups map[string][]*targetgroup.Group,
func verifyPresence(t *testing.T, tSets map[poolKey]map[string]*targetgroup.Group, poolKey poolKey, label string, present bool) {
t.Helper()
if _, ok := tSets[poolKey]; !ok {
t.Fatalf("'%s' should be present in Pool keys: %v", poolKey, tSets)
return
}
_, ok := tSets[poolKey]
require.True(t, ok, "'%s' should be present in Pool keys: %v", poolKey, tSets)
match := false
var mergedTargets string
@ -776,7 +774,7 @@ func verifyPresence(t *testing.T, tSets map[poolKey]map[string]*targetgroup.Grou
if !present {
msg = "not"
}
t.Fatalf("%q should %s be present in Targets labels: %q", label, msg, mergedTargets)
require.FailNow(t, "%q should %s be present in Targets labels: %q", label, msg, mergedTargets)
}
}
@ -1088,22 +1086,14 @@ func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
syncedTargets = <-discoveryManager.SyncCh()
p = pk("static", "prometheus", 1)
targetGroups, ok := discoveryManager.targets[p]
if !ok {
t.Fatalf("'%v' should be present in target groups", p)
}
require.True(t, ok, "'%v' should be present in target groups", p)
group, ok := targetGroups[""]
if !ok {
t.Fatalf("missing '' key in target groups %v", targetGroups)
}
require.True(t, ok, "missing '' key in target groups %v", targetGroups)
if len(group.Targets) != 0 {
t.Fatalf("Invalid number of targets: expected 0, got %d", len(group.Targets))
}
require.Empty(t, group.Targets, "Invalid number of targets.")
require.Len(t, syncedTargets, 1)
require.Len(t, syncedTargets["prometheus"], 1)
if lbls := syncedTargets["prometheus"][0].Labels; lbls != nil {
t.Fatalf("Unexpected Group: expected nil Labels, got %v", lbls)
}
require.Nil(t, syncedTargets["prometheus"][0].Labels)
}
func TestIdenticalConfigurationsAreCoalesced(t *testing.T) {
@ -1131,9 +1121,7 @@ func TestIdenticalConfigurationsAreCoalesced(t *testing.T) {
syncedTargets := <-discoveryManager.SyncCh()
verifyPresence(t, discoveryManager.targets, pk("static", "prometheus", 0), "{__address__=\"foo:9090\"}", true)
verifyPresence(t, discoveryManager.targets, pk("static", "prometheus2", 0), "{__address__=\"foo:9090\"}", true)
if len(discoveryManager.providers) != 1 {
t.Fatalf("Invalid number of providers: expected 1, got %d", len(discoveryManager.providers))
}
require.Len(t, discoveryManager.providers, 1, "Invalid number of providers.")
require.Len(t, syncedTargets, 2)
verifySyncedPresence(t, syncedTargets, "prometheus", "{__address__=\"foo:9090\"}", true)
require.Len(t, syncedTargets["prometheus"], 1)
@ -1231,9 +1219,7 @@ func TestGaugeFailedConfigs(t *testing.T) {
<-discoveryManager.SyncCh()
failedCount := client_testutil.ToFloat64(discoveryManager.metrics.FailedConfigs)
if failedCount != 3 {
t.Fatalf("Expected to have 3 failed configs, got: %v", failedCount)
}
require.Equal(t, 3.0, failedCount, "Expected to have 3 failed configs.")
c["prometheus"] = Configs{
staticConfig("foo:9090"),
@ -1242,9 +1228,7 @@ func TestGaugeFailedConfigs(t *testing.T) {
<-discoveryManager.SyncCh()
failedCount = client_testutil.ToFloat64(discoveryManager.metrics.FailedConfigs)
if failedCount != 0 {
t.Fatalf("Expected to get no failed config, got: %v", failedCount)
}
require.Equal(t, 0.0, failedCount, "Expected to get no failed config.")
}
func TestCoordinationWithReceiver(t *testing.T) {
@ -1388,19 +1372,14 @@ func TestCoordinationWithReceiver(t *testing.T) {
time.Sleep(expected.delay)
select {
case <-ctx.Done():
t.Fatalf("step %d: no update received in the expected timeframe", i)
require.FailNow(t, "step %d: no update received in the expected timeframe", i)
case tgs, ok := <-mgr.SyncCh():
if !ok {
t.Fatalf("step %d: discovery manager channel is closed", i)
}
if len(tgs) != len(expected.tgs) {
t.Fatalf("step %d: target groups mismatch, got: %d, expected: %d\ngot: %#v\nexpected: %#v",
i, len(tgs), len(expected.tgs), tgs, expected.tgs)
}
require.True(t, ok, "step %d: discovery manager channel is closed", i)
require.Equal(t, len(expected.tgs), len(tgs), "step %d: targets mismatch", i)
for k := range expected.tgs {
if _, ok := tgs[k]; !ok {
t.Fatalf("step %d: target group not found: %s\ngot: %#v", i, k, tgs)
}
_, ok := tgs[k]
require.True(t, ok, "step %d: target group not found: %s", i, k)
assertEqualGroups(t, tgs[k], expected.tgs[k])
}
}

395
discovery/marathon/marathon_test.go
View file

@ -69,23 +69,15 @@ func TestMarathonSDHandleError(t *testing.T) {
}
)
tgs, err := testUpdateServices(client)
if !errors.Is(err, errTesting) {
t.Fatalf("Expected error: %s", err)
}
if len(tgs) != 0 {
t.Fatalf("Got group: %s", tgs)
}
require.ErrorIs(t, err, errTesting)
require.Empty(t, tgs, "Expected no target groups.")
}
func TestMarathonSDEmptyList(t *testing.T) {
client := func(_ context.Context, _ *http.Client, _ string) (*appList, error) { return &appList{}, nil }
tgs, err := testUpdateServices(client)
if err != nil {
t.Fatalf("Got error: %s", err)
}
if len(tgs) > 0 {
t.Fatalf("Got group: %v", tgs)
}
require.NoError(t, err)
require.Empty(t, tgs, "Expected no target groups.")
}
func marathonTestAppList(labels map[string]string, runningTasks int) *appList {
@ -119,28 +111,16 @@ func TestMarathonSDSendGroup(t *testing.T) {
return marathonTestAppList(marathonValidLabel, 1), nil
}
tgs, err := testUpdateServices(client)
if err != nil {
t.Fatalf("Got error: %s", err)
}
if len(tgs) != 1 {
t.Fatal("Expected 1 target group, got", len(tgs))
}
require.NoError(t, err)
require.Len(t, tgs, 1, "Expected 1 target group.")
tg := tgs[0]
require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
require.Len(t, tg.Targets, 1, "Expected 1 target.")
if tg.Source != "test-service" {
t.Fatalf("Wrong target group name: %s", tg.Source)
}
if len(tg.Targets) != 1 {
t.Fatalf("Wrong number of targets: %v", tg.Targets)
}
tgt := tg.Targets[0]
if tgt[model.AddressLabel] != "mesos-slave1:31000" {
t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
}
if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" {
t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
}
require.Equal(t, "mesos-slave1:31000", string(tgt[model.AddressLabel]), "Wrong target address.")
require.Equal(t, "yes", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.")
}
func TestMarathonSDRemoveApp(t *testing.T) {
@ -153,40 +133,27 @@ func TestMarathonSDRemoveApp(t *testing.T) {
defer refreshMetrics.Unregister()
md, err := NewDiscovery(cfg, nil, metrics)
if err != nil {
t.Fatalf("%s", err)
}
require.NoError(t, err)
md.appsClient = func(_ context.Context, _ *http.Client, _ string) (*appList, error) {
return marathonTestAppList(marathonValidLabel, 1), nil
}
tgs, err := md.refresh(context.Background())
if err != nil {
t.Fatalf("Got error on first update: %s", err)
}
if len(tgs) != 1 {
t.Fatal("Expected 1 targetgroup, got", len(tgs))
}
require.NoError(t, err, "Got error on first update.")
require.Len(t, tgs, 1, "Expected 1 targetgroup.")
tg1 := tgs[0]
md.appsClient = func(_ context.Context, _ *http.Client, _ string) (*appList, error) {
return marathonTestAppList(marathonValidLabel, 0), nil
}
tgs, err = md.refresh(context.Background())
if err != nil {
t.Fatalf("Got error on second update: %s", err)
}
if len(tgs) != 1 {
t.Fatal("Expected 1 targetgroup, got", len(tgs))
}
require.NoError(t, err, "Got error on second update.")
require.Len(t, tgs, 1, "Expected 1 targetgroup.")
tg2 := tgs[0]
if tg2.Source != tg1.Source {
if len(tg2.Targets) > 0 {
t.Errorf("Got a non-empty target set: %s", tg2.Targets)
}
t.Fatalf("Source is different: %s != %s", tg1.Source, tg2.Source)
}
require.NotEmpty(t, tg2.Targets, "Got a non-empty target set.")
require.Equal(t, tg1.Source, tg2.Source, "Source is different.")
}
func marathonTestAppListWithMultiplePorts(labels map[string]string, runningTasks int) *appList {
@ -221,34 +188,22 @@ func TestMarathonSDSendGroupWithMultiplePort(t *testing.T) {
return marathonTestAppListWithMultiplePorts(marathonValidLabel, 1), nil
}
tgs, err := testUpdateServices(client)
if err != nil {
t.Fatalf("Got error: %s", err)
}
if len(tgs) != 1 {
t.Fatal("Expected 1 target group, got", len(tgs))
}
tg := tgs[0]
require.NoError(t, err)
require.Len(t, tgs, 1, "Expected 1 target group.")
tg := tgs[0]
require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
require.Len(t, tg.Targets, 2, "Wrong number of targets.")
if tg.Source != "test-service" {
t.Fatalf("Wrong target group name: %s", tg.Source)
}
if len(tg.Targets) != 2 {
t.Fatalf("Wrong number of targets: %v", tg.Targets)
}
tgt := tg.Targets[0]
if tgt[model.AddressLabel] != "mesos-slave1:31000" {
t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
}
if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" {
t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
}
require.Equal(t, "mesos-slave1:31000", string(tgt[model.AddressLabel]), "Wrong target address.")
require.Equal(t, "yes", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]),
"Wrong portMappings label from the first port: %s", tgt[model.AddressLabel])
tgt = tg.Targets[1]
if tgt[model.AddressLabel] != "mesos-slave1:32000" {
t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
}
if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
}
require.Equal(t, "mesos-slave1:32000", string(tgt[model.AddressLabel]), "Wrong target address.")
require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]),
"Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
}
func marathonTestZeroTaskPortAppList(labels map[string]string, runningTasks int) *appList {
@ -278,20 +233,12 @@ func TestMarathonZeroTaskPorts(t *testing.T) {
return marathonTestZeroTaskPortAppList(marathonValidLabel, 1), nil
}
tgs, err := testUpdateServices(client)
if err != nil {
t.Fatalf("Got error: %s", err)
}
if len(tgs) != 1 {
t.Fatal("Expected 1 target group, got", len(tgs))
}
tg := tgs[0]
require.NoError(t, err)
require.Len(t, tgs, 1, "Expected 1 target group.")
if tg.Source != "test-service-zero-ports" {
t.Fatalf("Wrong target group name: %s", tg.Source)
}
if len(tg.Targets) != 0 {
t.Fatalf("Wrong number of targets: %v", tg.Targets)
}
tg := tgs[0]
require.Equal(t, "test-service-zero-ports", tg.Source, "Wrong target group name.")
require.Empty(t, tg.Targets, "Wrong number of targets.")
}
func Test500ErrorHttpResponseWithValidJSONBody(t *testing.T) {
@ -306,9 +253,7 @@ func Test500ErrorHttpResponseWithValidJSONBody(t *testing.T) {
defer ts.Close()
// Execute test case and validate behavior.
_, err := testUpdateServices(nil)
if err == nil {
t.Fatalf("Expected error for 5xx HTTP response from marathon server, got nil")
}
require.Error(t, err, "Expected error for 5xx HTTP response from marathon server.")
}
func marathonTestAppListWithPortDefinitions(labels map[string]string, runningTasks int) *appList {
@ -346,40 +291,24 @@ func TestMarathonSDSendGroupWithPortDefinitions(t *testing.T) {
return marathonTestAppListWithPortDefinitions(marathonValidLabel, 1), nil
}
tgs, err := testUpdateServices(client)
if err != nil {
t.Fatalf("Got error: %s", err)
}
if len(tgs) != 1 {
t.Fatal("Expected 1 target group, got", len(tgs))
}
tg := tgs[0]
require.NoError(t, err)
require.Len(t, tgs, 1, "Expected 1 target group.")
tg := tgs[0]
require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
require.Len(t, tg.Targets, 2, "Wrong number of targets.")
if tg.Source != "test-service" {
t.Fatalf("Wrong target group name: %s", tg.Source)
}
if len(tg.Targets) != 2 {
t.Fatalf("Wrong number of targets: %v", tg.Targets)
}
tgt := tg.Targets[0]
if tgt[model.AddressLabel] != "mesos-slave1:1234" {
t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
}
if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
}
if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
}
require.Equal(t, "mesos-slave1:1234", string(tgt[model.AddressLabel]), "Wrong target address.")
require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]),
"Wrong portMappings label from the first port.")
require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]),
"Wrong portDefinitions label from the first port.")
tgt = tg.Targets[1]
if tgt[model.AddressLabel] != "mesos-slave1:5678" {
t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
}
if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
}
if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "yes" {
t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
}
require.Equal(t, "mesos-slave1:5678", string(tgt[model.AddressLabel]), "Wrong target address.")
require.Empty(t, tgt[model.LabelName(portMappingLabelPrefix+"prometheus")], "Wrong portMappings label from the second port.")
require.Equal(t, "yes", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.")
}
func marathonTestAppListWithPortDefinitionsRequirePorts(labels map[string]string, runningTasks int) *appList {
@ -416,40 +345,22 @@ func TestMarathonSDSendGroupWithPortDefinitionsRequirePorts(t *testing.T) {
return marathonTestAppListWithPortDefinitionsRequirePorts(marathonValidLabel, 1), nil
}
tgs, err := testUpdateServices(client)
if err != nil {
t.Fatalf("Got error: %s", err)
}
if len(tgs) != 1 {
t.Fatal("Expected 1 target group, got", len(tgs))
}
tg := tgs[0]
require.NoError(t, err)
require.Len(t, tgs, 1, "Expected 1 target group.")
tg := tgs[0]
require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
require.Len(t, tg.Targets, 2, "Wrong number of targets.")
if tg.Source != "test-service" {
t.Fatalf("Wrong target group name: %s", tg.Source)
}
if len(tg.Targets) != 2 {
t.Fatalf("Wrong number of targets: %v", tg.Targets)
}
tgt := tg.Targets[0]
if tgt[model.AddressLabel] != "mesos-slave1:31000" {
t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
}
if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
}
if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
}
require.Equal(t, "mesos-slave1:31000", string(tgt[model.AddressLabel]), "Wrong target address.")
require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.")
require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.")
tgt = tg.Targets[1]
if tgt[model.AddressLabel] != "mesos-slave1:32000" {
t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
}
if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
}
if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "yes" {
t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
}
require.Equal(t, "mesos-slave1:32000", string(tgt[model.AddressLabel]), "Wrong target address.")
require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.")
require.Equal(t, "yes", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.")
}
func marathonTestAppListWithPorts(labels map[string]string, runningTasks int) *appList {
@ -481,40 +392,22 @@ func TestMarathonSDSendGroupWithPorts(t *testing.T) {
return marathonTestAppListWithPorts(marathonValidLabel, 1), nil
}
tgs, err := testUpdateServices(client)
if err != nil {
t.Fatalf("Got error: %s", err)
}
if len(tgs) != 1 {
t.Fatal("Expected 1 target group, got", len(tgs))
}
tg := tgs[0]
require.NoError(t, err)
require.Len(t, tgs, 1, "Expected 1 target group.")
tg := tgs[0]
require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
require.Len(t, tg.Targets, 2, "Wrong number of targets.")
if tg.Source != "test-service" {
t.Fatalf("Wrong target group name: %s", tg.Source)
}
if len(tg.Targets) != 2 {
t.Fatalf("Wrong number of targets: %v", tg.Targets)
}
tgt := tg.Targets[0]
if tgt[model.AddressLabel] != "mesos-slave1:31000" {
t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
}
if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
}
if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
}
require.Equal(t, "mesos-slave1:31000", string(tgt[model.AddressLabel]), "Wrong target address.")
require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.")
require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.")
tgt = tg.Targets[1]
if tgt[model.AddressLabel] != "mesos-slave1:32000" {
t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
}
if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
}
if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
}
require.Equal(t, "mesos-slave1:32000", string(tgt[model.AddressLabel]), "Wrong target address.")
require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.")
require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.")
}
func marathonTestAppListWithContainerPortMappings(labels map[string]string, runningTasks int) *appList {
@ -555,40 +448,22 @@ func TestMarathonSDSendGroupWithContainerPortMappings(t *testing.T) {
return marathonTestAppListWithContainerPortMappings(marathonValidLabel, 1), nil
}
tgs, err := testUpdateServices(client)
if err != nil {
t.Fatalf("Got error: %s", err)
}
if len(tgs) != 1 {
t.Fatal("Expected 1 target group, got", len(tgs))
}
tg := tgs[0]
require.NoError(t, err)
require.Len(t, tgs, 1, "Expected 1 target group.")
tg := tgs[0]
require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
require.Len(t, tg.Targets, 2, "Wrong number of targets.")
if tg.Source != "test-service" {
t.Fatalf("Wrong target group name: %s", tg.Source)
}
if len(tg.Targets) != 2 {
t.Fatalf("Wrong number of targets: %v", tg.Targets)
}
tgt := tg.Targets[0]
if tgt[model.AddressLabel] != "mesos-slave1:12345" {
t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
}
if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" {
t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
}
if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
}
require.Equal(t, "mesos-slave1:12345", string(tgt[model.AddressLabel]), "Wrong target address.")
require.Equal(t, "yes", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.")
require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.")
tgt = tg.Targets[1]
if tgt[model.AddressLabel] != "mesos-slave1:32000" {
t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
}
if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
}
if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
}
require.Equal(t, "mesos-slave1:32000", string(tgt[model.AddressLabel]), "Wrong target address.")
require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.")
require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.")
}
func marathonTestAppListWithDockerContainerPortMappings(labels map[string]string, runningTasks int) *appList {
@ -629,40 +504,22 @@ func TestMarathonSDSendGroupWithDockerContainerPortMappings(t *testing.T) {
return marathonTestAppListWithDockerContainerPortMappings(marathonValidLabel, 1), nil
}
tgs, err := testUpdateServices(client)
if err != nil {
t.Fatalf("Got error: %s", err)
}
if len(tgs) != 1 {
t.Fatal("Expected 1 target group, got", len(tgs))
}
tg := tgs[0]
require.NoError(t, err)
require.Len(t, tgs, 1, "Expected 1 target group.")
tg := tgs[0]
require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
require.Len(t, tg.Targets, 2, "Wrong number of targets.")
if tg.Source != "test-service" {
t.Fatalf("Wrong target group name: %s", tg.Source)
}
if len(tg.Targets) != 2 {
t.Fatalf("Wrong number of targets: %v", tg.Targets)
}
tgt := tg.Targets[0]
if tgt[model.AddressLabel] != "mesos-slave1:31000" {
t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
}
if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" {
t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
}
if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
}
require.Equal(t, "mesos-slave1:31000", string(tgt[model.AddressLabel]), "Wrong target address.")
require.Equal(t, "yes", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.")
require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.")
tgt = tg.Targets[1]
if tgt[model.AddressLabel] != "mesos-slave1:12345" {
t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
}
if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
}
if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
}
require.Equal(t, "mesos-slave1:12345", string(tgt[model.AddressLabel]), "Wrong target address.")
require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.")
require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.")
}
func marathonTestAppListWithContainerNetworkAndPortMappings(labels map[string]string, runningTasks int) *appList {
@ -707,38 +564,20 @@ func TestMarathonSDSendGroupWithContainerNetworkAndPortMapping(t *testing.T) {
return marathonTestAppListWithContainerNetworkAndPortMappings(marathonValidLabel, 1), nil
}
tgs, err := testUpdateServices(client)
if err != nil {
t.Fatalf("Got error: %s", err)
}
if len(tgs) != 1 {
t.Fatal("Expected 1 target group, got", len(tgs))
}
tg := tgs[0]
require.NoError(t, err)
require.Len(t, tgs, 1, "Expected 1 target group.")
tg := tgs[0]
require.Equal(t, "test-service", tg.Source, "Wrong target group name.")
require.Len(t, tg.Targets, 2, "Wrong number of targets.")
if tg.Source != "test-service" {
t.Fatalf("Wrong target group name: %s", tg.Source)
}
if len(tg.Targets) != 2 {
t.Fatalf("Wrong number of targets: %v", tg.Targets)
}
tgt := tg.Targets[0]
if tgt[model.AddressLabel] != "1.2.3.4:8080" {
t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
}
if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" {
t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
}
if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
}
require.Equal(t, "1.2.3.4:8080", string(tgt[model.AddressLabel]), "Wrong target address.")
require.Equal(t, "yes", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.")
require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.")
tgt = tg.Targets[1]
if tgt[model.AddressLabel] != "1.2.3.4:1234" {
t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
}
if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
}
if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
}
require.Equal(t, "1.2.3.4:1234", string(tgt[model.AddressLabel]), "Wrong target address.")
require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.")
require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.")
}

12
discovery/openstack/mock_test.go
View file

@ -18,6 +18,8 @@ import (
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/require"
)
// SDMock is the interface for the OpenStack mock.
@ -49,15 +51,13 @@ func (m *SDMock) Setup() {
const tokenID = "cbc36478b0bd8e67e89469c7749d4127"
func testMethod(t *testing.T, r *http.Request, expected string) {
if expected != r.Method {
t.Errorf("Request method = %v, expected %v", r.Method, expected)
}
require.Equal(t, expected, r.Method, "Unexpected request method.")
}
func testHeader(t *testing.T, r *http.Request, header, expected string) {
if actual := r.Header.Get(header); expected != actual {
t.Errorf("Header %s = %s, expected %s", header, actual, expected)
}
t.Helper()
actual := r.Header.Get(header)
require.Equal(t, expected, actual, "Unexpected value for request header %s.", header)
}
// HandleVersionsSuccessfully mocks version call.

2
discovery/refresh/refresh_test.go
View file

@ -97,7 +97,7 @@ func TestRefresh(t *testing.T) {
defer tick.Stop()
select {
case <-ch:
t.Fatal("Unexpected target group")
require.FailNow(t, "Unexpected target group")
case <-tick.C:
}
}

5
discovery/zookeeper/zookeeper_test.go
View file

@ -18,6 +18,7 @@ import (
"time"
"github.com/prometheus/common/model"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
)
@ -31,7 +32,5 @@ func TestNewDiscoveryError(t *testing.T) {
time.Second, []string{"/"},
nil,
func(data []byte, path string) (model.LabelSet, error) { return nil, nil })
if err == nil {
t.Fatalf("expected error, got nil")
}
require.Error(t, err)
}

4
docs/feature_flags.md
View file

@ -34,7 +34,7 @@ Activating the remote write receiver via a feature flag is deprecated. Use `--we
[OpenMetrics](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#exemplars) introduces the ability for scrape targets to add exemplars to certain metrics. Exemplars are references to data outside of the MetricSet. A common use case are IDs of program traces.
Exemplar storage is implemented as a fixed size circular buffer that stores exemplars in memory for all series. Enabling this feature will enable the storage of exemplars scraped by Prometheus. The config file block [storage](configuration/configuration.md#configuration-file)/[exemplars](configuration/configuration.md#exemplars) can be used to control the size of circular buffer by # of exemplars. An exemplar with just a `traceID=<jaeger-trace-id>` uses roughly 100 bytes of memory via the in-memory exemplar storage. If the exemplar storage is enabled, we will also append the exemplars to WAL for local persistence (for WAL duration).
Exemplar storage is implemented as a fixed size circular buffer that stores exemplars in memory for all series. Enabling this feature will enable the storage of exemplars scraped by Prometheus. The config file block [storage](configuration/configuration.md#configuration-file)/[exemplars](configuration/configuration.md#exemplars) can be used to control the size of circular buffer by # of exemplars. An exemplar with just a `trace_id=<jaeger-trace-id>` uses roughly 100 bytes of memory via the in-memory exemplar storage. If the exemplar storage is enabled, we will also append the exemplars to WAL for local persistence (for WAL duration).
## Memory snapshot on shutdown
@ -230,4 +230,4 @@ The number of concurrent rule evaluations can be configured with `--rules.max-co
When enabled, Prometheus will store metadata in-memory and keep track of
metadata changes as WAL records on a per-series basis. This must be used if
you are also using remote write 2.0 as it will only gather metadata from the WAL.
you are also using remote write 2.0 as it will only gather metadata from the WAL.

6
docs/querying/api.md
View file

@ -404,7 +404,7 @@ $ curl -g 'http://localhost:9090/api/v1/query_exemplars?query=test_exemplar_metr
"exemplars": [
{
"labels": {
"traceID": "EpTxMJ40fUus7aGY"
"trace_id": "EpTxMJ40fUus7aGY"
},
"value": "6",
"timestamp": 1600096945.479
@ -421,14 +421,14 @@ $ curl -g 'http://localhost:9090/api/v1/query_exemplars?query=test_exemplar_metr
"exemplars": [
{
"labels": {
"traceID": "Olp9XHlq763ccsfa"
"trace_id": "Olp9XHlq763ccsfa"
},
"value": "19",
"timestamp": 1600096955.479
},
{
"labels": {
"traceID": "hCtjygkIHwAN9vs4"
"trace_id": "hCtjygkIHwAN9vs4"
},
"value": "20",
"timestamp": 1600096965.489

113
docs/querying/basics.md
View file

@ -14,7 +14,7 @@ systems via the [HTTP API](api.md).
## Examples
This document is meant as a reference. For learning, it might be easier to
This document is a Prometheus basic language reference. For learning, it may be easier to
start with a couple of [examples](examples.md).
## Expression language data types
@ -28,9 +28,9 @@ evaluate to one of four types:
* **String** - a simple string value; currently unused
Depending on the use-case (e.g. when graphing vs. displaying the output of an
expression), only some of these types are legal as the result from a
expression), only some of these types are legal as the result of a
user-specified expression. For example, an expression that returns an instant
vector is the only type that can be directly graphed.
vector is the only type which can be graphed.
_Notes about the experimental native histograms:_
@ -46,16 +46,15 @@ _Notes about the experimental native histograms:_
### String literals
Strings may be specified as literals in single quotes, double quotes or
backticks.
String literals are designated by single quotes, double quotes or backticks.
PromQL follows the same [escaping rules as
Go](https://golang.org/ref/spec#String_literals). In single or double quotes a
Go](https://golang.org/ref/spec#String_literals). For string literals in single or double quotes, a
backslash begins an escape sequence, which may be followed by `a`, `b`, `f`,
`n`, `r`, `t`, `v` or `\`. Specific characters can be provided using octal
(`\nnn`) or hexadecimal (`\xnn`, `\unnnn` and `\Unnnnnnnn`).
`n`, `r`, `t`, `v` or `\`. Specific characters can be provided using octal
(`\nnn`) or hexadecimal (`\xnn`, `\unnnn` and `\Unnnnnnnn`) notations.
No escaping is processed inside backticks. Unlike Go, Prometheus does not discard newlines inside backticks.
Conversely, escape characters are not parsed in string literals designated by backticks. It is important to note that, unlike Go, Prometheus does not discard newlines inside backticks.
Example:
@ -83,13 +82,17 @@ Examples:
-Inf
NaN
## Time series Selectors
## Time series selectors
Time series selectors are responsible for selecting the times series and raw or inferred sample timestamps and values.
Time series *selectors* are not to be confused with higher level concept of instant and range *queries* that can execute the time series *selectors*. A higher level instant query would evaluate the given selector at one point in time, however the range query would evaluate the selector at multiple different times in between a minimum and maximum timestamp at regular steps.
### Instant vector selectors
Instant vector selectors allow the selection of a set of time series and a
single sample value for each at a given timestamp (instant): in the simplest
form, only a metric name is specified. This results in an instant vector
single sample value for each at a given timestamp (point in time). In the simplest
form, only a metric name is specified, which results in an instant vector
containing elements for all time series that have this metric name.
This example selects all time series that have the `http_requests_total` metric
@ -97,7 +100,7 @@ name:
http_requests_total
It is possible to filter these time series further by appending a comma separated list of label
It is possible to filter these time series further by appending a comma-separated list of label
matchers in curly braces (`{}`).
This example selects only those time series with the `http_requests_total`
@ -124,6 +127,33 @@ For example, this selects all `http_requests_total` time series for `staging`,
Label matchers that match empty label values also select all time series that
do not have the specific label set at all. It is possible to have multiple matchers for the same label name.
For example, given the dataset:
http_requests_total
http_requests_total{replica="rep-a"}
http_requests_total{replica="rep-b"}
http_requests_total{environment="development"}
The query `http_requests_total{environment=""}` would match and return:
http_requests_total
http_requests_total{replica="rep-a"}
http_requests_total{replica="rep-b"}
and would exclude:
http_requests_total{environment="development"}
Multiple matchers can be used for the same label name; they all must pass for a result to be returned.
The query:
http_requests_total{replica!="rep-a",replica=~"rep.*"}
Would then match:
http_requests_total{replica="rep-b"}
Vector selectors must either specify a name or at least one label matcher
that does not match the empty string. The following expression is illegal:
@ -178,11 +208,13 @@ following units:
* `s` - seconds
* `m` - minutes
* `h` - hours
* `d` - days - assuming a day has always 24h
* `w` - weeks - assuming a week has always 7d
* `y` - years - assuming a year has always 365d
* `d` - days - assuming a day always has 24h
* `w` - weeks - assuming a week always has 7d
* `y` - years - assuming a year always has 365d<sup>1</sup>
Time durations can be combined, by concatenation. Units must be ordered from the
<sup>1</sup> For days in a year, the leap day is ignored, and conversely, for a minute, a leap second is ignored.
Time durations can be combined by concatenation. Units must be ordered from the
longest to the shortest. A given unit must only appear once in a time duration.
Here are some examples of valid time durations:
@ -217,8 +249,7 @@ that `http_requests_total` had a week ago:
rate(http_requests_total[5m] offset 1w)
For comparisons with temporal shifts forward in time, a negative offset
can be specified:
When querying for samples in the past, a negative offset will enable temporal comparisons forward in time:
rate(http_requests_total[5m] offset -1w)
@ -249,11 +280,11 @@ The same works for range vectors. This returns the 5-minute rate that
rate(http_requests_total[5m] @ 1609746000)
The `@` modifier supports all representation of float literals described
above within the limits of `int64`. It can also be used along
with the `offset` modifier where the offset is applied relative to the `@`
modifier time irrespective of which modifier is written first.
These 2 queries will produce the same result.
The `@` modifier supports all representations of numeric literals described above.
It works with the `offset` modifier where the offset is applied relative to the `@`
modifier time. The results are the same irrespective of the order of the modifiers.
For example, these two queries will produce the same result:
# offset after @
http_requests_total @ 1609746000 offset 5m
@ -299,33 +330,35 @@ PromQL supports line comments that start with `#`. Example:
### Staleness
When queries are run, timestamps at which to sample data are selected
The timestamps at which to sample data, during a query, are selected
independently of the actual present time series data. This is mainly to support
cases like aggregation (`sum`, `avg`, and so on), where multiple aggregated
time series do not exactly align in time. Because of their independence,
time series do not precisely align in time. Because of their independence,
Prometheus needs to assign a value at those timestamps for each relevant time
series. It does so by simply taking the newest sample before this timestamp.
series. It does so by taking the newest sample before this timestamp within the lookback period.
The lookback period is 5 minutes by default.
If a target scrape or rule evaluation no longer returns a sample for a time
series that was previously present, that time series will be marked as stale.
If a target is removed, its previously returned time series will be marked as
stale soon afterwards.
series that was previously present, this time series will be marked as stale.
If a target is removed, the previously retrieved time series will be marked as
stale soon after removal.
If a query is evaluated at a sampling timestamp after a time series is marked
stale, then no value is returned for that time series. If new samples are
subsequently ingested for that time series, they will be returned as normal.
as stale, then no value is returned for that time series. If new samples are
subsequently ingested for that time series, they will be returned as expected.
If no sample is found (by default) 5 minutes before a sampling timestamp,
no value is returned for that time series at this point in time. This
effectively means that time series "disappear" from graphs at times where their
latest collected sample is older than 5 minutes or after they are marked stale.
A time series will go stale when it is no longer exported, or the target no
longer exists. Such time series will disappear from graphs
at the times of their latest collected sample, and they will not be returned
in queries after they are marked stale.
Staleness will not be marked for time series that have timestamps included in
their scrapes. Only the 5 minute threshold will be applied in that case.
Some exporters, which put their own timestamps on samples, get a different behaviour:
series that stop being exported take the last value for (by default) 5 minutes before
disappearing. The `track_timestamps_staleness` setting can change this.
### Avoiding slow queries and overloads
If a query needs to operate on a very large amount of data, graphing it might
If a query needs to operate on a substantial amount of data, graphing it might
time out or overload the server or browser. Thus, when constructing queries
over unknown data, always start building the query in the tabular view of
Prometheus's expression browser until the result set seems reasonable
@ -336,7 +369,7 @@ rule](../configuration/recording_rules.md#recording-rules).
This is especially relevant for Prometheus's query language, where a bare
metric name selector like `api_http_requests_total` could expand to thousands
of time series with different labels. Also keep in mind that expressions which
of time series with different labels. Also, keep in mind that expressions that
aggregate over many time series will generate load on the server even if the
output is only a small number of time series. This is similar to how it would
be slow to sum all values of a column in a relational database, even if the

28
docs/querying/functions.md
View file

@ -175,6 +175,27 @@ Special cases are:
`floor(v instant-vector)` rounds the sample values of all elements in `v` down
to the nearest integer.
## `histogram_avg()`
_This function only acts on native histograms, which are an experimental
feature. The behavior of this function may change in future versions of
Prometheus, including its removal from PromQL._
`histogram_avg(v instant-vector)` returns the arithmetic average of observed values stored in
a native histogram. Samples that are not native histograms are ignored and do
not show up in the returned vector.
Use `histogram_avg` as demonstrated below to compute the average request duration
over a 5-minute window from a native histogram:
histogram_avg(rate(http_request_duration_seconds[5m]))
Which is equivalent to the following query:
histogram_sum(rate(http_request_duration_seconds[5m]))
/
histogram_count(rate(http_request_duration_seconds[5m]))
## `histogram_count()` and `histogram_sum()`
_Both functions only act on native histograms, which are an experimental
@ -193,13 +214,6 @@ Use `histogram_count` in the following way to calculate a rate of observations
histogram_count(rate(http_request_duration_seconds[10m]))
The additional use of `histogram_sum` enables the calculation of the average of
observed values (in this case corresponding to “average request duration”):
histogram_sum(rate(http_request_duration_seconds[10m]))
/
histogram_count(rate(http_request_duration_seconds[10m]))
## `histogram_fraction()`
_This function only acts on native histograms, which are an experimental

2
documentation/prometheus-mixin/alerts.libsonnet
View file

@ -122,7 +122,7 @@
alert: 'PrometheusNotIngestingSamples',
expr: |||
(
rate(prometheus_tsdb_head_samples_appended_total{%(prometheusSelector)s}[5m]) <= 0
sum without(type) (rate(prometheus_tsdb_head_samples_appended_total{%(prometheusSelector)s}[5m])) <= 0
and
(
sum without(scrape_job) (prometheus_target_metadata_cache_entries{%(prometheusSelector)s}) > 0

4
go.mod
View file

@ -28,6 +28,7 @@ require (
github.com/go-zookeeper/zk v1.0.3
github.com/gogo/protobuf v1.3.2
github.com/golang/snappy v0.0.4
github.com/google/go-cmp v0.6.0
github.com/google/pprof v0.0.0-20240117000934-35fc243c5815
github.com/google/uuid v1.5.0
github.com/gophercloud/gophercloud v1.8.0
@ -51,7 +52,7 @@ require (
github.com/prometheus/alertmanager v0.26.0
github.com/prometheus/client_golang v1.18.0
github.com/prometheus/client_model v0.5.0
github.com/prometheus/common v0.46.0
github.com/prometheus/common v0.47.0
github.com/prometheus/common/assets v0.2.0
github.com/prometheus/common/sigv4 v0.1.0
github.com/prometheus/exporter-toolkit v0.11.0
@ -135,7 +136,6 @@ require (
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/s2a-go v0.1.7 // indirect

4
go.sum
View file

@ -670,8 +670,8 @@ github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8b
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y=
github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ=
github.com/prometheus/common v0.47.0 h1:p5Cz0FNHo7SnWOmWmoRozVcjEp0bIVU8cV7OShpjL1k=
github.com/prometheus/common v0.47.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
github.com/prometheus/common/assets v0.2.0 h1:0P5OrzoHrYBOSM1OigWL3mY8ZvV2N4zIE/5AahrSrfM=
github.com/prometheus/common/assets v0.2.0/go.mod h1:D17UVUE12bHbim7HzwUvtqm6gwBEaDQ0F+hIGbFbccI=
github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4=

6
model/labels/labels_stringlabels.go
View file

@ -450,14 +450,12 @@ func (ls Labels) DropMetricName() Labels {
return ls
}
// InternStrings calls intern on every string value inside ls, replacing them with what it returns.
// InternStrings is a no-op because it would only save when the whole set of labels is identical.
func (ls *Labels) InternStrings(intern func(string) string) {
ls.data = intern(ls.data)
}
// ReleaseStrings calls release on every string value inside ls.
// ReleaseStrings is a no-op for the same reason as InternStrings.
func (ls Labels) ReleaseStrings(release func(string)) {
release(ls.data)
}
// Labels returns the labels from the builder.

3
model/labels/labels_test.go
View file

@ -708,7 +708,8 @@ func TestScratchBuilder(t *testing.T) {
func TestLabels_Hash(t *testing.T) {
lbls := FromStrings("foo", "bar", "baz", "qux")
require.Equal(t, lbls.Hash(), lbls.Hash())
hash1, hash2 := lbls.Hash(), lbls.Hash()
require.Equal(t, hash1, hash2)
require.NotEqual(t, lbls.Hash(), FromStrings("foo", "bar").Hash(), "different labels match.")
}

3
model/relabel/relabel_test.go
View file

@ -22,6 +22,7 @@ import (
"gopkg.in/yaml.v2"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/util/testutil"
)
func TestRelabel(t *testing.T) {
@ -591,7 +592,7 @@ func TestRelabel(t *testing.T) {
res, keep := Process(test.input, test.relabel...)
require.Equal(t, !test.drop, keep)
if keep {
require.Equal(t, test.output, res)
testutil.RequireEqual(t, test.output, res)
}
}
}

3
model/textparse/openmetricslex.l
View file

@ -50,12 +50,15 @@ S [ ]
<sComment>TYPE{S} l.state = sMeta1; return tType
<sComment>UNIT{S} l.state = sMeta1; return tUnit
<sComment>"EOF"\n? l.state = sInit; return tEOFWord
<sMeta1>\"(\\.|[^\\"])*\" l.state = sMeta2; return tMName
<sMeta1>{M}({M}|{D})* l.state = sMeta2; return tMName
<sMeta2>{S}{C}*\n l.state = sInit; return tText
{M}({M}|{D})* l.state = sValue; return tMName
<sValue>\{ l.state = sLabels; return tBraceOpen
\{ l.state = sLabels; return tBraceOpen
<sLabels>{L}({L}|{D})* return tLName
<sLabels>\"(\\.|[^\\"])*\" l.state = sLabels; return tQString
<sLabels>\} l.state = sValue; return tBraceClose
<sLabels>= l.state = sLValue; return tEqual
<sLabels>, return tComma

590
model/textparse/openmetricslex.l.go
View file

File diff suppressed because it is too large Load diff

195
model/textparse/openmetricsparse.go
View file

@ -81,6 +81,12 @@ type OpenMetricsParser struct {
ts int64
hasTS bool
start int
// offsets is a list of offsets into series that describe the positions
// of the metric name and label names and values for this series.
// p.offsets[0] is the start character of the metric name.
// p.offsets[1] is the end of the metric name.
// Subsequently, p.offsets is a pair of pair of offsets for the positions
// of the label name and value start and end characters.
offsets []int
eOffsets []int
@ -153,20 +159,18 @@ func (p *OpenMetricsParser) Metric(l *labels.Labels) string {
s := string(p.series)
p.builder.Reset()
p.builder.Add(labels.MetricName, s[:p.offsets[0]-p.start])
metricName := unreplace(s[p.offsets[0]-p.start : p.offsets[1]-p.start])
p.builder.Add(labels.MetricName, metricName)
for i := 1; i < len(p.offsets); i += 4 {
for i := 2; i < len(p.offsets); i += 4 {
a := p.offsets[i] - p.start
b := p.offsets[i+1] - p.start
label := unreplace(s[a:b])
c := p.offsets[i+2] - p.start
d := p.offsets[i+3] - p.start
value := unreplace(s[c:d])
value := s[c:d]
// Replacer causes allocations. Replace only when necessary.
if strings.IndexByte(s[c:d], byte('\\')) >= 0 {
value = lvalReplacer.Replace(value)
}
p.builder.Add(s[a:b], value)
p.builder.Add(label, value)
}
p.builder.Sort()
@ -255,7 +259,13 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
case tHelp, tType, tUnit:
switch t2 := p.nextToken(); t2 {
case tMName:
p.offsets = append(p.offsets, p.l.start, p.l.i)
mStart := p.l.start
mEnd := p.l.i
if p.l.b[mStart] == '"' && p.l.b[mEnd-1] == '"' {
mStart++
mEnd--
}
p.offsets = append(p.offsets, mStart, mEnd)
default:
return EntryInvalid, p.parseError("expected metric name after "+t.String(), t2)
}
@ -312,58 +322,33 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
return EntryUnit, nil
}
case tBraceOpen:
// We found a brace, so make room for the eventual metric name. If these
// values aren't updated, then the metric name was not set inside the
// braces and we can return an error.
if len(p.offsets) == 0 {
p.offsets = []int{-1, -1}
}
if p.offsets, err = p.parseLVals(p.offsets, false); err != nil {
return EntryInvalid, err
}
p.series = p.l.b[p.start:p.l.i]
return p.parseMetricSuffix(p.nextToken())
case tMName:
p.offsets = append(p.offsets, p.l.i)
p.offsets = append(p.offsets, p.start, p.l.i)
p.series = p.l.b[p.start:p.l.i]
t2 := p.nextToken()
if t2 == tBraceOpen {
p.offsets, err = p.parseLVals(p.offsets)
p.offsets, err = p.parseLVals(p.offsets, false)
if err != nil {
return EntryInvalid, err
}
p.series = p.l.b[p.start:p.l.i]
t2 = p.nextToken()
}
p.val, err = p.getFloatValue(t2, "metric")
if err != nil {
return EntryInvalid, err
}
p.hasTS = false
switch t2 := p.nextToken(); t2 {
case tEOF:
return EntryInvalid, errors.New("data does not end with # EOF")
case tLinebreak:
break
case tComment:
if err := p.parseComment(); err != nil {
return EntryInvalid, err
}
case tTimestamp:
p.hasTS = true
var ts float64
// A float is enough to hold what we need for millisecond resolution.
if ts, err = parseFloat(yoloString(p.l.buf()[1:])); err != nil {
return EntryInvalid, fmt.Errorf("%w while parsing: %q", err, p.l.b[p.start:p.l.i])
}
if math.IsNaN(ts) || math.IsInf(ts, 0) {
return EntryInvalid, fmt.Errorf("invalid timestamp %f", ts)
}
p.ts = int64(ts * 1000)
switch t3 := p.nextToken(); t3 {
case tLinebreak:
case tComment:
if err := p.parseComment(); err != nil {
return EntryInvalid, err
}
default:
return EntryInvalid, p.parseError("expected next entry after timestamp", t3)
}
default:
return EntryInvalid, p.parseError("expected timestamp or # symbol", t2)
}
return EntrySeries, nil
return p.parseMetricSuffix(t2)
default:
err = p.parseError("expected a valid start token", t)
@ -374,7 +359,7 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
func (p *OpenMetricsParser) parseComment() error {
var err error
// Parse the labels.
p.eOffsets, err = p.parseLVals(p.eOffsets)
p.eOffsets, err = p.parseLVals(p.eOffsets, true)
if err != nil {
return err
}
@ -415,38 +400,47 @@ func (p *OpenMetricsParser) parseComment() error {
return nil
}
func (p *OpenMetricsParser) parseLVals(offsets []int) ([]int, error) {
first := true
func (p *OpenMetricsParser) parseLVals(offsets []int, isExemplar bool) ([]int, error) {
t := p.nextToken()
for {
t := p.nextToken()
curTStart := p.l.start
curTI := p.l.i
switch t {
case tBraceClose:
return offsets, nil
case tComma:
if first {
return nil, p.parseError("expected label name or left brace", t)
}
t = p.nextToken()
if t != tLName {
case tLName:
case tQString:
default:
return nil, p.parseError("expected label name", t)
}
t = p.nextToken()
// A quoted string followed by a comma or brace is a metric name. Set the
// offsets and continue processing. If this is an exemplar, this format
// is not allowed.
if t == tComma || t == tBraceClose {
if isExemplar {
return nil, p.parseError("expected label name", t)
}
case tLName:
if !first {
return nil, p.parseError("expected comma", t)
if offsets[0] != -1 || offsets[1] != -1 {
return nil, fmt.Errorf("metric name already set while parsing: %q", p.l.b[p.start:p.l.i])
}
default:
if first {
return nil, p.parseError("expected label name or left brace", t)
offsets[0] = curTStart + 1
offsets[1] = curTI - 1
if t == tBraceClose {
return offsets, nil
}
return nil, p.parseError("expected comma or left brace", t)
t = p.nextToken()
continue
}
first = false
// t is now a label name.
// We have a label name, and it might be quoted.
if p.l.b[curTStart] == '"' {
curTStart++
curTI--
}
offsets = append(offsets, curTStart, curTI)
offsets = append(offsets, p.l.start, p.l.i)
if t := p.nextToken(); t != tEqual {
if t != tEqual {
return nil, p.parseError("expected equal", t)
}
if t := p.nextToken(); t != tLValue {
@ -459,9 +453,64 @@ func (p *OpenMetricsParser) parseLVals(offsets []int) ([]int, error) {
// The openMetricsLexer ensures the value string is quoted. Strip first
// and last character.
offsets = append(offsets, p.l.start+1, p.l.i-1)
// Free trailing commas are allowed.
t = p.nextToken()
if t == tComma {
t = p.nextToken()
} else if t != tBraceClose {
return nil, p.parseError("expected comma or brace close", t)
}
}
}
// parseMetricSuffix parses the end of the line after the metric name and
// labels. It starts parsing with the provided token.
func (p *OpenMetricsParser) parseMetricSuffix(t token) (Entry, error) {
if p.offsets[0] == -1 {
return EntryInvalid, fmt.Errorf("metric name not set while parsing: %q", p.l.b[p.start:p.l.i])
}
var err error
p.val, err = p.getFloatValue(t, "metric")
if err != nil {
return EntryInvalid, err
}
p.hasTS = false
switch t2 := p.nextToken(); t2 {
case tEOF:
return EntryInvalid, errors.New("data does not end with # EOF")
case tLinebreak:
break
case tComment:
if err := p.parseComment(); err != nil {
return EntryInvalid, err
}
case tTimestamp:
p.hasTS = true
var ts float64
// A float is enough to hold what we need for millisecond resolution.
if ts, err = parseFloat(yoloString(p.l.buf()[1:])); err != nil {
return EntryInvalid, fmt.Errorf("%w while parsing: %q", err, p.l.b[p.start:p.l.i])
}
if math.IsNaN(ts) || math.IsInf(ts, 0) {
return EntryInvalid, fmt.Errorf("invalid timestamp %f", ts)
}
p.ts = int64(ts * 1000)
switch t3 := p.nextToken(); t3 {
case tLinebreak:
case tComment:
if err := p.parseComment(); err != nil {
return EntryInvalid, err
}
default:
return EntryInvalid, p.parseError("expected next entry after timestamp", t3)
}
}
return EntrySeries, nil
}
func (p *OpenMetricsParser) getFloatValue(t token, after string) (float64, error) {
if t != tValue {
return 0, p.parseError(fmt.Sprintf("expected value after %v", after), t)

166
model/textparse/openmetricsparse_test.go
View file

@ -23,6 +23,7 @@ import (
"github.com/prometheus/prometheus/model/exemplar"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/util/testutil"
)
func TestOpenMetricsParse(t *testing.T) {
@ -251,6 +252,137 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5`
var res labels.Labels
for {
et, err := p.Next()
if errors.Is(err, io.EOF) {
break
}
require.NoError(t, err)
switch et {
case EntrySeries:
m, ts, v := p.Series()
var e exemplar.Exemplar
p.Metric(&res)
found := p.Exemplar(&e)
require.Equal(t, exp[i].m, string(m))
require.Equal(t, exp[i].t, ts)
require.Equal(t, exp[i].v, v)
testutil.RequireEqual(t, exp[i].lset, res)
if exp[i].e == nil {
require.False(t, found)
} else {
require.True(t, found)
testutil.RequireEqual(t, *exp[i].e, e)
}
case EntryType:
m, typ := p.Type()
require.Equal(t, exp[i].m, string(m))
require.Equal(t, exp[i].typ, typ)
case EntryHelp:
m, h := p.Help()
require.Equal(t, exp[i].m, string(m))
require.Equal(t, exp[i].help, string(h))
case EntryUnit:
m, u := p.Unit()
require.Equal(t, exp[i].m, string(m))
require.Equal(t, exp[i].unit, string(u))
case EntryComment:
require.Equal(t, exp[i].comment, string(p.Comment()))
}
i++
}
require.Len(t, exp, i)
}
func TestUTF8OpenMetricsParse(t *testing.T) {
oldValidationScheme := model.NameValidationScheme
model.NameValidationScheme = model.UTF8Validation
defer func() {
model.NameValidationScheme = oldValidationScheme
}()
input := `# HELP "go.gc_duration_seconds" A summary of the GC invocation durations.
# TYPE "go.gc_duration_seconds" summary
# UNIT "go.gc_duration_seconds" seconds
{"go.gc_duration_seconds",quantile="0"} 4.9351e-05
{"go.gc_duration_seconds",quantile="0.25"} 7.424100000000001e-05
{"go.gc_duration_seconds",quantile="0.5",a="b"} 8.3835e-05
{"http.status",q="0.9",a="b"} 8.3835e-05
{"http.status",q="0.9",a="b"} 8.3835e-05
{q="0.9","http.status",a="b"} 8.3835e-05
{"go.gc_duration_seconds_sum"} 0.004304266
{"Heizölrückstoßabdämpfung 10€ metric with \"interesting\" {character\nchoices}","strange©™\n'quoted' \"name\""="6"} 10.0`
input += "\n# EOF\n"
exp := []struct {
lset labels.Labels
m string
t *int64
v float64
typ model.MetricType
help string
unit string
comment string
e *exemplar.Exemplar
}{
{
m: "go.gc_duration_seconds",
help: "A summary of the GC invocation durations.",
}, {
m: "go.gc_duration_seconds",
typ: model.MetricTypeSummary,
}, {
m: "go.gc_duration_seconds",
unit: "seconds",
}, {
m: `{"go.gc_duration_seconds",quantile="0"}`,
v: 4.9351e-05,
lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0"),
}, {
m: `{"go.gc_duration_seconds",quantile="0.25"}`,
v: 7.424100000000001e-05,
lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0.25"),
}, {
m: `{"go.gc_duration_seconds",quantile="0.5",a="b"}`,
v: 8.3835e-05,
lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0.5", "a", "b"),
}, {
m: `{"http.status",q="0.9",a="b"}`,
v: 8.3835e-05,
lset: labels.FromStrings("__name__", "http.status", "q", "0.9", "a", "b"),
}, {
m: `{"http.status",q="0.9",a="b"}`,
v: 8.3835e-05,
lset: labels.FromStrings("__name__", "http.status", "q", "0.9", "a", "b"),
}, {
m: `{q="0.9","http.status",a="b"}`,
v: 8.3835e-05,
lset: labels.FromStrings("__name__", "http.status", "q", "0.9", "a", "b"),
}, {
m: `{"go.gc_duration_seconds_sum"}`,
v: 0.004304266,
lset: labels.FromStrings("__name__", "go.gc_duration_seconds_sum"),
}, {
m: `{"Heizölrückstoßabdämpfung 10€ metric with \"interesting\" {character\nchoices}","strange©™\n'quoted' \"name\""="6"}`,
v: 10.0,
lset: labels.FromStrings("__name__", `Heizölrückstoßabdämpfung 10€ metric with "interesting" {character
choices}`, "strange©™\n'quoted' \"name\"", "6"),
},
}
p := NewOpenMetricsParser([]byte(input))
i := 0
var res labels.Labels
for {
et, err := p.Next()
if errors.Is(err, io.EOF) {
@ -456,17 +588,13 @@ func TestOpenMetricsParseErrors(t *testing.T) {
input: "a{b='c'} 1\n# EOF\n",
err: "expected label value, got \"'\" (\"INVALID\") while parsing: \"a{b='\"",
},
{
input: "a{b=\"c\",} 1\n# EOF\n",
err: "expected label name, got \"} \" (\"BCLOSE\") while parsing: \"a{b=\\\"c\\\",} \"",
},
{
input: "a{,b=\"c\"} 1\n# EOF\n",
err: "expected label name or left brace, got \",b\" (\"COMMA\") while parsing: \"a{,b\"",
err: "expected label name, got \",b\" (\"COMMA\") while parsing: \"a{,b\"",
},
{
input: "a{b=\"c\"d=\"e\"} 1\n# EOF\n",
err: "expected comma, got \"d=\" (\"LNAME\") while parsing: \"a{b=\\\"c\\\"d=\"",
err: "expected comma or brace close, got \"d=\" (\"LNAME\") while parsing: \"a{b=\\\"c\\\"d=\"",
},
{
input: "a{b=\"c\",,d=\"e\"} 1\n# EOF\n",
@ -478,12 +606,24 @@ func TestOpenMetricsParseErrors(t *testing.T) {
},
{
input: "a{\xff=\"foo\"} 1\n# EOF\n",
err: "expected label name or left brace, got \"\\xff\" (\"INVALID\") while parsing: \"a{\\xff\"",
err: "expected label name, got \"\\xff\" (\"INVALID\") while parsing: \"a{\\xff\"",
},
{
input: "a{b=\"\xff\"} 1\n# EOF\n",
err: "invalid UTF-8 label value: \"\\\"\\xff\\\"\"",
},
{
input: `{"a","b = "c"}
# EOF
`,
err: "expected equal, got \"c\\\"\" (\"LNAME\") while parsing: \"{\\\"a\\\",\\\"b = \\\"c\\\"\"",
},
{
input: `{"a",b\nc="d"} 1
# EOF
`,
err: "expected equal, got \"\\\\\" (\"INVALID\") while parsing: \"{\\\"a\\\",b\\\\\"",
},
{
input: "a true\n",
err: "strconv.ParseFloat: parsing \"true\": invalid syntax while parsing: \"a true\"",
@ -494,7 +634,7 @@ func TestOpenMetricsParseErrors(t *testing.T) {
},
{
input: "empty_label_name{=\"\"} 0\n# EOF\n",
err: "expected label name or left brace, got \"=\\\"\" (\"EQUAL\") while parsing: \"empty_label_name{=\\\"\"",
err: "expected label name, got \"=\\\"\" (\"EQUAL\") while parsing: \"empty_label_name{=\\\"\"",
},
{
input: "foo 1_2\n\n# EOF\n",
@ -524,6 +664,14 @@ func TestOpenMetricsParseErrors(t *testing.T) {
input: `custom_metric_total 1 # {aa="bb"}`,
err: "expected value after exemplar labels, got \"}\" (\"EOF\") while parsing: \"custom_metric_total 1 # {aa=\\\"bb\\\"}\"",
},
{
input: `custom_metric_total 1 # {bb}`,
err: "expected label name, got \"}\" (\"BCLOSE\") while parsing: \"custom_metric_total 1 # {bb}\"",
},
{
input: `custom_metric_total 1 # {bb, a="dd"}`,
err: "expected label name, got \", \" (\"COMMA\") while parsing: \"custom_metric_total 1 # {bb, \"",
},
{
input: `custom_metric_total 1 # {aa="bb",,cc="dd"} 1`,
err: "expected label name, got \",c\" (\"COMMA\") while parsing: \"custom_metric_total 1 # {aa=\\\"bb\\\",,c\"",
@ -550,7 +698,7 @@ func TestOpenMetricsParseErrors(t *testing.T) {
},
{
input: `{b="c",} 1`,
err: "expected a valid start token, got \"{\" (\"INVALID\") while parsing: \"{\"",
err: "metric name not set while parsing: \"{b=\\\"c\\\",} 1\"",
},
{
input: `a 1 NaN`,

3
model/textparse/promlex.l
View file

@ -66,12 +66,15 @@ C [^\n]
# return l.consumeComment()
<sComment>HELP[\t ]+ l.state = sMeta1; return tHelp
<sComment>TYPE[\t ]+ l.state = sMeta1; return tType
<sMeta1>\"(\\.|[^\\"])*\" l.state = sMeta2; return tMName
<sMeta1>{M}({M}|{D})* l.state = sMeta2; return tMName
<sMeta2>{C}* l.state = sInit; return tText
{M}({M}|{D})* l.state = sValue; return tMName
<sValue>\{ l.state = sLabels; return tBraceOpen
\{ l.state = sLabels; return tBraceOpen
<sLabels>{L}({L}|{D})* return tLName
<sLabels>\"(\\.|[^\\"])*\" l.state = sLabels; return tQString
<sLabels>\} l.state = sValue; return tBraceClose
<sLabels>= l.state = sLValue; return tEqual
<sLabels>, return tComma

352
model/textparse/promlex.l.go
View file

@ -51,19 +51,19 @@ yystate0:
case 0: // start condition: INITIAL
goto yystart1
case 1: // start condition: sComment
goto yystart8
goto yystart9
case 2: // start condition: sMeta1
goto yystart19
goto yystart20
case 3: // start condition: sMeta2
goto yystart21
goto yystart25
case 4: // start condition: sLabels
goto yystart24
goto yystart28
case 5: // start condition: sLValue
goto yystart29
case 6: // start condition: sValue
goto yystart33
case 7: // start condition: sTimestamp
goto yystart36
case 6: // start condition: sValue
goto yystart40
case 7: // start condition: sTimestamp
goto yystart43
}
yystate1:
@ -82,6 +82,8 @@ yystart1:
goto yystate3
case c == '\x00':
goto yystate2
case c == '{':
goto yystate8
}
yystate2:
@ -123,40 +125,35 @@ yystate7:
c = l.next()
switch {
default:
goto yyrule10
goto yyrule11
case c >= '0' && c <= ':' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
goto yystate7
}
yystate8:
c = l.next()
yystart8:
goto yyrule13
yystate9:
c = l.next()
yystart9:
switch {
default:
goto yyabort
case c == 'H':
goto yystate9
goto yystate10
case c == 'T':
goto yystate14
goto yystate15
case c == '\t' || c == ' ':
goto yystate3
}
yystate9:
c = l.next()
switch {
default:
goto yyabort
case c == 'E':
goto yystate10
}
yystate10:
c = l.next()
switch {
default:
goto yyabort
case c == 'L':
case c == 'E':
goto yystate11
}
@ -165,7 +162,7 @@ yystate11:
switch {
default:
goto yyabort
case c == 'P':
case c == 'L':
goto yystate12
}
@ -174,7 +171,7 @@ yystate12:
switch {
default:
goto yyabort
case c == '\t' || c == ' ':
case c == 'P':
goto yystate13
}
@ -182,18 +179,18 @@ yystate13:
c = l.next()
switch {
default:
goto yyrule6
goto yyabort
case c == '\t' || c == ' ':
goto yystate13
goto yystate14
}
yystate14:
c = l.next()
switch {
default:
goto yyabort
case c == 'Y':
goto yystate15
goto yyrule6
case c == '\t' || c == ' ':
goto yystate14
}
yystate15:
@ -201,7 +198,7 @@ yystate15:
switch {
default:
goto yyabort
case c == 'P':
case c == 'Y':
goto yystate16
}
@ -210,7 +207,7 @@ yystate16:
switch {
default:
goto yyabort
case c == 'E':
case c == 'P':
goto yystate17
}
@ -219,7 +216,7 @@ yystate17:
switch {
default:
goto yyabort
case c == '\t' || c == ' ':
case c == 'E':
goto yystate18
}
@ -227,167 +224,167 @@ yystate18:
c = l.next()
switch {
default:
goto yyrule7
goto yyabort
case c == '\t' || c == ' ':
goto yystate18
goto yystate19
}
yystate19:
c = l.next()
yystart19:
switch {
default:
goto yyabort
case c == ':' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
goto yystate20
goto yyrule7
case c == '\t' || c == ' ':
goto yystate3
goto yystate19
}
yystate20:
c = l.next()
yystart20:
switch {
default:
goto yyrule8
case c >= '0' && c <= ':' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
goto yystate20
goto yyabort
case c == '"':
goto yystate21
case c == ':' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
goto yystate24
case c == '\t' || c == ' ':
goto yystate3
}
yystate21:
c = l.next()
yystart21:
switch {
default:
goto yyrule9
case c == '\t' || c == ' ':
goto yystate23
case c >= '\x01' && c <= '\b' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'ÿ':
goto yyabort
case c == '"':
goto yystate22
case c == '\\':
goto yystate23
case c >= '\x01' && c <= '!' || c >= '#' && c <= '[' || c >= ']' && c <= 'ÿ':
goto yystate21
}
yystate22:
c = l.next()
switch {
default:
goto yyrule9
case c >= '\x01' && c <= '\t' || c >= '\v' && c <= 'ÿ':
goto yystate22
}
goto yyrule8
yystate23:
c = l.next()
switch {
default:
goto yyrule3
case c == '\t' || c == ' ':
goto yystate23
case c >= '\x01' && c <= '\b' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'ÿ':
goto yystate22
goto yyabort
case c >= '\x01' && c <= '\t' || c >= '\v' && c <= 'ÿ':
goto yystate21
}
yystate24:
c = l.next()
yystart24:
switch {
default:
goto yyabort
case c == ',':
goto yystate25
case c == '=':
goto yystate26
case c == '\t' || c == ' ':
goto yystate3
case c == '}':
goto yystate28
case c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
goto yystate27
goto yyrule9
case c >= '0' && c <= ':' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
goto yystate24
}
yystate25:
c = l.next()
goto yyrule15
yystart25:
switch {
default:
goto yyrule10
case c == '\t' || c == ' ':
goto yystate27
case c >= '\x01' && c <= '\b' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'ÿ':
goto yystate26
}
yystate26:
c = l.next()
goto yyrule14
switch {
default:
goto yyrule10
case c >= '\x01' && c <= '\t' || c >= '\v' && c <= 'ÿ':
goto yystate26
}
yystate27:
c = l.next()
switch {
default:
goto yyrule12
case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
goto yyrule3
case c == '\t' || c == ' ':
goto yystate27
case c >= '\x01' && c <= '\b' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'ÿ':
goto yystate26
}
yystate28:
c = l.next()
goto yyrule13
yystart28:
switch {
default:
goto yyabort
case c == '"':
goto yystate29
case c == ',':
goto yystate32
case c == '=':
goto yystate33
case c == '\t' || c == ' ':
goto yystate3
case c == '}':
goto yystate35
case c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
goto yystate34
}
yystate29:
c = l.next()
yystart29:
switch {
default:
goto yyabort
case c == '"':
goto yystate30
case c == '\t' || c == ' ':
goto yystate3
case c == '\\':
goto yystate31
case c >= '\x01' && c <= '!' || c >= '#' && c <= '[' || c >= ']' && c <= 'ÿ':
goto yystate29
}
yystate30:
c = l.next()
switch {
default:
goto yyabort
case c == '"':
goto yystate31
case c == '\\':
goto yystate32
case c >= '\x01' && c <= '!' || c >= '#' && c <= '[' || c >= ']' && c <= 'ÿ':
goto yystate30
}
goto yyrule15
yystate31:
c = l.next()
goto yyrule16
yystate32:
c = l.next()
switch {
default:
goto yyabort
case c >= '\x01' && c <= '\t' || c >= '\v' && c <= 'ÿ':
goto yystate30
goto yystate29
}
yystate32:
c = l.next()
goto yyrule18
yystate33:
c = l.next()
yystart33:
switch {
default:
goto yyabort
case c == '\t' || c == ' ':
goto yystate3
case c == '{':
goto yystate35
case c >= '\x01' && c <= '\b' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'z' || c >= '|' && c <= 'ÿ':
goto yystate34
}
goto yyrule17
yystate34:
c = l.next()
switch {
default:
goto yyrule17
case c >= '\x01' && c <= '\b' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'z' || c >= '|' && c <= 'ÿ':
goto yyrule14
case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
goto yystate34
}
yystate35:
c = l.next()
goto yyrule11
goto yyrule16
yystate36:
c = l.next()
@ -395,25 +392,90 @@ yystart36:
switch {
default:
goto yyabort
case c == '\n':
case c == '"':
goto yystate37
case c == '\t' || c == ' ':
goto yystate3
case c >= '0' && c <= '9':
goto yystate38
}
yystate37:
c = l.next()
goto yyrule19
switch {
default:
goto yyabort
case c == '"':
goto yystate38
case c == '\\':
goto yystate39
case c >= '\x01' && c <= '!' || c >= '#' && c <= '[' || c >= ']' && c <= 'ÿ':
goto yystate37
}
yystate38:
c = l.next()
goto yyrule19
yystate39:
c = l.next()
switch {
default:
goto yyrule18
goto yyabort
case c >= '\x01' && c <= '\t' || c >= '\v' && c <= 'ÿ':
goto yystate37
}
yystate40:
c = l.next()
yystart40:
switch {
default:
goto yyabort
case c == '\t' || c == ' ':
goto yystate3
case c == '{':
goto yystate42
case c >= '\x01' && c <= '\b' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'z' || c >= '|' && c <= 'ÿ':
goto yystate41
}
yystate41:
c = l.next()
switch {
default:
goto yyrule20
case c >= '\x01' && c <= '\b' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'z' || c >= '|' && c <= 'ÿ':
goto yystate41
}
yystate42:
c = l.next()
goto yyrule12
yystate43:
c = l.next()
yystart43:
switch {
default:
goto yyabort
case c == '\n':
goto yystate44
case c == '\t' || c == ' ':
goto yystate3
case c >= '0' && c <= '9':
goto yystate38
goto yystate45
}
yystate44:
c = l.next()
goto yyrule22
yystate45:
c = l.next()
switch {
default:
goto yyrule21
case c >= '0' && c <= '9':
goto yystate45
}
yyrule1: // \0
@ -451,67 +513,85 @@ yyrule7: // TYPE[\t ]+
return tType
goto yystate0
}
yyrule8: // {M}({M}|{D})*
yyrule8: // \"(\\.|[^\\"])*\"
{
l.state = sMeta2
return tMName
goto yystate0
}
yyrule9: // {C}*
yyrule9: // {M}({M}|{D})*
{
l.state = sMeta2
return tMName
goto yystate0
}
yyrule10: // {C}*
{
l.state = sInit
return tText
goto yystate0
}
yyrule10: // {M}({M}|{D})*
yyrule11: // {M}({M}|{D})*
{
l.state = sValue
return tMName
goto yystate0
}
yyrule11: // \{
yyrule12: // \{
{
l.state = sLabels
return tBraceOpen
goto yystate0
}
yyrule12: // {L}({L}|{D})*
yyrule13: // \{
{
l.state = sLabels
return tBraceOpen
goto yystate0
}
yyrule14: // {L}({L}|{D})*
{
return tLName
}
yyrule13: // \}
yyrule15: // \"(\\.|[^\\"])*\"
{
l.state = sLabels
return tQString
goto yystate0
}
yyrule16: // \}
{
l.state = sValue
return tBraceClose
goto yystate0
}
yyrule14: // =
yyrule17: // =
{
l.state = sLValue
return tEqual
goto yystate0
}
yyrule15: // ,
yyrule18: // ,
{
return tComma
}
yyrule16: // \"(\\.|[^\\"])*\"
yyrule19: // \"(\\.|[^\\"])*\"
{
l.state = sLabels
return tLValue
goto yystate0
}
yyrule17: // [^{ \t\n]+
yyrule20: // [^{ \t\n]+
{
l.state = sTimestamp
return tValue
goto yystate0
}
yyrule18: // {D}+
yyrule21: // {D}+
{
return tTimestamp
}
yyrule19: // \n
yyrule22: // \n
if true { // avoid go vet determining the below panic will not be reached
l.state = sInit
return tLinebreak
@ -520,9 +600,7 @@ yyrule19: // \n
panic("unreachable")
yyabort: // no lexem recognized
//
// silence unused label errors for build and satisfy go vet reachability analysis
//
{
if false {
goto yyabort
@ -534,26 +612,26 @@ yyabort: // no lexem recognized
goto yystate1
}
if false {
goto yystate8
goto yystate9
}
if false {
goto yystate19
goto yystate20
}
if false {
goto yystate21
goto yystate25
}
if false {
goto yystate24
}
if false {
goto yystate29
}
if false {
goto yystate33
goto yystate28
}
if false {
goto yystate36
}
if false {
goto yystate40
}
if false {
goto yystate43
}
}
// Workaround to gobble up comments that started with a HELP or TYPE

155
model/textparse/promparse.go
View file

@ -57,6 +57,7 @@ const (
tComment
tBlank
tMName
tQString
tBraceOpen
tBraceClose
tLName
@ -93,6 +94,8 @@ func (t token) String() string {
return "BLANK"
case tMName:
return "MNAME"
case tQString:
return "QSTRING"
case tBraceOpen:
return "BOPEN"
case tBraceClose:
@ -153,6 +156,12 @@ type PromParser struct {
ts int64
hasTS bool
start int
// offsets is a list of offsets into series that describe the positions
// of the metric name and label names and values for this series.
// p.offsets[0] is the start character of the metric name.
// p.offsets[1] is the end of the metric name.
// Subsequently, p.offsets is a pair of pair of offsets for the positions
// of the label name and value start and end characters.
offsets []int
}
@ -218,20 +227,17 @@ func (p *PromParser) Metric(l *labels.Labels) string {
s := string(p.series)
p.builder.Reset()
p.builder.Add(labels.MetricName, s[:p.offsets[0]-p.start])
metricName := unreplace(s[p.offsets[0]-p.start : p.offsets[1]-p.start])
p.builder.Add(labels.MetricName, metricName)
for i := 1; i < len(p.offsets); i += 4 {
for i := 2; i < len(p.offsets); i += 4 {
a := p.offsets[i] - p.start
b := p.offsets[i+1] - p.start
label := unreplace(s[a:b])
c := p.offsets[i+2] - p.start
d := p.offsets[i+3] - p.start
value := s[c:d]
// Replacer causes allocations. Replace only when necessary.
if strings.IndexByte(s[c:d], byte('\\')) >= 0 {
value = lvalReplacer.Replace(value)
}
p.builder.Add(s[a:b], value)
value := unreplace(s[c:d])
p.builder.Add(label, value)
}
p.builder.Sort()
@ -289,7 +295,13 @@ func (p *PromParser) Next() (Entry, error) {
case tHelp, tType:
switch t2 := p.nextToken(); t2 {
case tMName:
p.offsets = append(p.offsets, p.l.start, p.l.i)
mStart := p.l.start
mEnd := p.l.i
if p.l.b[mStart] == '"' && p.l.b[mEnd-1] == '"' {
mStart++
mEnd--
}
p.offsets = append(p.offsets, mStart, mEnd)
default:
return EntryInvalid, p.parseError("expected metric name after "+t.String(), t2)
}
@ -301,7 +313,7 @@ func (p *PromParser) Next() (Entry, error) {
p.text = []byte{}
}
default:
return EntryInvalid, fmt.Errorf("expected text in %s", t.String())
return EntryInvalid, fmt.Errorf("expected text in %s, got %v", t.String(), t2.String())
}
switch t {
case tType:
@ -339,12 +351,24 @@ func (p *PromParser) Next() (Entry, error) {
return EntryInvalid, p.parseError("linebreak expected after comment", t)
}
return EntryComment, nil
case tBraceOpen:
// We found a brace, so make room for the eventual metric name. If these
// values aren't updated, then the metric name was not set inside the
// braces and we can return an error.
if len(p.offsets) == 0 {
p.offsets = []int{-1, -1}
}
if err := p.parseLVals(); err != nil {
return EntryInvalid, err
}
case tMName:
p.offsets = append(p.offsets, p.l.i)
p.series = p.l.b[p.start:p.l.i]
return p.parseMetricSuffix(p.nextToken())
case tMName:
p.offsets = append(p.offsets, p.start, p.l.i)
p.series = p.l.b[p.start:p.l.i]
t2 := p.nextToken()
// If there's a brace, consume and parse the label values.
if t2 == tBraceOpen {
if err := p.parseLVals(); err != nil {
return EntryInvalid, err
@ -352,32 +376,7 @@ func (p *PromParser) Next() (Entry, error) {
p.series = p.l.b[p.start:p.l.i]
t2 = p.nextToken()
}
if t2 != tValue {
return EntryInvalid, p.parseError("expected value after metric", t2)
}
if p.val, err = parseFloat(yoloString(p.l.buf())); err != nil {
return EntryInvalid, fmt.Errorf("%w while parsing: %q", err, p.l.b[p.start:p.l.i])
}
// Ensure canonical NaN value.
if math.IsNaN(p.val) {
p.val = math.Float64frombits(value.NormalNaN)
}
p.hasTS = false
switch t := p.nextToken(); t {
case tLinebreak:
break
case tTimestamp:
p.hasTS = true
if p.ts, err = strconv.ParseInt(yoloString(p.l.buf()), 10, 64); err != nil {
return EntryInvalid, fmt.Errorf("%w while parsing: %q", err, p.l.b[p.start:p.l.i])
}
if t2 := p.nextToken(); t2 != tLinebreak {
return EntryInvalid, p.parseError("expected next entry after timestamp", t2)
}
default:
return EntryInvalid, p.parseError("expected timestamp or new record", t)
}
return EntrySeries, nil
return p.parseMetricSuffix(t2)
default:
err = p.parseError("expected a valid start token", t)
@ -385,19 +384,43 @@ func (p *PromParser) Next() (Entry, error) {
return EntryInvalid, err
}
// parseLVals parses the contents inside the braces.
func (p *PromParser) parseLVals() error {
t := p.nextToken()
for {
curTStart := p.l.start
curTI := p.l.i
switch t {
case tBraceClose:
return nil
case tLName:
case tQString:
default:
return p.parseError("expected label name", t)
}
p.offsets = append(p.offsets, p.l.start, p.l.i)
if t := p.nextToken(); t != tEqual {
t = p.nextToken()
// A quoted string followed by a comma or brace is a metric name. Set the
// offsets and continue processing.
if t == tComma || t == tBraceClose {
if p.offsets[0] != -1 || p.offsets[1] != -1 {
return fmt.Errorf("metric name already set while parsing: %q", p.l.b[p.start:p.l.i])
}
p.offsets[0] = curTStart + 1
p.offsets[1] = curTI - 1
if t == tBraceClose {
return nil
}
t = p.nextToken()
continue
}
// We have a label name, and it might be quoted.
if p.l.b[curTStart] == '"' {
curTStart++
curTI--
}
p.offsets = append(p.offsets, curTStart, curTI)
if t != tEqual {
return p.parseError("expected equal", t)
}
if t := p.nextToken(); t != tLValue {
@ -411,13 +434,51 @@ func (p *PromParser) parseLVals() error {
// and last character.
p.offsets = append(p.offsets, p.l.start+1, p.l.i-1)
// Free trailing commas are allowed.
// Free trailing commas are allowed. NOTE: this allows spaces between label
// names, unlike in OpenMetrics. It is not clear if this is intended or an
// accidental bug.
if t = p.nextToken(); t == tComma {
t = p.nextToken()
}
}
}
// parseMetricSuffix parses the end of the line after the metric name and
// labels. It starts parsing with the provided token.
func (p *PromParser) parseMetricSuffix(t token) (Entry, error) {
if p.offsets[0] == -1 {
return EntryInvalid, fmt.Errorf("metric name not set while parsing: %q", p.l.b[p.start:p.l.i])
}
if t != tValue {
return EntryInvalid, p.parseError("expected value after metric", t)
}
var err error
if p.val, err = parseFloat(yoloString(p.l.buf())); err != nil {
return EntryInvalid, fmt.Errorf("%w while parsing: %q", err, p.l.b[p.start:p.l.i])
}
// Ensure canonical NaN value.
if math.IsNaN(p.val) {
p.val = math.Float64frombits(value.NormalNaN)
}
p.hasTS = false
switch t := p.nextToken(); t {
case tLinebreak:
break
case tTimestamp:
p.hasTS = true
if p.ts, err = strconv.ParseInt(yoloString(p.l.buf()), 10, 64); err != nil {
return EntryInvalid, fmt.Errorf("%w while parsing: %q", err, p.l.b[p.start:p.l.i])
}
if t2 := p.nextToken(); t2 != tLinebreak {
return EntryInvalid, p.parseError("expected next entry after timestamp", t2)
}
default:
return EntryInvalid, p.parseError("expected timestamp or new record", t)
}
return EntrySeries, nil
}
var lvalReplacer = strings.NewReplacer(
`\"`, "\"",
`\\`, "\\",
@ -429,6 +490,14 @@ var helpReplacer = strings.NewReplacer(
`\n`, "\n",
)
func unreplace(s string) string {
// Replacer causes allocations. Replace only when necessary.
if strings.IndexByte(s, byte('\\')) >= 0 {
return lvalReplacer.Replace(s)
}
return s
}
func yoloString(b []byte) string {
return *((*string)(unsafe.Pointer(&b)))
}

143
model/textparse/promparse_test.go
View file

@ -26,6 +26,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/util/testutil"
)
func TestPromParse(t *testing.T) {
@ -47,6 +48,7 @@ go_gc_duration_seconds{ quantile="1.0", a="b" } 8.3835e-05
go_gc_duration_seconds { quantile="1.0", a="b" } 8.3835e-05
go_gc_duration_seconds { quantile= "1.0", a= "b", } 8.3835e-05
go_gc_duration_seconds { quantile = "1.0", a = "b" } 8.3835e-05
go_gc_duration_seconds { quantile = "2.0" a = "b" } 8.3835e-05
go_gc_duration_seconds_count 99
some:aggregate:rate5m{a_b="c"} 1
# HELP go_goroutines Number of goroutines that currently exist.
@ -129,6 +131,11 @@ testmetric{label="\"bar\""} 1`
m: `go_gc_duration_seconds { quantile = "1.0", a = "b" }`,
v: 8.3835e-05,
lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "1.0", "a", "b"),
}, {
// NOTE: Unlike OpenMetrics, Promparse allows spaces between label terms. This appears to be unintended and should probably be fixed.
m: `go_gc_duration_seconds { quantile = "2.0" a = "b" }`,
v: 8.3835e-05,
lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "2.0", "a", "b"),
}, {
m: `go_gc_duration_seconds_count`,
v: 99,
@ -175,6 +182,132 @@ testmetric{label="\"bar\""} 1`
var res labels.Labels
for {
et, err := p.Next()
if errors.Is(err, io.EOF) {
break
}
require.NoError(t, err)
switch et {
case EntrySeries:
m, ts, v := p.Series()
p.Metric(&res)
require.Equal(t, exp[i].m, string(m))
require.Equal(t, exp[i].t, ts)
require.Equal(t, exp[i].v, v)
testutil.RequireEqual(t, exp[i].lset, res)
case EntryType:
m, typ := p.Type()
require.Equal(t, exp[i].m, string(m))
require.Equal(t, exp[i].typ, typ)
case EntryHelp:
m, h := p.Help()
require.Equal(t, exp[i].m, string(m))
require.Equal(t, exp[i].help, string(h))
case EntryComment:
require.Equal(t, exp[i].comment, string(p.Comment()))
}
i++
}
require.Len(t, exp, i)
}
func TestUTF8PromParse(t *testing.T) {
oldValidationScheme := model.NameValidationScheme
model.NameValidationScheme = model.UTF8Validation
defer func() {
model.NameValidationScheme = oldValidationScheme
}()
input := `# HELP "go.gc_duration_seconds" A summary of the GC invocation durations.
# TYPE "go.gc_duration_seconds" summary
{"go.gc_duration_seconds",quantile="0"} 4.9351e-05
{"go.gc_duration_seconds",quantile="0.25",} 7.424100000000001e-05
{"go.gc_duration_seconds",quantile="0.5",a="b"} 8.3835e-05
{"go.gc_duration_seconds",quantile="0.8", a="b"} 8.3835e-05
{"go.gc_duration_seconds", quantile="0.9", a="b"} 8.3835e-05
{"go.gc_duration_seconds", quantile="1.0", a="b" } 8.3835e-05
{ "go.gc_duration_seconds", quantile="1.0", a="b" } 8.3835e-05
{ "go.gc_duration_seconds", quantile= "1.0", a= "b", } 8.3835e-05
{ "go.gc_duration_seconds", quantile = "1.0", a = "b" } 8.3835e-05
{"go.gc_duration_seconds_count"} 99
{"Heizölrückstoßabdämpfung 10€ metric with \"interesting\" {character\nchoices}","strange©™\n'quoted' \"name\""="6"} 10.0`
exp := []struct {
lset labels.Labels
m string
t *int64
v float64
typ model.MetricType
help string
comment string
}{
{
m: "go.gc_duration_seconds",
help: "A summary of the GC invocation durations.",
}, {
m: "go.gc_duration_seconds",
typ: model.MetricTypeSummary,
}, {
m: `{"go.gc_duration_seconds",quantile="0"}`,
v: 4.9351e-05,
lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0"),
}, {
m: `{"go.gc_duration_seconds",quantile="0.25",}`,
v: 7.424100000000001e-05,
lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0.25"),
}, {
m: `{"go.gc_duration_seconds",quantile="0.5",a="b"}`,
v: 8.3835e-05,
lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0.5", "a", "b"),
}, {
m: `{"go.gc_duration_seconds",quantile="0.8", a="b"}`,
v: 8.3835e-05,
lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0.8", "a", "b"),
}, {
m: `{"go.gc_duration_seconds", quantile="0.9", a="b"}`,
v: 8.3835e-05,
lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0.9", "a", "b"),
}, {
m: `{"go.gc_duration_seconds", quantile="1.0", a="b" }`,
v: 8.3835e-05,
lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "1.0", "a", "b"),
}, {
m: `{ "go.gc_duration_seconds", quantile="1.0", a="b" }`,
v: 8.3835e-05,
lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "1.0", "a", "b"),
}, {
m: `{ "go.gc_duration_seconds", quantile= "1.0", a= "b", }`,
v: 8.3835e-05,
lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "1.0", "a", "b"),
}, {
m: `{ "go.gc_duration_seconds", quantile = "1.0", a = "b" }`,
v: 8.3835e-05,
lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "1.0", "a", "b"),
}, {
m: `{"go.gc_duration_seconds_count"}`,
v: 99,
lset: labels.FromStrings("__name__", "go.gc_duration_seconds_count"),
}, {
m: `{"Heizölrückstoßabdämpfung 10€ metric with \"interesting\" {character\nchoices}","strange©™\n'quoted' \"name\""="6"}`,
v: 10.0,
lset: labels.FromStrings("__name__", `Heizölrückstoßabdämpfung 10€ metric with "interesting" {character
choices}`, "strange©™\n'quoted' \"name\"", "6"),
},
}
p := NewPromParser([]byte(input))
i := 0
var res labels.Labels
for {
et, err := p.Next()
if errors.Is(err, io.EOF) {
@ -237,6 +370,14 @@ func TestPromParseErrors(t *testing.T) {
input: "a{b=\"\xff\"} 1\n",
err: "invalid UTF-8 label value: \"\\\"\\xff\\\"\"",
},
{
input: `{"a", "b = "c"}`,
err: "expected equal, got \"c\\\"\" (\"LNAME\") while parsing: \"{\\\"a\\\", \\\"b = \\\"c\\\"\"",
},
{
input: `{"a",b\nc="d"} 1`,
err: "expected equal, got \"\\\\\" (\"INVALID\") while parsing: \"{\\\"a\\\",b\\\\\"",
},
{
input: "a true\n",
err: "strconv.ParseFloat: parsing \"true\": invalid syntax while parsing: \"a true\"",
@ -267,7 +408,7 @@ func TestPromParseErrors(t *testing.T) {
},
{
input: `{a="ok"} 1`,
err: "expected a valid start token, got \"{\" (\"INVALID\") while parsing: \"{\"",
err: "metric name not set while parsing: \"{a=\\\"ok\\\"} 1\"",
},
{
input: "# TYPE #\n#EOF\n",

57
model/textparse/protobufparse.go
View file

@ -56,6 +56,8 @@ type ProtobufParser struct {
fieldPos int
fieldsDone bool // true if no more fields of a Summary or (legacy) Histogram to be processed.
redoClassic bool // true after parsing a native histogram if we need to parse it again as a classic histogram.
// exemplarPos is the position within the exemplars slice of a native histogram.
exemplarPos int
// exemplarReturned is set to true each time an exemplar has been
// returned, and set back to false upon each Next() call.
@ -304,8 +306,9 @@ func (p *ProtobufParser) Metric(l *labels.Labels) string {
// Exemplar writes the exemplar of the current sample into the passed
// exemplar. It returns if an exemplar exists or not. In case of a native
// histogram, the legacy bucket section is still used for exemplars. To ingest
// all exemplars, call the Exemplar method repeatedly until it returns false.
// histogram, the exemplars in the native histogram will be returned.
// If this field is empty, the classic bucket section is still used for exemplars.
// To ingest all exemplars, call the Exemplar method repeatedly until it returns false.
func (p *ProtobufParser) Exemplar(ex *exemplar.Exemplar) bool {
if p.exemplarReturned && p.state == EntrySeries {
// We only ever return one exemplar per (non-native-histogram) series.
@ -317,28 +320,42 @@ func (p *ProtobufParser) Exemplar(ex *exemplar.Exemplar) bool {
case dto.MetricType_COUNTER:
exProto = m.GetCounter().GetExemplar()
case dto.MetricType_HISTOGRAM, dto.MetricType_GAUGE_HISTOGRAM:
bb := m.GetHistogram().GetBucket()
isClassic := p.state == EntrySeries
if p.fieldPos < 0 {
if isClassic {
return false // At _count or _sum.
if !isClassic && len(m.GetHistogram().GetExemplars()) > 0 {
exs := m.GetHistogram().GetExemplars()
for p.exemplarPos < len(exs) {
exProto = exs[p.exemplarPos]
p.exemplarPos++
if exProto != nil && exProto.GetTimestamp() != nil {
break
}
}
p.fieldPos = 0 // Start at 1st bucket for native histograms.
}
for p.fieldPos < len(bb) {
exProto = bb[p.fieldPos].GetExemplar()
if isClassic {
break
if exProto != nil && exProto.GetTimestamp() == nil {
return false
}
p.fieldPos++
// We deliberately drop exemplars with no timestamp only for native histograms.
if exProto != nil && (isClassic || exProto.GetTimestamp() != nil) {
break // Found a classic histogram exemplar or a native histogram exemplar with a timestamp.
} else {
bb := m.GetHistogram().GetBucket()
if p.fieldPos < 0 {
if isClassic {
return false // At _count or _sum.
}
p.fieldPos = 0 // Start at 1st bucket for native histograms.
}
for p.fieldPos < len(bb) {
exProto = bb[p.fieldPos].GetExemplar()
if isClassic {
break
}
p.fieldPos++
// We deliberately drop exemplars with no timestamp only for native histograms.
if exProto != nil && (isClassic || exProto.GetTimestamp() != nil) {
break // Found a classic histogram exemplar or a native histogram exemplar with a timestamp.
}
}
// If the last exemplar for native histograms has no timestamp, ignore it.
if !isClassic && exProto.GetTimestamp() == nil {
return false
}
}
// If the last exemplar for native histograms has no timestamp, ignore it.
if !isClassic && exProto.GetTimestamp() == nil {
return false
}
default:
return false

238
model/textparse/protobufparse_test.go
View file

@ -27,6 +27,7 @@ import (
"github.com/prometheus/prometheus/model/exemplar"
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/util/testutil"
dto "github.com/prometheus/prometheus/prompb/io/prometheus/client"
)
@ -595,6 +596,105 @@ metric: <
>
>
`,
`name: "test_histogram_with_native_histogram_exemplars"
help: "A histogram with native histogram exemplars."
type: HISTOGRAM
metric: <
histogram: <
sample_count: 175
sample_sum: 0.0008280461746287094
bucket: <
cumulative_count: 2
upper_bound: -0.0004899999999999998
>
bucket: <
cumulative_count: 4
upper_bound: -0.0003899999999999998
exemplar: <
label: <
name: "dummyID"
value: "59727"
>
value: -0.00039
timestamp: <
seconds: 1625851155
nanos: 146848499
>
>
>
bucket: <
cumulative_count: 16
upper_bound: -0.0002899999999999998
exemplar: <
label: <
name: "dummyID"
value: "5617"
>
value: -0.00029
>
>
schema: 3
zero_threshold: 2.938735877055719e-39
zero_count: 2
negative_span: <
offset: -162
length: 1
>
negative_span: <
offset: 23
length: 4
>
negative_delta: 1
negative_delta: 3
negative_delta: -2
negative_delta: -1
negative_delta: 1
positive_span: <
offset: -161
length: 1
>
positive_span: <
offset: 8
length: 3
>
positive_delta: 1
positive_delta: 2
positive_delta: -1
positive_delta: -1
exemplars: <
label: <
name: "dummyID"
value: "59780"
>
value: -0.00039
timestamp: <
seconds: 1625851155
nanos: 146848499
>
>
exemplars: <
label: <
name: "dummyID"
value: "5617"
>
value: -0.00029
>
exemplars: <
label: <
name: "dummyID"
value: "59772"
>
value: -0.00052
timestamp: <
seconds: 1625851160
nanos: 156848499
>
>
>
timestamp_ms: 1234568
>
`,
}
@ -1140,6 +1240,42 @@ func TestProtobufParse(t *testing.T) {
"__name__", "test_gaugehistogram_with_createdtimestamp",
),
},
{
m: "test_histogram_with_native_histogram_exemplars",
help: "A histogram with native histogram exemplars.",
},
{
m: "test_histogram_with_native_histogram_exemplars",
typ: model.MetricTypeHistogram,
},
{
m: "test_histogram_with_native_histogram_exemplars",
t: 1234568,
shs: &histogram.Histogram{
Count: 175,
ZeroCount: 2,
Sum: 0.0008280461746287094,
ZeroThreshold: 2.938735877055719e-39,
Schema: 3,
PositiveSpans: []histogram.Span{
{Offset: -161, Length: 1},
{Offset: 8, Length: 3},
},
NegativeSpans: []histogram.Span{
{Offset: -162, Length: 1},
{Offset: 23, Length: 4},
},
PositiveBuckets: []int64{1, 2, -1, -1},
NegativeBuckets: []int64{1, 3, -2, -1, 1},
},
lset: labels.FromStrings(
"__name__", "test_histogram_with_native_histogram_exemplars",
),
e: []exemplar.Exemplar{
{Labels: labels.FromStrings("dummyID", "59780"), Value: -0.00039, HasTs: true, Ts: 1625851155146},
{Labels: labels.FromStrings("dummyID", "59772"), Value: -0.00052, HasTs: true, Ts: 1625851160156},
},
},
},
},
{
@ -1958,6 +2094,100 @@ func TestProtobufParse(t *testing.T) {
"__name__", "test_gaugehistogram_with_createdtimestamp",
),
},
{ // 94
m: "test_histogram_with_native_histogram_exemplars",
help: "A histogram with native histogram exemplars.",
},
{ // 95
m: "test_histogram_with_native_histogram_exemplars",
typ: model.MetricTypeHistogram,
},
{ // 96
m: "test_histogram_with_native_histogram_exemplars",
t: 1234568,
shs: &histogram.Histogram{
Count: 175,
ZeroCount: 2,
Sum: 0.0008280461746287094,
ZeroThreshold: 2.938735877055719e-39,
Schema: 3,
PositiveSpans: []histogram.Span{
{Offset: -161, Length: 1},
{Offset: 8, Length: 3},
},
NegativeSpans: []histogram.Span{
{Offset: -162, Length: 1},
{Offset: 23, Length: 4},
},
PositiveBuckets: []int64{1, 2, -1, -1},
NegativeBuckets: []int64{1, 3, -2, -1, 1},
},
lset: labels.FromStrings(
"__name__", "test_histogram_with_native_histogram_exemplars",
),
e: []exemplar.Exemplar{
{Labels: labels.FromStrings("dummyID", "59780"), Value: -0.00039, HasTs: true, Ts: 1625851155146},
{Labels: labels.FromStrings("dummyID", "59772"), Value: -0.00052, HasTs: true, Ts: 1625851160156},
},
},
{ // 97
m: "test_histogram_with_native_histogram_exemplars_count",
t: 1234568,
v: 175,
lset: labels.FromStrings(
"__name__", "test_histogram_with_native_histogram_exemplars_count",
),
},
{ // 98
m: "test_histogram_with_native_histogram_exemplars_sum",
t: 1234568,
v: 0.0008280461746287094,
lset: labels.FromStrings(
"__name__", "test_histogram_with_native_histogram_exemplars_sum",
),
},
{ // 99
m: "test_histogram_with_native_histogram_exemplars_bucket\xffle\xff-0.0004899999999999998",
t: 1234568,
v: 2,
lset: labels.FromStrings(
"__name__", "test_histogram_with_native_histogram_exemplars_bucket",
"le", "-0.0004899999999999998",
),
},
{ // 100
m: "test_histogram_with_native_histogram_exemplars_bucket\xffle\xff-0.0003899999999999998",
t: 1234568,
v: 4,
lset: labels.FromStrings(
"__name__", "test_histogram_with_native_histogram_exemplars_bucket",
"le", "-0.0003899999999999998",
),
e: []exemplar.Exemplar{
{Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00039, HasTs: true, Ts: 1625851155146},
},
},
{ // 101
m: "test_histogram_with_native_histogram_exemplars_bucket\xffle\xff-0.0002899999999999998",
t: 1234568,
v: 16,
lset: labels.FromStrings(
"__name__", "test_histogram_with_native_histogram_exemplars_bucket",
"le", "-0.0002899999999999998",
),
e: []exemplar.Exemplar{
{Labels: labels.FromStrings("dummyID", "5617"), Value: -0.00029, HasTs: false},
},
},
{ // 102
m: "test_histogram_with_native_histogram_exemplars_bucket\xffle\xff+Inf",
t: 1234568,
v: 175,
lset: labels.FromStrings(
"__name__", "test_histogram_with_native_histogram_exemplars_bucket",
"le", "+Inf",
),
},
},
},
}
@ -1993,12 +2223,12 @@ func TestProtobufParse(t *testing.T) {
require.Equal(t, int64(0), exp[i].t, "i: %d", i)
}
require.Equal(t, exp[i].v, v, "i: %d", i)
require.Equal(t, exp[i].lset, res, "i: %d", i)
testutil.RequireEqual(t, exp[i].lset, res, "i: %d", i)
if len(exp[i].e) == 0 {
require.False(t, eFound, "i: %d", i)
} else {
require.True(t, eFound, "i: %d", i)
require.Equal(t, exp[i].e[0], e, "i: %d", i)
testutil.RequireEqual(t, exp[i].e[0], e, "i: %d", i)
require.False(t, p.Exemplar(&e), "too many exemplars returned, i: %d", i)
}
if exp[i].ct != 0 {
@ -2017,7 +2247,7 @@ func TestProtobufParse(t *testing.T) {
} else {
require.Equal(t, int64(0), exp[i].t, "i: %d", i)
}
require.Equal(t, exp[i].lset, res, "i: %d", i)
testutil.RequireEqual(t, exp[i].lset, res, "i: %d", i)
require.Equal(t, exp[i].m, string(m), "i: %d", i)
if shs != nil {
require.Equal(t, exp[i].shs, shs, "i: %d", i)
@ -2026,7 +2256,7 @@ func TestProtobufParse(t *testing.T) {
}
j := 0
for e := (exemplar.Exemplar{}); p.Exemplar(&e); j++ {
require.Equal(t, exp[i].e[j], e, "i: %d", i)
testutil.RequireEqual(t, exp[i].e[j], e, "i: %d", i)
e = exemplar.Exemplar{}
}
require.Len(t, exp[i].e, j, "not enough exemplars found, i: %d", i)

12
promql/engine.go
View file

@ -1494,10 +1494,14 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, annotations.Annotatio
otherInArgs[j][0].F = otherArgs[j][0].Floats[step].F
}
}
maxt := ts - offset
mint := maxt - selRange
// Evaluate the matrix selector for this series for this step.
floats, histograms = ev.matrixIterSlice(it, mint, maxt, floats, histograms)
// Evaluate the matrix selector for this series
// for this step, but only if this is the 1st
// iteration or no @ modifier has been used.
if ts == ev.startTimestamp || selVS.Timestamp == nil {
maxt := ts - offset
mint := maxt - selRange
floats, histograms = ev.matrixIterSlice(it, mint, maxt, floats, histograms)
}
if len(floats)+len(histograms) == 0 {
continue
}

15
promql/engine_test.go
View file

@ -38,6 +38,7 @@ import (
"github.com/prometheus/prometheus/util/annotations"
"github.com/prometheus/prometheus/util/stats"
"github.com/prometheus/prometheus/util/teststorage"
"github.com/prometheus/prometheus/util/testutil"
)
func TestMain(m *testing.M) {
@ -1631,7 +1632,7 @@ load 1ms
sort.Sort(expMat)
sort.Sort(res.Value.(Matrix))
}
require.Equal(t, c.result, res.Value, "query %q failed", c.query)
testutil.RequireEqual(t, c.result, res.Value, "query %q failed", c.query)
})
}
}
@ -1956,7 +1957,7 @@ func TestSubquerySelector(t *testing.T) {
require.Equal(t, c.Result.Err, res.Err)
mat := res.Value.(Matrix)
sort.Sort(mat)
require.Equal(t, c.Result.Value, mat)
testutil.RequireEqual(t, c.Result.Value, mat)
})
}
})
@ -2001,7 +2002,7 @@ load 1m
res := qry.Exec(context.Background())
require.NoError(t, res.Err)
require.Equal(t, expectedResult, res.Value)
testutil.RequireEqual(t, expectedResult, res.Value)
}
type FakeQueryLogger struct {
@ -3147,7 +3148,7 @@ func TestRangeQuery(t *testing.T) {
res := qry.Exec(context.Background())
require.NoError(t, res.Err)
require.Equal(t, c.Result, res.Value)
testutil.RequireEqual(t, c.Result, res.Value)
})
}
}
@ -4347,7 +4348,7 @@ func TestNativeHistogram_Sum_Count_Add_AvgOperator(t *testing.T) {
vector, err := res.Vector()
require.NoError(t, err)
require.Equal(t, exp, vector)
testutil.RequireEqual(t, exp, vector)
}
// sum().
@ -4605,7 +4606,7 @@ func TestNativeHistogram_SubOperator(t *testing.T) {
}
}
require.Equal(t, exp, vector)
testutil.RequireEqual(t, exp, vector)
}
// - operator.
@ -4753,7 +4754,7 @@ func TestNativeHistogram_MulDivOperator(t *testing.T) {
vector, err := res.Vector()
require.NoError(t, err)
require.Equal(t, exp, vector)
testutil.RequireEqual(t, exp, vector)
}
// histogram * scalar.

18
promql/functions.go
View file

@ -1081,6 +1081,23 @@ func funcHistogramSum(vals []parser.Value, args parser.Expressions, enh *EvalNod
return enh.Out, nil
}
// === histogram_avg(Vector parser.ValueTypeVector) (Vector, Annotations) ===
func funcHistogramAvg(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) {
inVec := vals[0].(Vector)
for _, sample := range inVec {
// Skip non-histogram samples.
if sample.H == nil {
continue
}
enh.Out = append(enh.Out, Sample{
Metric: sample.Metric.DropMetricName(),
F: sample.H.Sum / sample.H.Count,
})
}
return enh.Out, nil
}
// === histogram_stddev(Vector parser.ValueTypeVector) (Vector, Annotations) ===
func funcHistogramStdDev(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) {
inVec := vals[0].(Vector)
@ -1532,6 +1549,7 @@ var FunctionCalls = map[string]FunctionCall{
"deriv": funcDeriv,
"exp": funcExp,
"floor": funcFloor,
"histogram_avg": funcHistogramAvg,
"histogram_count": funcHistogramCount,
"histogram_fraction": funcHistogramFraction,
"histogram_quantile": funcHistogramQuantile,

5
promql/parser/functions.go
View file

@ -167,6 +167,11 @@ var Functions = map[string]*Function{
ArgTypes: []ValueType{ValueTypeVector},
ReturnType: ValueTypeVector,
},
"histogram_avg": {
Name: "histogram_avg",
ArgTypes: []ValueType{ValueTypeVector},
ReturnType: ValueTypeVector,
},
"histogram_count": {
Name: "histogram_count",
ArgTypes: []ValueType{ValueTypeVector},

22
promql/parser/generated_parser.y
View file

@ -161,7 +161,7 @@ START_METRIC_SELECTOR
// Type definitions for grammar rules.
%type <matchers> label_match_list
%type <matcher> label_matcher
%type <item> aggregate_op grouping_label match_op maybe_label metric_identifier unary_op at_modifier_preprocessors
%type <item> aggregate_op grouping_label match_op maybe_label metric_identifier unary_op at_modifier_preprocessors string_identifier
%type <labels> label_set metric
%type <lblList> label_set_list
%type <label> label_set_item
@ -582,7 +582,13 @@ label_match_list: label_match_list COMMA label_matcher
;
label_matcher : IDENTIFIER match_op STRING
{ $$ = yylex.(*parser).newLabelMatcher($1, $2, $3); }
{ $$ = yylex.(*parser).newLabelMatcher($1, $2, $3); }
| string_identifier match_op STRING
{ $$ = yylex.(*parser).newLabelMatcher($1, $2, $3); }
| string_identifier
{ $$ = yylex.(*parser).newMetricNameMatcher($1); }
| string_identifier match_op error
{ yylex.(*parser).unexpected("label matching", "string"); $$ = nil}
| IDENTIFIER match_op error
{ yylex.(*parser).unexpected("label matching", "string"); $$ = nil}
| IDENTIFIER error
@ -901,7 +907,17 @@ string_literal : STRING
PosRange: $1.PositionRange(),
}
}
;
;
string_identifier : STRING
{
$$ = Item{
Typ: METRIC_IDENTIFIER,
Pos: $1.PositionRange().Start,
Val: yylex.(*parser).unquoteString($1.Val),
}
}
;
/*
* Wrappers for optional arguments.

1013
promql/parser/generated_parser.y.go
View file

File diff suppressed because it is too large Load diff

10
promql/parser/lex_test.go
View file

@ -815,16 +815,10 @@ func TestLexer(t *testing.T) {
hasError = true
}
}
if !hasError {
t.Logf("%d: input %q", i, test.input)
require.Fail(t, "expected lexing error but did not fail")
}
require.True(t, hasError, "%d: input %q, expected lexing error but did not fail", i, test.input)
continue
}
if lastItem.Typ == ERROR {
t.Logf("%d: input %q", i, test.input)
require.Fail(t, "unexpected lexing error at position %d: %s", lastItem.Pos, lastItem)
}
require.NotEqual(t, ERROR, lastItem.Typ, "%d: input %q, unexpected lexing error at position %d: %s", i, test.input, lastItem.Pos, lastItem)
eofItem := Item{EOF, posrange.Pos(len(test.input)), ""}
require.Equal(t, lastItem, eofItem, "%d: input %q", i, test.input)

23
promql/parser/parse.go
View file

@ -417,6 +417,8 @@ func (p *parser) newBinaryExpression(lhs Node, op Item, modifiers, rhs Node) *Bi
}
func (p *parser) assembleVectorSelector(vs *VectorSelector) {
// If the metric name was set outside the braces, add a matcher for it.
// If the metric name was inside the braces we don't need to do anything.
if vs.Name != "" {
nameMatcher, err := labels.NewMatcher(labels.MatchEqual, labels.MetricName, vs.Name)
if err != nil {
@ -789,6 +791,18 @@ func (p *parser) checkAST(node Node) (typ ValueType) {
// Skip the check for non-empty matchers because an explicit
// metric name is a non-empty matcher.
break
} else {
// We also have to make sure a metric name was not set twice inside the
// braces.
foundMetricName := ""
for _, m := range n.LabelMatchers {
if m != nil && m.Name == labels.MetricName {
if foundMetricName != "" {
p.addParseErrf(n.PositionRange(), "metric name must not be set twice: %q or %q", foundMetricName, m.Value)
}
foundMetricName = m.Value
}
}
}
// A Vector selector must contain at least one non-empty matcher to prevent
@ -872,6 +886,15 @@ func (p *parser) newLabelMatcher(label, operator, value Item) *labels.Matcher {
return m
}
func (p *parser) newMetricNameMatcher(value Item) *labels.Matcher {
m, err := labels.NewMatcher(labels.MatchEqual, labels.MetricName, value.Val)
if err != nil {
p.addParseErr(value.PositionRange(), err)
}
return m
}
// addOffset is used to set the offset in the generated parser.
func (p *parser) addOffset(e Node, offset time.Duration) {
var orgoffsetp *time.Duration

68
promql/parser/parse_test.go
View file

@ -26,6 +26,7 @@ import (
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/util/testutil"
"github.com/prometheus/prometheus/promql/parser/posrange"
)
@ -473,6 +474,22 @@ var testExpr = []struct {
StartPos: 1,
},
},
{
input: ` +{"some_metric"}`,
expected: &UnaryExpr{
Op: ADD,
Expr: &VectorSelector{
LabelMatchers: []*labels.Matcher{
MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "some_metric"),
},
PosRange: posrange.PositionRange{
Start: 2,
End: 17,
},
},
StartPos: 1,
},
},
{
input: "",
fail: true,
@ -1701,6 +1718,33 @@ var testExpr = []struct {
},
},
},
{
input: `{"foo"}`,
expected: &VectorSelector{
// When a metric is named inside the braces, the Name field is not set.
LabelMatchers: []*labels.Matcher{
MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "foo"),
},
PosRange: posrange.PositionRange{
Start: 0,
End: 7,
},
},
},
{
input: `{"foo", a="bc"}`,
expected: &VectorSelector{
// When a metric is named inside the braces, the Name field is not set.
LabelMatchers: []*labels.Matcher{
MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "foo"),
MustLabelMatcher(labels.MatchEqual, "a", "bc"),
},
PosRange: posrange.PositionRange{
Start: 0,
End: 15,
},
},
},
{
input: `foo{NaN='bc'}`,
expected: &VectorSelector{
@ -1746,6 +1790,23 @@ var testExpr = []struct {
},
},
},
{
// Metric name in the middle of selector list is fine.
input: `{a="b", foo!="bar", "foo", test=~"test", bar!~"baz"}`,
expected: &VectorSelector{
LabelMatchers: []*labels.Matcher{
MustLabelMatcher(labels.MatchEqual, "a", "b"),
MustLabelMatcher(labels.MatchNotEqual, "foo", "bar"),
MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "foo"),
MustLabelMatcher(labels.MatchRegexp, "test", "test"),
MustLabelMatcher(labels.MatchNotRegexp, "bar", "baz"),
},
PosRange: posrange.PositionRange{
Start: 0,
End: 52,
},
},
},
{
input: `foo{a="b", foo!="bar", test=~"test", bar!~"baz",}`,
expected: &VectorSelector{
@ -1870,6 +1931,11 @@ var testExpr = []struct {
fail: true,
errMsg: `unexpected identifier "lol" in label matching, expected "," or "}"`,
},
{
input: `foo{"a"=}`,
fail: true,
errMsg: `unexpected "}" in label matching, expected string`,
},
// Test matrix selector.
{
input: "test[5s]",
@ -4018,7 +4084,7 @@ func TestParseSeries(t *testing.T) {
if !test.fail {
require.NoError(t, err)
require.Equal(t, test.expectedMetric, metric, "error on input '%s'", test.input)
testutil.RequireEqual(t, test.expectedMetric, metric, "error on input '%s'", test.input)
require.Equal(t, test.expectedValues, vals, "error in input '%s'", test.input)
} else {
require.Error(t, err)

8
promql/testdata/functions.test vendored
View file

@ -221,19 +221,19 @@ eval instant at 50m deriv(testcounter_reset_middle[100m])
# intercept at t=0: 6.818181818181818
# intercept at t=3000: 38.63636363636364
# intercept at t=3000+3600: 76.81818181818181
eval instant at 50m predict_linear(testcounter_reset_middle[100m], 3600)
eval instant at 50m predict_linear(testcounter_reset_middle[50m], 3600)
{} 76.81818181818181
# intercept at t = 3000+3600 = 6600
eval instant at 50m predict_linear(testcounter_reset_middle[100m] @ 3000, 3600)
eval instant at 50m predict_linear(testcounter_reset_middle[50m] @ 3000, 3600)
{} 76.81818181818181
# intercept at t = 600+3600 = 4200
eval instant at 10m predict_linear(testcounter_reset_middle[100m] @ 3000, 3600)
eval instant at 10m predict_linear(testcounter_reset_middle[50m] @ 3000, 3600)
{} 51.36363636363637
# intercept at t = 4200+3600 = 7800
eval instant at 70m predict_linear(testcounter_reset_middle[100m] @ 3000, 3600)
eval instant at 70m predict_linear(testcounter_reset_middle[50m] @ 3000, 3600)
{} 89.54545454545455
# With http_requests, there is a sample value exactly at the end of

34
promql/testdata/native_histograms.test vendored
View file

@ -11,6 +11,9 @@ eval instant at 5m histogram_count(empty_histogram)
eval instant at 5m histogram_sum(empty_histogram)
{} 0
eval instant at 5m histogram_avg(empty_histogram)
{} NaN
eval instant at 5m histogram_fraction(-Inf, +Inf, empty_histogram)
{} NaN
@ -31,6 +34,10 @@ eval instant at 5m histogram_count(single_histogram)
eval instant at 5m histogram_sum(single_histogram)
{} 5
# histogram_avg calculates the average from sum and count properties.
eval instant at 5m histogram_avg(single_histogram)
{} 1.25
# We expect half of the values to fall in the range 1 < x <= 2.
eval instant at 5m histogram_fraction(1, 2, single_histogram)
{} 0.5
@ -55,6 +62,9 @@ eval instant at 5m histogram_count(multi_histogram)
eval instant at 5m histogram_sum(multi_histogram)
{} 5
eval instant at 5m histogram_avg(multi_histogram)
{} 1.25
eval instant at 5m histogram_fraction(1, 2, multi_histogram)
{} 0.5
@ -69,6 +79,9 @@ eval instant at 50m histogram_count(multi_histogram)
eval instant at 50m histogram_sum(multi_histogram)
{} 5
eval instant at 50m histogram_avg(multi_histogram)
{} 1.25
eval instant at 50m histogram_fraction(1, 2, multi_histogram)
{} 0.5
@ -89,6 +102,9 @@ eval instant at 5m histogram_count(incr_histogram)
eval instant at 5m histogram_sum(incr_histogram)
{} 6
eval instant at 5m histogram_avg(incr_histogram)
{} 1.2
# We expect 3/5ths of the values to fall in the range 1 < x <= 2.
eval instant at 5m histogram_fraction(1, 2, incr_histogram)
{} 0.6
@ -106,6 +122,9 @@ eval instant at 50m histogram_count(incr_histogram)
eval instant at 50m histogram_sum(incr_histogram)
{} 24
eval instant at 50m histogram_avg(incr_histogram)
{} 1.7142857142857142
# We expect 12/14ths of the values to fall in the range 1 < x <= 2.
eval instant at 50m histogram_fraction(1, 2, incr_histogram)
{} 0.8571428571428571
@ -140,6 +159,9 @@ eval instant at 5m histogram_count(low_res_histogram)
eval instant at 5m histogram_sum(low_res_histogram)
{} 8
eval instant at 5m histogram_avg(low_res_histogram)
{} 1.6
# We expect all values to fall into the lower-resolution bucket with the range 1 < x <= 4.
eval instant at 5m histogram_fraction(1, 4, low_res_histogram)
{} 1
@ -157,6 +179,9 @@ eval instant at 5m histogram_count(single_zero_histogram)
eval instant at 5m histogram_sum(single_zero_histogram)
{} 0.25
eval instant at 5m histogram_avg(single_zero_histogram)
{} 0.25
# When only the zero bucket is populated, or there are negative buckets, the distribution is assumed to be equally
# distributed around zero; i.e. that there are an equal number of positive and negative observations. Therefore the
# entire distribution must lie within the full range of the zero bucket, in this case: -0.5 < x <= +0.5.
@ -179,6 +204,9 @@ eval instant at 5m histogram_count(negative_histogram)
eval instant at 5m histogram_sum(negative_histogram)
{} -5
eval instant at 5m histogram_avg(negative_histogram)
{} -1.25
# We expect half of the values to fall in the range -2 < x <= -1.
eval instant at 5m histogram_fraction(-2, -1, negative_histogram)
{} 0.5
@ -199,6 +227,9 @@ eval instant at 10m histogram_count(two_samples_histogram)
eval instant at 10m histogram_sum(two_samples_histogram)
{} -4
eval instant at 10m histogram_avg(two_samples_histogram)
{} -1
eval instant at 10m histogram_fraction(-2, -1, two_samples_histogram)
{} 0.5
@ -217,6 +248,9 @@ eval instant at 5m histogram_count(balanced_histogram)
eval instant at 5m histogram_sum(balanced_histogram)
{} 0
eval instant at 5m histogram_avg(balanced_histogram)
{} 0
eval instant at 5m histogram_fraction(0, 4, balanced_histogram)
{} 0.5

21
rules/alerting.go
View file

@ -142,6 +142,9 @@ type AlertingRule struct {
active map[uint64]*Alert
logger log.Logger
noDependentRules *atomic.Bool
noDependencyRules *atomic.Bool
}
// NewAlertingRule constructs a new AlertingRule.
@ -168,6 +171,8 @@ func NewAlertingRule(
evaluationTimestamp: atomic.NewTime(time.Time{}),
evaluationDuration: atomic.NewDuration(0),
lastError: atomic.NewError(nil),
noDependentRules: atomic.NewBool(false),
noDependencyRules: atomic.NewBool(false),
}
}
@ -317,6 +322,22 @@ func (r *AlertingRule) Restored() bool {
return r.restored.Load()
}
func (r *AlertingRule) SetNoDependentRules(noDependentRules bool) {
r.noDependentRules.Store(noDependentRules)
}
func (r *AlertingRule) NoDependentRules() bool {
return r.noDependentRules.Load()
}
func (r *AlertingRule) SetNoDependencyRules(noDependencyRules bool) {
r.noDependencyRules.Store(noDependencyRules)
}
func (r *AlertingRule) NoDependencyRules() bool {
return r.noDependencyRules.Load()
}
// resolvedRetention is the duration for which a resolved alert instance
// is kept in memory state and consequently repeatedly sent to the AlertManager.
const resolvedRetention = 15 * time.Minute

57
rules/alerting_test.go
View file

@ -31,6 +31,7 @@ import (
"github.com/prometheus/prometheus/promql/parser"
"github.com/prometheus/prometheus/storage"
"github.com/prometheus/prometheus/util/teststorage"
"github.com/prometheus/prometheus/util/testutil"
)
var testEngine = promql.NewEngine(promql.EngineOpts{
@ -180,7 +181,7 @@ func TestAlertingRuleLabelsUpdate(t *testing.T) {
}
}
require.Equal(t, result, filteredRes)
testutil.RequireEqual(t, result, filteredRes)
}
evalTime := baseTime.Add(time.Duration(len(results)) * time.Minute)
res, err := rule.Eval(context.TODO(), evalTime, EngineQueryFunc(testEngine, storage), nil, 0)
@ -278,7 +279,7 @@ func TestAlertingRuleExternalLabelsInTemplate(t *testing.T) {
}
}
require.Equal(t, result, filteredRes)
testutil.RequireEqual(t, result, filteredRes)
}
func TestAlertingRuleExternalURLInTemplate(t *testing.T) {
@ -371,7 +372,7 @@ func TestAlertingRuleExternalURLInTemplate(t *testing.T) {
}
}
require.Equal(t, result, filteredRes)
testutil.RequireEqual(t, result, filteredRes)
}
func TestAlertingRuleEmptyLabelFromTemplate(t *testing.T) {
@ -425,7 +426,7 @@ func TestAlertingRuleEmptyLabelFromTemplate(t *testing.T) {
require.Equal(t, "ALERTS_FOR_STATE", smplName)
}
}
require.Equal(t, result, filteredRes)
testutil.RequireEqual(t, result, filteredRes)
}
func TestAlertingRuleQueryInTemplate(t *testing.T) {
@ -718,7 +719,7 @@ func TestSendAlertsDontAffectActiveAlerts(t *testing.T) {
// The relabel rule changes a1=1 to a1=bug.
// But the labels with the AlertingRule should not be changed.
require.Equal(t, labels.FromStrings("a1", "1"), rule.active[h].Labels)
testutil.RequireEqual(t, labels.FromStrings("a1", "1"), rule.active[h].Labels)
}
func TestKeepFiringFor(t *testing.T) {
@ -823,7 +824,7 @@ func TestKeepFiringFor(t *testing.T) {
}
}
require.Equal(t, result, filteredRes)
testutil.RequireEqual(t, result, filteredRes)
}
evalTime := baseTime.Add(time.Duration(len(results)) * time.Minute)
res, err := rule.Eval(context.TODO(), evalTime, EngineQueryFunc(testEngine, storage), nil, 0)
@ -870,7 +871,7 @@ func TestPendingAndKeepFiringFor(t *testing.T) {
for _, smpl := range res {
smplName := smpl.Metric.Get("__name__")
if smplName == "ALERTS" {
require.Equal(t, result, smpl)
testutil.RequireEqual(t, result, smpl)
} else {
// If not 'ALERTS', it has to be 'ALERTS_FOR_STATE'.
require.Equal(t, "ALERTS_FOR_STATE", smplName)
@ -920,3 +921,45 @@ func TestAlertingEvalWithOrigin(t *testing.T) {
require.NoError(t, err)
require.Equal(t, detail, NewRuleDetail(rule))
}
func TestAlertingRule_SetNoDependentRules(t *testing.T) {
rule := NewAlertingRule(
"test",
&parser.NumberLiteral{Val: 1},
time.Minute,
0,
labels.FromStrings("test", "test"),
labels.EmptyLabels(),
labels.EmptyLabels(),
"",
true, log.NewNopLogger(),
)
require.False(t, rule.NoDependentRules())
rule.SetNoDependentRules(false)
require.False(t, rule.NoDependentRules())
rule.SetNoDependentRules(true)
require.True(t, rule.NoDependentRules())
}
func TestAlertingRule_SetNoDependencyRules(t *testing.T) {
rule := NewAlertingRule(
"test",
&parser.NumberLiteral{Val: 1},
time.Minute,
0,
labels.FromStrings("test", "test"),
labels.EmptyLabels(),
labels.EmptyLabels(),
"",
true, log.NewNopLogger(),
)
require.False(t, rule.NoDependencyRules())
rule.SetNoDependencyRules(false)
require.False(t, rule.NoDependencyRules())
rule.SetNoDependencyRules(true)
require.True(t, rule.NoDependencyRules())
}

8
rules/group.go
View file

@ -464,7 +464,7 @@ func (g *Group) Eval(ctx context.Context, ts time.Time) {
}(time.Now())
if sp.SpanContext().IsSampled() && sp.SpanContext().HasTraceID() {
logger = log.WithPrefix(logger, "traceID", sp.SpanContext().TraceID())
logger = log.WithPrefix(logger, "trace_id", sp.SpanContext().TraceID())
}
g.metrics.EvalTotal.WithLabelValues(GroupKey(g.File(), g.Name())).Inc()
@ -579,7 +579,7 @@ func (g *Group) Eval(ctx context.Context, ts time.Time) {
// If the rule has no dependencies, it can run concurrently because no other rules in this group depend on its output.
// Try run concurrently if there are slots available.
if ctrl := g.concurrencyController; ctrl.RuleEligible(g, rule) && ctrl.Allow() {
if ctrl := g.concurrencyController; isRuleEligibleForConcurrentExecution(rule) && ctrl.Allow() {
wg.Add(1)
go eval(i, rule, func() {
@ -1008,3 +1008,7 @@ func buildDependencyMap(rules []Rule) dependencyMap {
return dependencies
}
func isRuleEligibleForConcurrentExecution(rule Rule) bool {
return rule.NoDependentRules() && rule.NoDependencyRules()
}

43
rules/manager.go
View file

@ -119,6 +119,7 @@ type ManagerOptions struct {
MaxConcurrentEvals int64
ConcurrentEvalsEnabled bool
RuleConcurrencyController RuleConcurrencyController
RuleDependencyController RuleDependencyController
Metrics *Metrics
}
@ -142,6 +143,10 @@ func NewManager(o *ManagerOptions) *Manager {
}
}
if o.RuleDependencyController == nil {
o.RuleDependencyController = ruleDependencyController{}
}
m := &Manager{
groups: map[string]*Group{},
opts: o,
@ -188,8 +193,6 @@ func (m *Manager) Update(interval time.Duration, files []string, externalLabels
m.mtx.Lock()
defer m.mtx.Unlock()
m.opts.RuleConcurrencyController.Invalidate()
groups, errs := m.LoadGroups(interval, externalLabels, externalURL, groupEvalIterationFunc, files...)
if errs != nil {
@ -322,6 +325,9 @@ func (m *Manager) LoadGroups(
))
}
// Check dependencies between rules and store it on the Rule itself.
m.opts.RuleDependencyController.AnalyseRules(rules)
groups[GroupKey(fn, rg.Name)] = NewGroup(GroupOptions{
Name: rg.Name,
File: fn,
@ -418,24 +424,35 @@ func SendAlerts(s Sender, externalURL string) NotifyFunc {
}
}
// RuleConcurrencyController controls whether rules can be evaluated concurrently. Its purpose is to bound the amount
// of concurrency in rule evaluations to avoid overwhelming the Prometheus server with additional query load and ensure
// the correctness of rules running concurrently. Concurrency is controlled globally, not on a per-group basis.
type RuleConcurrencyController interface {
// RuleEligible determines if the rule can guarantee correct results while running concurrently.
RuleEligible(g *Group, r Rule) bool
// RuleDependencyController controls whether a set of rules have dependencies between each other.
type RuleDependencyController interface {
// AnalyseRules analyses dependencies between the input rules. For each rule that it's guaranteed
// not having any dependants and/or dependency, this function should call Rule.SetNoDependentRules(true)
// and/or Rule.SetNoDependencyRules(true).
AnalyseRules(rules []Rule)
}
type ruleDependencyController struct{}
// AnalyseRules implements RuleDependencyController.
func (c ruleDependencyController) AnalyseRules(rules []Rule) {
depMap := buildDependencyMap(rules)
for _, r := range rules {
r.SetNoDependentRules(depMap.dependents(r) == 0)
r.SetNoDependencyRules(depMap.dependencies(r) == 0)
}
}
// RuleConcurrencyController controls concurrency for rules that are safe to be evaluated concurrently.
// Its purpose is to bound the amount of concurrency in rule evaluations to avoid overwhelming the Prometheus
// server with additional query load. Concurrency is controlled globally, not on a per-group basis.
type RuleConcurrencyController interface {
// Allow determines whether any concurrent evaluation slots are available.
// If Allow() returns true, then Done() must be called to release the acquired slot.
Allow() bool
// Done releases a concurrent evaluation slot.
Done()
// Invalidate instructs the controller to invalidate its state.
// This should be called when groups are modified (during a reload, for instance), because the controller may
// store some state about each group in order to more efficiently determine rule eligibility.
Invalidate()
}
// concurrentRuleEvalController holds a weighted semaphore which controls the concurrent evaluation of rules.

67
rules/manager_test.go
View file

@ -42,6 +42,7 @@ import (
"github.com/prometheus/prometheus/tsdb/chunkenc"
"github.com/prometheus/prometheus/tsdb/tsdbutil"
"github.com/prometheus/prometheus/util/teststorage"
prom_testutil "github.com/prometheus/prometheus/util/testutil"
)
func TestMain(m *testing.M) {
@ -180,7 +181,7 @@ func TestAlertingRule(t *testing.T) {
sort.Slice(filteredRes, func(i, j int) bool {
return labels.Compare(filteredRes[i].Metric, filteredRes[j].Metric) < 0
})
require.Equal(t, test.result, filteredRes)
prom_testutil.RequireEqual(t, test.result, filteredRes)
for _, aa := range rule.ActiveAlerts() {
require.Zero(t, aa.Labels.Get(model.MetricNameLabel), "%s label set on active alert: %s", model.MetricNameLabel, aa.Labels)
@ -330,7 +331,7 @@ func TestForStateAddSamples(t *testing.T) {
sort.Slice(filteredRes, func(i, j int) bool {
return labels.Compare(filteredRes[i].Metric, filteredRes[j].Metric) < 0
})
require.Equal(t, test.result, filteredRes)
prom_testutil.RequireEqual(t, test.result, filteredRes)
for _, aa := range rule.ActiveAlerts() {
require.Zero(t, aa.Labels.Get(model.MetricNameLabel), "%s label set on active alert: %s", model.MetricNameLabel, aa.Labels)
@ -1314,6 +1315,8 @@ func TestRuleGroupEvalIterationFunc(t *testing.T) {
evaluationTimestamp: atomic.NewTime(time.Time{}),
evaluationDuration: atomic.NewDuration(0),
lastError: atomic.NewError(nil),
noDependentRules: atomic.NewBool(false),
noDependencyRules: atomic.NewBool(false),
}
group := NewGroup(GroupOptions{
@ -1407,6 +1410,66 @@ func TestNativeHistogramsInRecordingRules(t *testing.T) {
require.Equal(t, chunkenc.ValNone, it.Next())
}
func TestManager_LoadGroups_ShouldCheckWhetherEachRuleHasDependentsAndDependencies(t *testing.T) {
storage := teststorage.New(t)
t.Cleanup(func() {
require.NoError(t, storage.Close())
})
ruleManager := NewManager(&ManagerOptions{
Context: context.Background(),
Logger: log.NewNopLogger(),
Appendable: storage,
QueryFunc: func(ctx context.Context, q string, ts time.Time) (promql.Vector, error) { return nil, nil },
})
t.Run("load a mix of dependent and independent rules", func(t *testing.T) {
groups, errs := ruleManager.LoadGroups(time.Second, labels.EmptyLabels(), "", nil, []string{"fixtures/rules_multiple.yaml"}...)
require.Empty(t, errs)
require.Len(t, groups, 1)
expected := map[string]struct {
noDependentRules bool
noDependencyRules bool
}{
"job:http_requests:rate1m": {
noDependentRules: true,
noDependencyRules: true,
},
"job:http_requests:rate5m": {
noDependentRules: true,
noDependencyRules: true,
},
"job:http_requests:rate15m": {
noDependentRules: true,
noDependencyRules: false,
},
"TooManyRequests": {
noDependentRules: false,
noDependencyRules: true,
},
}
for _, r := range ruleManager.Rules() {
exp, ok := expected[r.Name()]
require.Truef(t, ok, "rule: %s", r.String())
require.Equalf(t, exp.noDependentRules, r.NoDependentRules(), "rule: %s", r.String())
require.Equalf(t, exp.noDependencyRules, r.NoDependencyRules(), "rule: %s", r.String())
}
})
t.Run("load only independent rules", func(t *testing.T) {
groups, errs := ruleManager.LoadGroups(time.Second, labels.EmptyLabels(), "", nil, []string{"fixtures/rules_multiple_independent.yaml"}...)
require.Empty(t, errs)
require.Len(t, groups, 1)
for _, r := range ruleManager.Rules() {
require.Truef(t, r.NoDependentRules(), "rule: %s", r.String())
require.Truef(t, r.NoDependencyRules(), "rule: %s", r.String())
}
})
}
func TestDependencyMap(t *testing.T) {
ctx := context.Background()
opts := &ManagerOptions{

18
rules/origin.go
View file

@ -28,6 +28,14 @@ type RuleDetail struct {
Query string
Labels labels.Labels
Kind string
// NoDependentRules is set to true if it's guaranteed that in the rule group there's no other rule
// which depends on this one.
NoDependentRules bool
// NoDependencyRules is set to true if it's guaranteed that this rule doesn't depend on any other
// rule within the rule group.
NoDependencyRules bool
}
const (
@ -48,10 +56,12 @@ func NewRuleDetail(r Rule) RuleDetail {
}
return RuleDetail{
Name: r.Name(),
Query: r.Query().String(),
Labels: r.Labels(),
Kind: kind,
Name: r.Name(),
Query: r.Query().String(),
Labels: r.Labels(),
Kind: kind,
NoDependentRules: r.NoDependentRules(),
NoDependencyRules: r.NoDependencyRules(),
}
}

65
rules/origin_test.go
View file

@ -19,6 +19,7 @@ import (
"testing"
"time"
"github.com/go-kit/log"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/model/labels"
@ -43,9 +44,73 @@ func (u unknownRule) SetEvaluationDuration(time.Duration) {}
func (u unknownRule) GetEvaluationDuration() time.Duration { return 0 }
func (u unknownRule) SetEvaluationTimestamp(time.Time) {}
func (u unknownRule) GetEvaluationTimestamp() time.Time { return time.Time{} }
func (u unknownRule) SetNoDependentRules(bool) {}
func (u unknownRule) NoDependentRules() bool { return false }
func (u unknownRule) SetNoDependencyRules(bool) {}
func (u unknownRule) NoDependencyRules() bool { return false }
func TestNewRuleDetailPanics(t *testing.T) {
require.PanicsWithValue(t, `unknown rule type "rules.unknownRule"`, func() {
NewRuleDetail(unknownRule{})
})
}
func TestFromOriginContext(t *testing.T) {
t.Run("should return zero value if RuleDetail is missing in the context", func(t *testing.T) {
detail := FromOriginContext(context.Background())
require.Zero(t, detail)
// The zero value for NoDependentRules must be the most conservative option.
require.False(t, detail.NoDependentRules)
// The zero value for NoDependencyRules must be the most conservative option.
require.False(t, detail.NoDependencyRules)
})
}
func TestNewRuleDetail(t *testing.T) {
t.Run("should populate NoDependentRules and NoDependencyRules for a RecordingRule", func(t *testing.T) {
rule := NewRecordingRule("test", &parser.NumberLiteral{Val: 1}, labels.EmptyLabels())
detail := NewRuleDetail(rule)
require.False(t, detail.NoDependentRules)
require.False(t, detail.NoDependencyRules)
rule.SetNoDependentRules(true)
detail = NewRuleDetail(rule)
require.True(t, detail.NoDependentRules)
require.False(t, detail.NoDependencyRules)
rule.SetNoDependencyRules(true)
detail = NewRuleDetail(rule)
require.True(t, detail.NoDependentRules)
require.True(t, detail.NoDependencyRules)
})
t.Run("should populate NoDependentRules and NoDependencyRules for a AlertingRule", func(t *testing.T) {
rule := NewAlertingRule(
"test",
&parser.NumberLiteral{Val: 1},
time.Minute,
0,
labels.FromStrings("test", "test"),
labels.EmptyLabels(),
labels.EmptyLabels(),
"",
true, log.NewNopLogger(),
)
detail := NewRuleDetail(rule)
require.False(t, detail.NoDependentRules)
require.False(t, detail.NoDependencyRules)
rule.SetNoDependentRules(true)
detail = NewRuleDetail(rule)
require.True(t, detail.NoDependentRules)
require.False(t, detail.NoDependencyRules)
rule.SetNoDependencyRules(true)
detail = NewRuleDetail(rule)
require.True(t, detail.NoDependentRules)
require.True(t, detail.NoDependencyRules)
})
}

21
rules/recording.go
View file

@ -41,6 +41,9 @@ type RecordingRule struct {
lastError *atomic.Error
// Duration of how long it took to evaluate the recording rule.
evaluationDuration *atomic.Duration
noDependentRules *atomic.Bool
noDependencyRules *atomic.Bool
}
// NewRecordingRule returns a new recording rule.
@ -53,6 +56,8 @@ func NewRecordingRule(name string, vector parser.Expr, lset labels.Labels) *Reco
evaluationTimestamp: atomic.NewTime(time.Time{}),
evaluationDuration: atomic.NewDuration(0),
lastError: atomic.NewError(nil),
noDependentRules: atomic.NewBool(false),
noDependencyRules: atomic.NewBool(false),
}
}
@ -166,3 +171,19 @@ func (rule *RecordingRule) SetEvaluationTimestamp(ts time.Time) {
func (rule *RecordingRule) GetEvaluationTimestamp() time.Time {
return rule.evaluationTimestamp.Load()
}
func (rule *RecordingRule) SetNoDependentRules(noDependentRules bool) {
rule.noDependentRules.Store(noDependentRules)
}
func (rule *RecordingRule) NoDependentRules() bool {
return rule.noDependentRules.Load()
}
func (rule *RecordingRule) SetNoDependencyRules(noDependencyRules bool) {
rule.noDependencyRules.Store(noDependencyRules)
}
func (rule *RecordingRule) NoDependencyRules() bool {
return rule.noDependencyRules.Load()
}

25
rules/recording_test.go
View file

@ -25,6 +25,7 @@ import (
"github.com/prometheus/prometheus/promql"
"github.com/prometheus/prometheus/promql/parser"
"github.com/prometheus/prometheus/util/teststorage"
"github.com/prometheus/prometheus/util/testutil"
)
var (
@ -126,7 +127,7 @@ func TestRuleEval(t *testing.T) {
rule := NewRecordingRule("test_rule", scenario.expr, scenario.ruleLabels)
result, err := rule.Eval(context.TODO(), ruleEvaluationTime, EngineQueryFunc(testEngine, storage), nil, 0)
require.NoError(t, err)
require.Equal(t, scenario.expected, result)
testutil.RequireEqual(t, scenario.expected, result)
})
}
}
@ -249,3 +250,25 @@ func TestRecordingEvalWithOrigin(t *testing.T) {
require.NoError(t, err)
require.Equal(t, detail, NewRuleDetail(rule))
}
func TestRecordingRule_SetNoDependentRules(t *testing.T) {
rule := NewRecordingRule("1", &parser.NumberLiteral{Val: 1}, labels.EmptyLabels())
require.False(t, rule.NoDependentRules())
rule.SetNoDependentRules(false)
require.False(t, rule.NoDependentRules())
rule.SetNoDependentRules(true)
require.True(t, rule.NoDependentRules())
}
func TestRecordingRule_SetNoDependencyRules(t *testing.T) {
rule := NewRecordingRule("1", &parser.NumberLiteral{Val: 1}, labels.EmptyLabels())
require.False(t, rule.NoDependencyRules())
rule.SetNoDependencyRules(false)
require.False(t, rule.NoDependencyRules())
rule.SetNoDependencyRules(true)
require.True(t, rule.NoDependencyRules())
}

16
rules/rule.go
View file

@ -61,4 +61,20 @@ type Rule interface {
// GetEvaluationTimestamp returns last evaluation timestamp.
// NOTE: Used dynamically by rules.html template.
GetEvaluationTimestamp() time.Time
// SetNoDependentRules sets whether there's no other rule in the rule group that depends on this rule.
SetNoDependentRules(bool)
// NoDependentRules returns true if it's guaranteed that in the rule group there's no other rule
// which depends on this one. In case this function returns false there's no such guarantee, which
// means there may or may not be other rules depending on this one.
NoDependentRules() bool
// SetNoDependencyRules sets whether this rule doesn't depend on the output of any rule in the rule group.
SetNoDependencyRules(bool)
// NoDependencyRules returns true if it's guaranteed that this rule doesn't depend on the output of
// any other rule in the group. In case this function returns false there's no such guarantee, which
// means the rule may or may not depend on other rules.
NoDependencyRules() bool
}

6
scrape/helpers_test.go
View file

@ -18,6 +18,7 @@ import (
"context"
"encoding/binary"
"fmt"
"math"
"math/rand"
"strings"
"sync"
@ -71,6 +72,11 @@ type floatSample struct {
f float64
}
func equalFloatSamples(a, b floatSample) bool {
// Compare Float64bits so NaN values which are exactly the same will compare equal.
return labels.Equal(a.metric, b.metric) && a.t == b.t && math.Float64bits(a.f) == math.Float64bits(b.f)
}
type histogramSample struct {
t int64
h *histogram.Histogram

65
scrape/manager_test.go
View file

@ -40,6 +40,7 @@ import (
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/model/relabel"
"github.com/prometheus/prometheus/util/runutil"
"github.com/prometheus/prometheus/util/testutil"
)
func TestPopulateLabels(t *testing.T) {
@ -449,8 +450,8 @@ func TestPopulateLabels(t *testing.T) {
require.NoError(t, err)
}
require.Equal(t, c.in, in)
require.Equal(t, c.res, res)
require.Equal(t, c.resOrig, orig)
testutil.RequireEqual(t, c.res, res)
testutil.RequireEqual(t, c.resOrig, orig)
}
}
@ -458,9 +459,9 @@ func loadConfiguration(t testing.TB, c string) *config.Config {
t.Helper()
cfg := &config.Config{}
if err := yaml.UnmarshalStrict([]byte(c), cfg); err != nil {
t.Fatalf("Unable to load YAML config: %s", err)
}
err := yaml.UnmarshalStrict([]byte(c), cfg)
require.NoError(t, err, "Unable to load YAML config.")
return cfg
}
@ -533,42 +534,38 @@ scrape_configs:
}
// Apply the initial configuration.
if err := scrapeManager.ApplyConfig(cfg1); err != nil {
t.Fatalf("unable to apply configuration: %s", err)
}
err = scrapeManager.ApplyConfig(cfg1)
require.NoError(t, err, "Unable to apply configuration.")
select {
case <-ch:
t.Fatal("reload happened")
require.FailNow(t, "Reload happened.")
default:
}
// Apply a configuration for which the reload fails.
if err := scrapeManager.ApplyConfig(cfg2); err == nil {
t.Fatalf("expecting error but got none")
}
err = scrapeManager.ApplyConfig(cfg2)
require.Error(t, err, "Expecting error but got none.")
select {
case <-ch:
t.Fatal("reload happened")
require.FailNow(t, "Reload happened.")
default:
}
// Apply a configuration for which the reload succeeds.
if err := scrapeManager.ApplyConfig(cfg3); err != nil {
t.Fatalf("unable to apply configuration: %s", err)
}
err = scrapeManager.ApplyConfig(cfg3)
require.NoError(t, err, "Unable to apply configuration.")
select {
case <-ch:
default:
t.Fatal("reload didn't happen")
require.FailNow(t, "Reload didn't happen.")
}
// Re-applying the same configuration shouldn't trigger a reload.
if err := scrapeManager.ApplyConfig(cfg3); err != nil {
t.Fatalf("unable to apply configuration: %s", err)
}
err = scrapeManager.ApplyConfig(cfg3)
require.NoError(t, err, "Unable to apply configuration.")
select {
case <-ch:
t.Fatal("reload happened")
require.FailNow(t, "Reload happened.")
default:
}
}
@ -595,7 +592,7 @@ func TestManagerTargetsUpdates(t *testing.T) {
select {
case ts <- tgSent:
case <-time.After(10 * time.Millisecond):
t.Error("Scrape manager's channel remained blocked after the set threshold.")
require.Fail(t, "Scrape manager's channel remained blocked after the set threshold.")
}
}
@ -609,7 +606,7 @@ func TestManagerTargetsUpdates(t *testing.T) {
select {
case <-m.triggerReload:
default:
t.Error("No scrape loops reload was triggered after targets update.")
require.Fail(t, "No scrape loops reload was triggered after targets update.")
}
}
@ -622,9 +619,8 @@ global:
`
cfg := &config.Config{}
if err := yaml.UnmarshalStrict([]byte(cfgText), cfg); err != nil {
t.Fatalf("Unable to load YAML config cfgYaml: %s", err)
}
err := yaml.UnmarshalStrict([]byte(cfgText), cfg)
require.NoError(t, err, "Unable to load YAML config cfgYaml.")
return cfg
}
@ -636,25 +632,18 @@ global:
// Load the first config.
cfg1 := getConfig("ha1")
if err := scrapeManager.setOffsetSeed(cfg1.GlobalConfig.ExternalLabels); err != nil {
t.Error(err)
}
err = scrapeManager.setOffsetSeed(cfg1.GlobalConfig.ExternalLabels)
require.NoError(t, err)
offsetSeed1 := scrapeManager.offsetSeed
if offsetSeed1 == 0 {
t.Error("Offset seed has to be a hash of uint64")
}
require.NotZero(t, offsetSeed1, "Offset seed has to be a hash of uint64.")
// Load the first config.
cfg2 := getConfig("ha2")
if err := scrapeManager.setOffsetSeed(cfg2.GlobalConfig.ExternalLabels); err != nil {
t.Error(err)
}
require.NoError(t, scrapeManager.setOffsetSeed(cfg2.GlobalConfig.ExternalLabels))
offsetSeed2 := scrapeManager.offsetSeed
if offsetSeed1 == offsetSeed2 {
t.Error("Offset seed should not be the same on different set of external labels")
}
require.NotEqual(t, offsetSeed1, offsetSeed2, "Offset seed should not be the same on different set of external labels.")
}
func TestManagerScrapePools(t *testing.T) {

180
scrape/scrape_test.go
View file

@ -32,6 +32,7 @@ import (
"github.com/go-kit/log"
"github.com/gogo/protobuf/proto"
"github.com/google/go-cmp/cmp"
"github.com/prometheus/client_golang/prometheus"
dto "github.com/prometheus/client_model/go"
config_util "github.com/prometheus/common/config"
@ -72,15 +73,11 @@ func TestNewScrapePool(t *testing.T) {
sp, _ = newScrapePool(cfg, app, 0, nil, nil, &Options{}, newTestScrapeMetrics(t))
)
if a, ok := sp.appendable.(*nopAppendable); !ok || a != app {
t.Fatalf("Wrong sample appender")
}
if sp.config != cfg {
t.Fatalf("Wrong scrape config")
}
if sp.newLoop == nil {
t.Fatalf("newLoop function not initialized")
}
a, ok := sp.appendable.(*nopAppendable)
require.True(t, ok, "Failure to append.")
require.Equal(t, app, a, "Wrong sample appender.")
require.Equal(t, cfg, sp.config, "Wrong scrape config.")
require.NotNil(t, sp.newLoop, "newLoop function not initialized.")
}
func TestDroppedTargetsList(t *testing.T) {
@ -233,12 +230,10 @@ func TestScrapePoolStop(t *testing.T) {
select {
case <-time.After(5 * time.Second):
t.Fatalf("scrapeLoop.stop() did not return as expected")
require.Fail(t, "scrapeLoop.stop() did not return as expected")
case <-done:
// This should have taken at least as long as the last target slept.
if time.Since(stopTime) < time.Duration(numTargets*20)*time.Millisecond {
t.Fatalf("scrapeLoop.stop() exited before all targets stopped")
}
require.GreaterOrEqual(t, time.Since(stopTime), time.Duration(numTargets*20)*time.Millisecond, "scrapeLoop.stop() exited before all targets stopped")
}
mtx.Lock()
@ -324,12 +319,10 @@ func TestScrapePoolReload(t *testing.T) {
select {
case <-time.After(5 * time.Second):
t.Fatalf("scrapeLoop.reload() did not return as expected")
require.FailNow(t, "scrapeLoop.reload() did not return as expected")
case <-done:
// This should have taken at least as long as the last target slept.
if time.Since(reloadTime) < time.Duration(numTargets*20)*time.Millisecond {
t.Fatalf("scrapeLoop.stop() exited before all targets stopped")
}
require.GreaterOrEqual(t, time.Since(reloadTime), time.Duration(numTargets*20)*time.Millisecond, "scrapeLoop.stop() exited before all targets stopped")
}
mtx.Lock()
@ -703,13 +696,13 @@ func TestScrapeLoopStopBeforeRun(t *testing.T) {
select {
case <-stopDone:
t.Fatalf("Stopping terminated before run exited successfully")
require.FailNow(t, "Stopping terminated before run exited successfully.")
case <-time.After(500 * time.Millisecond):
}
// Running the scrape loop must exit before calling the scraper even once.
scraper.scrapeFunc = func(context.Context, io.Writer) error {
t.Fatalf("scraper was called for terminated scrape loop")
require.FailNow(t, "Scraper was called for terminated scrape loop.")
return nil
}
@ -722,13 +715,13 @@ func TestScrapeLoopStopBeforeRun(t *testing.T) {
select {
case <-runDone:
case <-time.After(1 * time.Second):
t.Fatalf("Running terminated scrape loop did not exit")
require.FailNow(t, "Running terminated scrape loop did not exit.")
}
select {
case <-stopDone:
case <-time.After(1 * time.Second):
t.Fatalf("Stopping did not terminate after running exited")
require.FailNow(t, "Stopping did not terminate after running exited.")
}
}
@ -765,14 +758,13 @@ func TestScrapeLoopStop(t *testing.T) {
select {
case <-signal:
case <-time.After(5 * time.Second):
t.Fatalf("Scrape wasn't stopped.")
require.FailNow(t, "Scrape wasn't stopped.")
}
// We expected 1 actual sample for each scrape plus 5 for report samples.
// At least 2 scrapes were made, plus the final stale markers.
if len(appender.resultFloats) < 6*3 || len(appender.resultFloats)%6 != 0 {
t.Fatalf("Expected at least 3 scrapes with 6 samples each, got %d samples", len(appender.resultFloats))
}
require.GreaterOrEqual(t, len(appender.resultFloats), 6*3, "Expected at least 3 scrapes with 6 samples each.")
require.Zero(t, len(appender.resultFloats)%6, "There is a scrape with missing samples.")
// All samples in a scrape must have the same timestamp.
var ts int64
for i, s := range appender.resultFloats {
@ -785,9 +777,7 @@ func TestScrapeLoopStop(t *testing.T) {
}
// All samples from the last scrape must be stale markers.
for _, s := range appender.resultFloats[len(appender.resultFloats)-5:] {
if !value.IsStaleNaN(s.f) {
t.Fatalf("Appended last sample not as expected. Wanted: stale NaN Got: %x", math.Float64bits(s.f))
}
require.True(t, value.IsStaleNaN(s.f), "Appended last sample not as expected. Wanted: stale NaN Got: %x", math.Float64bits(s.f))
}
}
@ -843,9 +833,9 @@ func TestScrapeLoopRun(t *testing.T) {
select {
case <-signal:
case <-time.After(5 * time.Second):
t.Fatalf("Cancellation during initial offset failed")
require.FailNow(t, "Cancellation during initial offset failed.")
case err := <-errc:
t.Fatalf("Unexpected error: %s", err)
require.FailNow(t, "Unexpected error: %s", err)
}
// The provided timeout must cause cancellation of the context passed down to the
@ -873,11 +863,9 @@ func TestScrapeLoopRun(t *testing.T) {
select {
case err := <-errc:
if !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("Expected timeout error but got: %s", err)
}
require.ErrorIs(t, err, context.DeadlineExceeded)
case <-time.After(3 * time.Second):
t.Fatalf("Expected timeout error but got none")
require.FailNow(t, "Expected timeout error but got none.")
}
// We already caught the timeout error and are certainly in the loop.
@ -890,9 +878,9 @@ func TestScrapeLoopRun(t *testing.T) {
case <-signal:
// Loop terminated as expected.
case err := <-errc:
t.Fatalf("Unexpected error: %s", err)
require.FailNow(t, "Unexpected error: %s", err)
case <-time.After(3 * time.Second):
t.Fatalf("Loop did not terminate on context cancellation")
require.FailNow(t, "Loop did not terminate on context cancellation")
}
}
@ -912,7 +900,7 @@ func TestScrapeLoopForcedErr(t *testing.T) {
sl.setForcedError(forcedErr)
scraper.scrapeFunc = func(context.Context, io.Writer) error {
t.Fatalf("should not be scraped")
require.FailNow(t, "Should not be scraped.")
return nil
}
@ -923,18 +911,16 @@ func TestScrapeLoopForcedErr(t *testing.T) {
select {
case err := <-errc:
if !errors.Is(err, forcedErr) {
t.Fatalf("Expected forced error but got: %s", err)
}
require.ErrorIs(t, err, forcedErr)
case <-time.After(3 * time.Second):
t.Fatalf("Expected forced error but got none")
require.FailNow(t, "Expected forced error but got none.")
}
cancel()
select {
case <-signal:
case <-time.After(5 * time.Second):
t.Fatalf("Scrape not stopped")
require.FailNow(t, "Scrape not stopped.")
}
}
@ -1141,7 +1127,7 @@ func TestScrapeLoopRunCreatesStaleMarkersOnFailedScrape(t *testing.T) {
select {
case <-signal:
case <-time.After(5 * time.Second):
t.Fatalf("Scrape wasn't stopped.")
require.FailNow(t, "Scrape wasn't stopped.")
}
// 1 successfully scraped sample, 1 stale marker after first fail, 5 report samples for
@ -1188,7 +1174,7 @@ func TestScrapeLoopRunCreatesStaleMarkersOnParseFailure(t *testing.T) {
select {
case <-signal:
case <-time.After(5 * time.Second):
t.Fatalf("Scrape wasn't stopped.")
require.FailNow(t, "Scrape wasn't stopped.")
}
// 1 successfully scraped sample, 1 stale marker after first fail, 5 report samples for
@ -1220,19 +1206,15 @@ func TestScrapeLoopCache(t *testing.T) {
scraper.scrapeFunc = func(ctx context.Context, w io.Writer) error {
switch numScrapes {
case 1, 2:
if _, ok := sl.cache.series["metric_a"]; !ok {
t.Errorf("metric_a missing from cache after scrape %d", numScrapes)
}
if _, ok := sl.cache.series["metric_b"]; !ok {
t.Errorf("metric_b missing from cache after scrape %d", numScrapes)
}
_, ok := sl.cache.series["metric_a"]
require.True(t, ok, "metric_a missing from cache after scrape %d", numScrapes)
_, ok = sl.cache.series["metric_b"]
require.True(t, ok, "metric_b missing from cache after scrape %d", numScrapes)
case 3:
if _, ok := sl.cache.series["metric_a"]; !ok {
t.Errorf("metric_a missing from cache after scrape %d", numScrapes)
}
if _, ok := sl.cache.series["metric_b"]; ok {
t.Errorf("metric_b present in cache after scrape %d", numScrapes)
}
_, ok := sl.cache.series["metric_a"]
require.True(t, ok, "metric_a missing from cache after scrape %d", numScrapes)
_, ok = sl.cache.series["metric_b"]
require.False(t, ok, "metric_b present in cache after scrape %d", numScrapes)
}
numScrapes++
@ -1257,7 +1239,7 @@ func TestScrapeLoopCache(t *testing.T) {
select {
case <-signal:
case <-time.After(5 * time.Second):
t.Fatalf("Scrape wasn't stopped.")
require.FailNow(t, "Scrape wasn't stopped.")
}
// 1 successfully scraped sample, 1 stale marker after first fail, 5 report samples for
@ -1305,12 +1287,10 @@ func TestScrapeLoopCacheMemoryExhaustionProtection(t *testing.T) {
select {
case <-signal:
case <-time.After(5 * time.Second):
t.Fatalf("Scrape wasn't stopped.")
require.FailNow(t, "Scrape wasn't stopped.")
}
if len(sl.cache.series) > 2000 {
t.Fatalf("More than 2000 series cached. Got: %d", len(sl.cache.series))
}
require.LessOrEqual(t, len(sl.cache.series), 2000, "More than 2000 series cached.")
}
func TestScrapeLoopAppend(t *testing.T) {
@ -1352,7 +1332,7 @@ func TestScrapeLoopAppend(t *testing.T) {
// Honor Labels should ignore labels with the same name.
title: "Honor Labels",
honorLabels: true,
scrapeLabels: `metric{n1="1" n2="2"} 0`,
scrapeLabels: `metric{n1="1", n2="2"} 0`,
discoveryLabels: []string{"n1", "0"},
expLset: labels.FromStrings("__name__", "metric", "n1", "1", "n2", "2"),
expValue: 0,
@ -1362,7 +1342,7 @@ func TestScrapeLoopAppend(t *testing.T) {
scrapeLabels: `metric NaN`,
discoveryLabels: nil,
expLset: labels.FromStrings("__name__", "metric"),
expValue: float64(value.NormalNaN),
expValue: math.Float64frombits(value.NormalNaN),
},
}
@ -1396,18 +1376,17 @@ func TestScrapeLoopAppend(t *testing.T) {
},
}
// When the expected value is NaN
// DeepEqual will report NaNs as being different,
// so replace it with the expected one.
if test.expValue == float64(value.NormalNaN) {
app.resultFloats[0].f = expected[0].f
}
t.Logf("Test:%s", test.title)
require.Equal(t, expected, app.resultFloats)
requireEqual(t, expected, app.resultFloats)
}
}
func requireEqual(t *testing.T, expected, actual interface{}, msgAndArgs ...interface{}) {
testutil.RequireEqualWithOptions(t, expected, actual,
[]cmp.Option{cmp.Comparer(equalFloatSamples), cmp.AllowUnexported(histogramSample{})},
msgAndArgs...)
}
func TestScrapeLoopAppendForConflictingPrefixedLabels(t *testing.T) {
testcases := map[string]struct {
targetLabels []string
@ -1427,7 +1406,7 @@ func TestScrapeLoopAppendForConflictingPrefixedLabels(t *testing.T) {
},
"One target label collides with existing label, plus existing label already with prefix 'exported": {
targetLabels: []string{"foo", "3"},
exposedLabels: `metric{foo="1" exported_foo="2"} 0`,
exposedLabels: `metric{foo="1", exported_foo="2"} 0`,
expected: []string{"__name__", "metric", "exported_exported_foo", "1", "exported_foo", "2", "foo", "3"},
},
"One target label collides with existing label, both already with prefix 'exported'": {
@ -1437,7 +1416,7 @@ func TestScrapeLoopAppendForConflictingPrefixedLabels(t *testing.T) {
},
"Two target labels collide with existing labels, both with and without prefix 'exported'": {
targetLabels: []string{"foo", "3", "exported_foo", "4"},
exposedLabels: `metric{foo="1" exported_foo="2"} 0`,
exposedLabels: `metric{foo="1", exported_foo="2"} 0`,
expected: []string{
"__name__", "metric", "exported_exported_foo", "1", "exported_exported_exported_foo",
"2", "exported_foo", "4", "foo", "3",
@ -1445,7 +1424,7 @@ func TestScrapeLoopAppendForConflictingPrefixedLabels(t *testing.T) {
},
"Extreme example": {
targetLabels: []string{"foo", "0", "exported_exported_foo", "1", "exported_exported_exported_foo", "2"},
exposedLabels: `metric{foo="3" exported_foo="4" exported_exported_exported_foo="5"} 0`,
exposedLabels: `metric{foo="3", exported_foo="4", exported_exported_exported_foo="5"} 0`,
expected: []string{
"__name__", "metric",
"exported_exported_exported_exported_exported_foo", "5",
@ -1471,7 +1450,7 @@ func TestScrapeLoopAppendForConflictingPrefixedLabels(t *testing.T) {
require.NoError(t, slApp.Commit())
require.Equal(t, []floatSample{
requireEqual(t, []floatSample{
{
metric: labels.FromStrings(tc.expected...),
t: timestamp.FromTime(time.Date(2000, 1, 1, 1, 0, 0, 0, time.UTC)),
@ -1541,9 +1520,7 @@ func TestScrapeLoopAppendSampleLimit(t *testing.T) {
now := time.Now()
slApp := sl.appender(context.Background())
total, added, seriesAdded, err := sl.append(app, []byte("metric_a 1\nmetric_b 1\nmetric_c 1\n"), "", now)
if !errors.Is(err, errSampleLimit) {
t.Fatalf("Did not see expected sample limit error: %s", err)
}
require.ErrorIs(t, err, errSampleLimit)
require.NoError(t, slApp.Rollback())
require.Equal(t, 3, total)
require.Equal(t, 3, added)
@ -1567,14 +1544,12 @@ func TestScrapeLoopAppendSampleLimit(t *testing.T) {
f: 1,
},
}
require.Equal(t, want, resApp.rolledbackFloats, "Appended samples not as expected:\n%s", appender)
requireEqual(t, want, resApp.rolledbackFloats, "Appended samples not as expected:\n%s", appender)
now = time.Now()
slApp = sl.appender(context.Background())
total, added, seriesAdded, err = sl.append(slApp, []byte("metric_a 1\nmetric_b 1\nmetric_c{deleteme=\"yes\"} 1\nmetric_d 1\nmetric_e 1\nmetric_f 1\nmetric_g 1\nmetric_h{deleteme=\"yes\"} 1\nmetric_i{deleteme=\"yes\"} 1\n"), "", now)
if !errors.Is(err, errSampleLimit) {
t.Fatalf("Did not see expected sample limit error: %s", err)
}
require.ErrorIs(t, err, errSampleLimit)
require.NoError(t, slApp.Rollback())
require.Equal(t, 9, total)
require.Equal(t, 6, added)
@ -1709,7 +1684,6 @@ func TestScrapeLoop_ChangingMetricString(t *testing.T) {
require.NoError(t, err)
require.NoError(t, slApp.Commit())
// DeepEqual will report NaNs as being different, so replace with a different value.
want := []floatSample{
{
metric: labels.FromStrings("__name__", "metric_a", "a", "1", "b", "1"),
@ -1741,11 +1715,6 @@ func TestScrapeLoopAppendStaleness(t *testing.T) {
require.NoError(t, err)
require.NoError(t, slApp.Commit())
ingestedNaN := math.Float64bits(app.resultFloats[1].f)
require.Equal(t, value.StaleNaN, ingestedNaN, "Appended stale sample wasn't as expected")
// DeepEqual will report NaNs as being different, so replace with a different value.
app.resultFloats[1].f = 42
want := []floatSample{
{
metric: labels.FromStrings(model.MetricNameLabel, "metric_a"),
@ -1755,10 +1724,10 @@ func TestScrapeLoopAppendStaleness(t *testing.T) {
{
metric: labels.FromStrings(model.MetricNameLabel, "metric_a"),
t: timestamp.FromTime(now.Add(time.Second)),
f: 42,
f: math.Float64frombits(value.StaleNaN),
},
}
require.Equal(t, want, app.resultFloats, "Appended samples not as expected:\n%s", appender)
requireEqual(t, want, app.resultFloats, "Appended samples not as expected:\n%s", appender)
}
func TestScrapeLoopAppendNoStalenessIfTimestamp(t *testing.T) {
@ -1801,8 +1770,6 @@ func TestScrapeLoopAppendStalenessIfTrackTimestampStaleness(t *testing.T) {
require.NoError(t, err)
require.NoError(t, slApp.Commit())
// DeepEqual will report NaNs as being different, so replace with a different value.
app.resultFloats[1].f = 42
want := []floatSample{
{
metric: labels.FromStrings(model.MetricNameLabel, "metric_a"),
@ -1812,10 +1779,10 @@ func TestScrapeLoopAppendStalenessIfTrackTimestampStaleness(t *testing.T) {
{
metric: labels.FromStrings(model.MetricNameLabel, "metric_a"),
t: timestamp.FromTime(now.Add(time.Second)),
f: 42,
f: math.Float64frombits(value.StaleNaN),
},
}
require.Equal(t, want, app.resultFloats, "Appended samples not as expected:\n%s", appender)
requireEqual(t, want, app.resultFloats, "Appended samples not as expected:\n%s", appender)
}
func TestScrapeLoopAppendExemplar(t *testing.T) {
@ -2183,9 +2150,9 @@ metric: <
_, _, _, err := sl.append(app, buf.Bytes(), test.contentType, now)
require.NoError(t, err)
require.NoError(t, app.Commit())
require.Equal(t, test.floats, app.resultFloats)
require.Equal(t, test.histograms, app.resultHistograms)
require.Equal(t, test.exemplars, app.resultExemplars)
requireEqual(t, test.floats, app.resultFloats)
requireEqual(t, test.histograms, app.resultHistograms)
requireEqual(t, test.exemplars, app.resultExemplars)
})
}
}
@ -2240,8 +2207,8 @@ func TestScrapeLoopAppendExemplarSeries(t *testing.T) {
require.NoError(t, app.Commit())
}
require.Equal(t, samples, app.resultFloats)
require.Equal(t, exemplars, app.resultExemplars)
requireEqual(t, samples, app.resultFloats)
requireEqual(t, exemplars, app.resultExemplars)
}
func TestScrapeLoopRunReportsTargetDownOnScrapeError(t *testing.T) {
@ -2317,7 +2284,7 @@ func TestScrapeLoopAppendGracefullyIfAmendOrOutOfOrderOrOutOfBounds(t *testing.T
f: 1,
},
}
require.Equal(t, want, app.resultFloats, "Appended samples not as expected:\n%s", appender)
requireEqual(t, want, app.resultFloats, "Appended samples not as expected:\n%s", appender)
require.Equal(t, 4, total)
require.Equal(t, 4, added)
require.Equal(t, 1, seriesAdded)
@ -2357,15 +2324,12 @@ func TestTargetScraperScrapeOK(t *testing.T) {
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if protobufParsing {
accept := r.Header.Get("Accept")
if !strings.HasPrefix(accept, "application/vnd.google.protobuf;") {
t.Errorf("Expected Accept header to prefer application/vnd.google.protobuf, got %q", accept)
}
require.True(t, strings.HasPrefix(accept, "application/vnd.google.protobuf;"),
"Expected Accept header to prefer application/vnd.google.protobuf.")
}
timeout := r.Header.Get("X-Prometheus-Scrape-Timeout-Seconds")
if timeout != expectedTimeout {
t.Errorf("Expected scrape timeout header %q, got %q", expectedTimeout, timeout)
}
require.Equal(t, expectedTimeout, timeout, "Expected scrape timeout header.")
w.Header().Set("Content-Type", `text/plain; version=0.0.4`)
w.Write([]byte("metric_a 1\nmetric_b 2\n"))
@ -2453,7 +2417,7 @@ func TestTargetScrapeScrapeCancel(t *testing.T) {
select {
case <-time.After(5 * time.Second):
t.Fatalf("Scrape function did not return unexpectedly")
require.FailNow(t, "Scrape function did not return unexpectedly.")
case err := <-errc:
require.NoError(t, err)
}
@ -3053,7 +3017,7 @@ func TestScrapeReportSingleAppender(t *testing.T) {
select {
case <-signal:
case <-time.After(5 * time.Second):
t.Fatalf("Scrape wasn't stopped.")
require.FailNow(t, "Scrape wasn't stopped.")
}
}

102
scrape/target_test.go
View file

@ -77,9 +77,7 @@ func TestTargetOffset(t *testing.T) {
buckets := make([]int, interval/bucketSize)
for _, offset := range offsets {
if offset < 0 || offset >= interval {
t.Fatalf("Offset %v out of bounds", offset)
}
require.InDelta(t, time.Duration(0), offset, float64(interval), "Offset %v out of bounds.", offset)
bucket := offset / bucketSize
buckets[bucket]++
@ -98,9 +96,7 @@ func TestTargetOffset(t *testing.T) {
diff = -diff
}
if float64(diff)/float64(avg) > tolerance {
t.Fatalf("Bucket out of tolerance bounds")
}
require.LessOrEqual(t, float64(diff)/float64(avg), tolerance, "Bucket out of tolerance bounds.")
}
}
@ -150,9 +146,7 @@ func TestNewHTTPBearerToken(t *testing.T) {
func(w http.ResponseWriter, r *http.Request) {
expected := "Bearer 1234"
received := r.Header.Get("Authorization")
if expected != received {
t.Fatalf("Authorization header was not set correctly: expected '%v', got '%v'", expected, received)
}
require.Equal(t, expected, received, "Authorization header was not set correctly.")
},
),
)
@ -162,13 +156,9 @@ func TestNewHTTPBearerToken(t *testing.T) {
BearerToken: "1234",
}
c, err := config_util.NewClientFromConfig(cfg, "test")
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
_, err = c.Get(server.URL)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
}
func TestNewHTTPBearerTokenFile(t *testing.T) {
@ -177,9 +167,7 @@ func TestNewHTTPBearerTokenFile(t *testing.T) {
func(w http.ResponseWriter, r *http.Request) {
expected := "Bearer 12345"
received := r.Header.Get("Authorization")
if expected != received {
t.Fatalf("Authorization header was not set correctly: expected '%v', got '%v'", expected, received)
}
require.Equal(t, expected, received, "Authorization header was not set correctly.")
},
),
)
@ -189,13 +177,9 @@ func TestNewHTTPBearerTokenFile(t *testing.T) {
BearerTokenFile: "testdata/bearertoken.txt",
}
c, err := config_util.NewClientFromConfig(cfg, "test")
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
_, err = c.Get(server.URL)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
}
func TestNewHTTPBasicAuth(t *testing.T) {
@ -203,9 +187,9 @@ func TestNewHTTPBasicAuth(t *testing.T) {
http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
username, password, ok := r.BasicAuth()
if !(ok && username == "user" && password == "password123") {
t.Fatalf("Basic authorization header was not set correctly: expected '%v:%v', got '%v:%v'", "user", "password123", username, password)
}
require.True(t, ok, "Basic authorization header was not set correctly.")
require.Equal(t, "user", username)
require.Equal(t, "password123", password)
},
),
)
@ -218,13 +202,9 @@ func TestNewHTTPBasicAuth(t *testing.T) {
},
}
c, err := config_util.NewClientFromConfig(cfg, "test")
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
_, err = c.Get(server.URL)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
}
func TestNewHTTPCACert(t *testing.T) {
@ -246,13 +226,9 @@ func TestNewHTTPCACert(t *testing.T) {
},
}
c, err := config_util.NewClientFromConfig(cfg, "test")
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
_, err = c.Get(server.URL)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
}
func TestNewHTTPClientCert(t *testing.T) {
@ -279,13 +255,9 @@ func TestNewHTTPClientCert(t *testing.T) {
},
}
c, err := config_util.NewClientFromConfig(cfg, "test")
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
_, err = c.Get(server.URL)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
}
func TestNewHTTPWithServerName(t *testing.T) {
@ -308,13 +280,9 @@ func TestNewHTTPWithServerName(t *testing.T) {
},
}
c, err := config_util.NewClientFromConfig(cfg, "test")
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
_, err = c.Get(server.URL)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
}
func TestNewHTTPWithBadServerName(t *testing.T) {
@ -337,31 +305,23 @@ func TestNewHTTPWithBadServerName(t *testing.T) {
},
}
c, err := config_util.NewClientFromConfig(cfg, "test")
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
_, err = c.Get(server.URL)
if err == nil {
t.Fatal("Expected error, got nil.")
}
require.Error(t, err)
}
func newTLSConfig(certName string, t *testing.T) *tls.Config {
tlsConfig := &tls.Config{}
caCertPool := x509.NewCertPool()
caCert, err := os.ReadFile(caCertPath)
if err != nil {
t.Fatalf("Couldn't set up TLS server: %v", err)
}
require.NoError(t, err, "Couldn't read CA cert.")
caCertPool.AppendCertsFromPEM(caCert)
tlsConfig.RootCAs = caCertPool
tlsConfig.ServerName = "127.0.0.1"
certPath := fmt.Sprintf("testdata/%s.cer", certName)
keyPath := fmt.Sprintf("testdata/%s.key", certName)
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
if err != nil {
t.Errorf("Unable to use specified server cert (%s) & key (%v): %s", certPath, keyPath, err)
}
require.NoError(t, err, "Unable to use specified server cert (%s) & key (%v).", certPath, keyPath)
tlsConfig.Certificates = []tls.Certificate{cert}
return tlsConfig
}
@ -375,9 +335,7 @@ func TestNewClientWithBadTLSConfig(t *testing.T) {
},
}
_, err := config_util.NewClientFromConfig(cfg, "test")
if err == nil {
t.Fatalf("Expected error, got nil.")
}
require.Error(t, err)
}
func TestTargetsFromGroup(t *testing.T) {
@ -389,15 +347,9 @@ func TestTargetsFromGroup(t *testing.T) {
}
lb := labels.NewBuilder(labels.EmptyLabels())
targets, failures := TargetsFromGroup(&targetgroup.Group{Targets: []model.LabelSet{{}, {model.AddressLabel: "localhost:9090"}}}, &cfg, false, nil, lb)
if len(targets) != 1 {
t.Fatalf("Expected 1 target, got %v", len(targets))
}
if len(failures) != 1 {
t.Fatalf("Expected 1 failure, got %v", len(failures))
}
if failures[0].Error() != expectedError {
t.Fatalf("Expected error %s, got %s", expectedError, failures[0])
}
require.Len(t, targets, 1)
require.Len(t, failures, 1)
require.EqualError(t, failures[0], expectedError)
}
func BenchmarkTargetsFromGroup(b *testing.B) {

10
scripts/genproto.sh
View file

@ -46,11 +46,11 @@ for dir in ${DIRS}; do
protoc --gogofast_out=Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types,paths=source_relative:. -I=. \
-I="${GOGOPROTO_PATH}" \
./io/prometheus/client/*.proto
sed -i.bak -E 's/import _ \"github.com\/gogo\/protobuf\/gogoproto\"//g' -- *.pb.go
sed -i.bak -E 's/import _ \"google\/protobuf\"//g' -- *.pb.go
sed -i.bak -E 's/\t_ \"google\/protobuf\"//g' -- *.pb.go
sed -i.bak -E 's/golang\/protobuf\/descriptor/gogo\/protobuf\/protoc-gen-gogo\/descriptor/g' -- *.go
sed -i.bak -E 's/golang\/protobuf/gogo\/protobuf/g' -- *.go
sed -i.bak -E 's/import _ \"github.com\/gogo\/protobuf\/gogoproto\"//g' *.pb.go
sed -i.bak -E 's/import _ \"google\/protobuf\"//g' *.pb.go
sed -i.bak -E 's/\t_ \"google\/protobuf\"//g' *.pb.go
sed -i.bak -E 's/golang\/protobuf\/descriptor/gogo\/protobuf\/protoc-gen-gogo\/descriptor/g' *.go
sed -i.bak -E 's/golang\/protobuf/gogo\/protobuf/g' *.go
rm -f -- *.bak
goimports -w ./*.go ./io/prometheus/client/*.go
popd

12
storage/remote/client.go
View file

@ -146,24 +146,24 @@ func NewWriteClient(name string, conf *ClientConfig) (WriteClient, error) {
}
t := httpClient.Transport
if len(conf.Headers) > 0 {
t = newInjectHeadersRoundTripper(conf.Headers, t)
}
if conf.SigV4Config != nil {
t, err = sigv4.NewSigV4RoundTripper(conf.SigV4Config, httpClient.Transport)
t, err = sigv4.NewSigV4RoundTripper(conf.SigV4Config, t)
if err != nil {
return nil, err
}
}
if conf.AzureADConfig != nil {
t, err = azuread.NewAzureADRoundTripper(conf.AzureADConfig, httpClient.Transport)
t, err = azuread.NewAzureADRoundTripper(conf.AzureADConfig, t)
if err != nil {
return nil, err
}
}
if len(conf.Headers) > 0 {
t = newInjectHeadersRoundTripper(conf.Headers, t)
}
httpClient.Transport = otelhttp.NewTransport(t)
return &Client{

1
storage/remote/otlptranslator/prometheus/normalize_label.go
View file

@ -27,6 +27,7 @@ var dropSanitizationGate = featuregate.GlobalRegistry().MustRegister(
//
// Exception is made for double-underscores which are allowed
func NormalizeLabel(label string) string {
// Trivial case
if len(label) == 0 {
return label

40
storage/remote/queue_manager.go
View file

@ -422,10 +422,11 @@ type QueueManager struct {
clientMtx sync.RWMutex
storeClient WriteClient
seriesMtx sync.Mutex // Covers seriesLabels, seriesMetadata and droppedSeries.
seriesMtx sync.Mutex // Covers seriesLabels, seriesMetadata, droppedSeries and builder.
seriesLabels map[chunks.HeadSeriesRef]labels.Labels
seriesMetadata map[chunks.HeadSeriesRef]*metadata.Metadata
droppedSeries map[chunks.HeadSeriesRef]struct{}
builder *labels.Builder
seriesSegmentMtx sync.Mutex // Covers seriesSegmentIndexes - if you also lock seriesMtx, take seriesMtx first.
seriesSegmentIndexes map[chunks.HeadSeriesRef]int
@ -497,6 +498,7 @@ func NewQueueManager(
seriesMetadata: make(map[chunks.HeadSeriesRef]*metadata.Metadata),
seriesSegmentIndexes: make(map[chunks.HeadSeriesRef]int),
droppedSeries: make(map[chunks.HeadSeriesRef]struct{}),
builder: labels.NewBuilder(labels.EmptyLabels()),
numShards: cfg.MinShards,
reshardChan: make(chan int),
@ -971,12 +973,14 @@ func (t *QueueManager) StoreSeries(series []record.RefSeries, index int) {
// Just make sure all the Refs of Series will insert into seriesSegmentIndexes map for tracking.
t.seriesSegmentIndexes[s.Ref] = index
ls := processExternalLabels(s.Labels, t.externalLabels)
lbls, keep := relabel.Process(ls, t.relabelConfigs...)
if !keep || lbls.IsEmpty() {
t.builder.Reset(s.Labels)
processExternalLabels(t.builder, t.externalLabels)
keep := relabel.ProcessBuilder(t.builder, t.relabelConfigs...)
if !keep {
t.droppedSeries[s.Ref] = struct{}{}
continue
}
lbls := t.builder.Labels()
t.internLabels(lbls)
// We should not ever be replacing a series labels in the map, but just
@ -1059,30 +1063,14 @@ func (t *QueueManager) releaseLabels(ls labels.Labels) {
ls.ReleaseStrings(t.interner.release)
}
// processExternalLabels merges externalLabels into ls. If ls contains
// a label in externalLabels, the value in ls wins.
func processExternalLabels(ls labels.Labels, externalLabels []labels.Label) labels.Labels {
if len(externalLabels) == 0 {
return ls
}
b := labels.NewScratchBuilder(ls.Len() + len(externalLabels))
j := 0
ls.Range(func(l labels.Label) {
for j < len(externalLabels) && l.Name > externalLabels[j].Name {
b.Add(externalLabels[j].Name, externalLabels[j].Value)
j++
// processExternalLabels merges externalLabels into b. If b contains
// a label in externalLabels, the value in b wins.
func processExternalLabels(b *labels.Builder, externalLabels []labels.Label) {
for _, el := range externalLabels {
if b.Get(el.Name) == "" {
b.Set(el.Name, el.Value)
}
if j < len(externalLabels) && l.Name == externalLabels[j].Name {
j++
}
b.Add(l.Name, l.Value)
})
for ; j < len(externalLabels); j++ {
b.Add(externalLabels[j].Name, externalLabels[j].Value)
}
return b.Labels()
}
func (t *QueueManager) updateShardsLoop() {

100
storage/remote/queue_manager_test.go
View file

@ -39,6 +39,7 @@ import (
"github.com/prometheus/prometheus/config"
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/model/relabel"
"github.com/prometheus/prometheus/model/timestamp"
"github.com/prometheus/prometheus/prompb"
writev2 "github.com/prometheus/prometheus/prompb/write/v2"
@ -46,6 +47,7 @@ import (
"github.com/prometheus/prometheus/tsdb/chunks"
"github.com/prometheus/prometheus/tsdb/record"
"github.com/prometheus/prometheus/util/runutil"
"github.com/prometheus/prometheus/util/testutil"
)
const defaultFlushDeadline = 1 * time.Minute
@ -843,7 +845,7 @@ func createExemplars(numExemplars, numSeries int) ([]record.RefExemplar, []recor
Ref: chunks.HeadSeriesRef(i),
T: int64(j),
V: float64(i),
Labels: labels.FromStrings("traceID", fmt.Sprintf("trace-%d", i)),
Labels: labels.FromStrings("trace_id", fmt.Sprintf("trace-%d", i)),
}
exemplars = append(exemplars, e)
}
@ -1186,29 +1188,30 @@ func (c *NopWriteClient) Store(context.Context, []byte, int) error { return nil
func (c *NopWriteClient) Name() string { return "nopwriteclient" }
func (c *NopWriteClient) Endpoint() string { return "http://test-remote.com/1234" }
// Extra labels to make a more realistic workload - taken from Kubernetes' embedded cAdvisor metrics.
var extraLabels []labels.Label = []labels.Label{
{Name: "kubernetes_io_arch", Value: "amd64"},
{Name: "kubernetes_io_instance_type", Value: "c3.somesize"},
{Name: "kubernetes_io_os", Value: "linux"},
{Name: "container_name", Value: "some-name"},
{Name: "failure_domain_kubernetes_io_region", Value: "somewhere-1"},
{Name: "failure_domain_kubernetes_io_zone", Value: "somewhere-1b"},
{Name: "id", Value: "/kubepods/burstable/pod6e91c467-e4c5-11e7-ace3-0a97ed59c75e/a3c8498918bd6866349fed5a6f8c643b77c91836427fb6327913276ebc6bde28"},
{Name: "image", Value: "registry/organisation/name@sha256:dca3d877a80008b45d71d7edc4fd2e44c0c8c8e7102ba5cbabec63a374d1d506"},
{Name: "instance", Value: "ip-111-11-1-11.ec2.internal"},
{Name: "job", Value: "kubernetes-cadvisor"},
{Name: "kubernetes_io_hostname", Value: "ip-111-11-1-11"},
{Name: "monitor", Value: "prod"},
{Name: "name", Value: "k8s_some-name_some-other-name-5j8s8_kube-system_6e91c467-e4c5-11e7-ace3-0a97ed59c75e_0"},
{Name: "namespace", Value: "kube-system"},
{Name: "pod_name", Value: "some-other-name-5j8s8"},
}
func BenchmarkSampleSend(b *testing.B) {
// Send one sample per series, which is the typical remote_write case
const numSamples = 1
const numSeries = 10000
// Extra labels to make a more realistic workload - taken from Kubernetes' embedded cAdvisor metrics.
extraLabels := []labels.Label{
{Name: "kubernetes_io_arch", Value: "amd64"},
{Name: "kubernetes_io_instance_type", Value: "c3.somesize"},
{Name: "kubernetes_io_os", Value: "linux"},
{Name: "container_name", Value: "some-name"},
{Name: "failure_domain_kubernetes_io_region", Value: "somewhere-1"},
{Name: "failure_domain_kubernetes_io_zone", Value: "somewhere-1b"},
{Name: "id", Value: "/kubepods/burstable/pod6e91c467-e4c5-11e7-ace3-0a97ed59c75e/a3c8498918bd6866349fed5a6f8c643b77c91836427fb6327913276ebc6bde28"},
{Name: "image", Value: "registry/organisation/name@sha256:dca3d877a80008b45d71d7edc4fd2e44c0c8c8e7102ba5cbabec63a374d1d506"},
{Name: "instance", Value: "ip-111-11-1-11.ec2.internal"},
{Name: "job", Value: "kubernetes-cadvisor"},
{Name: "kubernetes_io_hostname", Value: "ip-111-11-1-11"},
{Name: "monitor", Value: "prod"},
{Name: "name", Value: "k8s_some-name_some-other-name-5j8s8_kube-system_6e91c467-e4c5-11e7-ace3-0a97ed59c75e_0"},
{Name: "namespace", Value: "kube-system"},
{Name: "pod_name", Value: "some-other-name-5j8s8"},
}
samples, series := createTimeseries(numSamples, numSeries, extraLabels...)
c := NewNopWriteClient()
@ -1240,6 +1243,58 @@ func BenchmarkSampleSend(b *testing.B) {
b.StopTimer()
}
// Check how long it takes to add N series, including external labels processing.
func BenchmarkStoreSeries(b *testing.B) {
externalLabels := []labels.Label{
{Name: "cluster", Value: "mycluster"},
{Name: "replica", Value: "1"},
}
relabelConfigs := []*relabel.Config{{
SourceLabels: model.LabelNames{"namespace"},
Separator: ";",
Regex: relabel.MustNewRegexp("kube.*"),
TargetLabel: "job",
Replacement: "$1",
Action: relabel.Replace,
}}
testCases := []struct {
name string
externalLabels []labels.Label
ts []prompb.TimeSeries
relabelConfigs []*relabel.Config
}{
{name: "plain"},
{name: "externalLabels", externalLabels: externalLabels},
{name: "relabel", relabelConfigs: relabelConfigs},
{
name: "externalLabels+relabel",
externalLabels: externalLabels,
relabelConfigs: relabelConfigs,
},
}
// numSeries chosen to be big enough that StoreSeries dominates creating a new queue manager.
const numSeries = 1000
_, series := createTimeseries(0, numSeries, extraLabels...)
for _, tc := range testCases {
b.Run(tc.name, func(b *testing.B) {
for i := 0; i < b.N; i++ {
c := NewTestWriteClient(Version1)
dir := b.TempDir()
cfg := config.DefaultQueueConfig
mcfg := config.DefaultMetadataConfig
metrics := newQueueManagerMetrics(nil, "", "")
m := NewQueueManager(metrics, nil, nil, nil, dir, newEWMARate(ewmaWeight, shardUpdateDuration), cfg, mcfg, labels.EmptyLabels(), nil, c, defaultFlushDeadline, newPool(), newHighestTimestampMetric(), nil, false, false, Version1)
m.externalLabels = tc.externalLabels
m.relabelConfigs = tc.relabelConfigs
m.StoreSeries(series, 0)
}
})
}
}
func BenchmarkStartup(b *testing.B) {
dir := os.Getenv("WALDIR")
if dir == "" {
@ -1279,7 +1334,8 @@ func BenchmarkStartup(b *testing.B) {
}
func TestProcessExternalLabels(t *testing.T) {
for _, tc := range []struct {
b := labels.NewBuilder(labels.EmptyLabels())
for i, tc := range []struct {
labels labels.Labels
externalLabels []labels.Label
expected labels.Labels
@ -1340,7 +1396,9 @@ func TestProcessExternalLabels(t *testing.T) {
expected: labels.FromStrings("a", "b", "c", "d", "e", "f"),
},
} {
require.Equal(t, tc.expected, processExternalLabels(tc.labels, tc.externalLabels))
b.Reset(tc.labels)
processExternalLabels(b, tc.externalLabels)
testutil.RequireEqual(t, tc.expected, b.Labels(), "test %d", i)
}
}

3
storage/remote/read_test.go
View file

@ -28,6 +28,7 @@ import (
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/prompb"
"github.com/prometheus/prometheus/util/annotations"
"github.com/prometheus/prometheus/util/testutil"
)
func TestNoDuplicateReadConfigs(t *testing.T) {
@ -485,7 +486,7 @@ func TestSampleAndChunkQueryableClient(t *testing.T) {
got = append(got, ss.At().Labels())
}
require.NoError(t, ss.Err())
require.Equal(t, tc.expectedSeries, got)
testutil.RequireEqual(t, tc.expectedSeries, got)
})
}
}

2
storage/remote/write_handler.go
View file

@ -113,7 +113,7 @@ func (h *writeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent)
}
// checkAppendExemplarError modifies the AppendExamplar's returned error based on the error cause.
// checkAppendExemplarError modifies the AppendExemplar's returned error based on the error cause.
func (h *writeHandler) checkAppendExemplarError(err error, e exemplar.Exemplar, outOfOrderErrs *int) error {
unwrappedErr := errors.Unwrap(err)
if unwrappedErr == nil {

19
storage/remote/write_handler_test.go
View file

@ -26,6 +26,7 @@ import (
"time"
"github.com/go-kit/log"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/model/exemplar"
@ -35,6 +36,7 @@ import (
"github.com/prometheus/prometheus/prompb"
"github.com/prometheus/prometheus/storage"
"github.com/prometheus/prometheus/tsdb"
"github.com/prometheus/prometheus/util/testutil"
)
func TestRemoteWriteHandler(t *testing.T) {
@ -58,25 +60,25 @@ func TestRemoteWriteHandler(t *testing.T) {
j := 0
k := 0
for _, ts := range writeRequestFixture.Timeseries {
ls := labelProtosToLabels(ts.Labels)
labels := labelProtosToLabels(ts.Labels)
for _, s := range ts.Samples {
require.Equal(t, mockSample{ls, s.Timestamp, s.Value}, appendable.samples[i])
requireEqual(t, mockSample{labels, s.Timestamp, s.Value}, appendable.samples[i])
i++
}
for _, e := range ts.Exemplars {
exemplarLabels := labelProtosToLabels(e.Labels)
require.Equal(t, mockExemplar{ls, exemplarLabels, e.Timestamp, e.Value}, appendable.exemplars[j])
requireEqual(t, mockExemplar{labels, exemplarLabels, e.Timestamp, e.Value}, appendable.exemplars[j])
j++
}
for _, hp := range ts.Histograms {
if hp.IsFloatHistogram() {
fh := FloatHistogramProtoToFloatHistogram(hp)
require.Equal(t, mockHistogram{ls, hp.Timestamp, nil, fh}, appendable.histograms[k])
requireEqual(t, mockHistogram{labels, hp.Timestamp, nil, fh}, appendable.histograms[k])
} else {
h := HistogramProtoToHistogram(hp)
require.Equal(t, mockHistogram{ls, hp.Timestamp, h, nil}, appendable.histograms[k])
requireEqual(t, mockHistogram{labels, hp.Timestamp, h, nil}, appendable.histograms[k])
}
k++
@ -351,6 +353,13 @@ type mockHistogram struct {
fh *histogram.FloatHistogram
}
// Wrapper to instruct go-cmp package to compare a list of structs with unexported fields.
func requireEqual(t *testing.T, expected, actual interface{}, msgAndArgs ...interface{}) {
testutil.RequireEqualWithOptions(t, expected, actual,
[]cmp.Option{cmp.AllowUnexported(mockSample{}), cmp.AllowUnexported(mockExemplar{}), cmp.AllowUnexported(mockHistogram{})},
msgAndArgs...)
}
func (m *mockAppendable) Appender(_ context.Context) storage.Appender {
return m
}

33
tsdb/db_test.go
View file

@ -27,7 +27,6 @@ import (
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"testing"
"time"
@ -2327,9 +2326,7 @@ func TestBlockRanges(t *testing.T) {
app := db.Appender(ctx)
lbl := labels.FromStrings("a", "b")
_, err = app.Append(0, lbl, firstBlockMaxT-1, rand.Float64())
if err == nil {
t.Fatalf("appending a sample with a timestamp covered by a previous block shouldn't be possible")
}
require.Error(t, err, "appending a sample with a timestamp covered by a previous block shouldn't be possible")
_, err = app.Append(0, lbl, firstBlockMaxT+1, rand.Float64())
require.NoError(t, err)
_, err = app.Append(0, lbl, firstBlockMaxT+2, rand.Float64())
@ -2347,9 +2344,8 @@ func TestBlockRanges(t *testing.T) {
}
require.Len(t, db.Blocks(), 2, "no new block created after the set timeout")
if db.Blocks()[0].Meta().MaxTime > db.Blocks()[1].Meta().MinTime {
t.Fatalf("new block overlaps old:%v,new:%v", db.Blocks()[0].Meta(), db.Blocks()[1].Meta())
}
require.LessOrEqual(t, db.Blocks()[1].Meta().MinTime, db.Blocks()[0].Meta().MaxTime,
"new block overlaps old:%v,new:%v", db.Blocks()[0].Meta(), db.Blocks()[1].Meta())
// Test that wal records are skipped when an existing block covers the same time ranges
// and compaction doesn't create an overlapping block.
@ -2389,9 +2385,8 @@ func TestBlockRanges(t *testing.T) {
require.Len(t, db.Blocks(), 4, "no new block created after the set timeout")
if db.Blocks()[2].Meta().MaxTime > db.Blocks()[3].Meta().MinTime {
t.Fatalf("new block overlaps old:%v,new:%v", db.Blocks()[2].Meta(), db.Blocks()[3].Meta())
}
require.LessOrEqual(t, db.Blocks()[3].Meta().MinTime, db.Blocks()[2].Meta().MaxTime,
"new block overlaps old:%v,new:%v", db.Blocks()[2].Meta(), db.Blocks()[3].Meta())
}
// TestDBReadOnly ensures that opening a DB in readonly mode doesn't modify any files on the disk.
@ -3180,9 +3175,8 @@ func TestOpen_VariousBlockStates(t *testing.T) {
var loaded int
for _, l := range loadedBlocks {
if _, ok := expectedLoadedDirs[filepath.Join(tmpDir, l.meta.ULID.String())]; !ok {
t.Fatal("unexpected block", l.meta.ULID, "was loaded")
}
_, ok := expectedLoadedDirs[filepath.Join(tmpDir, l.meta.ULID.String())]
require.True(t, ok, "unexpected block", l.meta.ULID, "was loaded")
loaded++
}
require.Len(t, expectedLoadedDirs, loaded)
@ -3193,9 +3187,8 @@ func TestOpen_VariousBlockStates(t *testing.T) {
var ignored int
for _, f := range files {
if _, ok := expectedRemovedDirs[filepath.Join(tmpDir, f.Name())]; ok {
t.Fatal("expected", filepath.Join(tmpDir, f.Name()), "to be removed, but still exists")
}
_, ok := expectedRemovedDirs[filepath.Join(tmpDir, f.Name())]
require.False(t, ok, "expected", filepath.Join(tmpDir, f.Name()), "to be removed, but still exists")
if _, ok := expectedIgnoredDirs[filepath.Join(tmpDir, f.Name())]; ok {
ignored++
}
@ -3486,8 +3479,8 @@ func testQuerierShouldNotPanicIfHeadChunkIsTruncatedWhileReadingQueriedChunks(t
// the "cannot populate chunk XXX: not found" error occurred. This error can occur
// when the iterator tries to fetch an head chunk which has been offloaded because
// of the head compaction in the meanwhile.
if firstErr != nil && !strings.Contains(firstErr.Error(), "cannot populate chunk") {
t.Fatalf("unexpected error: %s", firstErr.Error())
if firstErr != nil {
require.ErrorContains(t, firstErr, "cannot populate chunk")
}
}
@ -4065,11 +4058,11 @@ func TestOOOWALWrite(t *testing.T) {
// The normal WAL.
actRecs := getRecords(path.Join(dir, "wal"))
require.Equal(t, inOrderRecords, actRecs)
testutil.RequireEqual(t, inOrderRecords, actRecs)
// The WBL.
actRecs = getRecords(path.Join(dir, wlog.WblDirName))
require.Equal(t, oooRecords, actRecs)
testutil.RequireEqual(t, oooRecords, actRecs)
}
// Tests https://github.com/prometheus/prometheus/issues/10291#issuecomment-1044373110.

4
tsdb/docs/format/index.md
View file

@ -82,6 +82,10 @@ Each series section is aligned to 16 bytes. The ID for a series is the `offset/1
Every series entry first holds its number of labels, followed by tuples of symbol table references that contain the label name and value. The label pairs are lexicographically sorted.
After the labels, the number of indexed chunks is encoded, followed by a sequence of metadata entries containing the chunks minimum (`mint`) and maximum (`maxt`) timestamp and a reference to its position in the chunk file. The `mint` is the time of the first sample and `maxt` is the time of the last sample in the chunk. Holding the time range data in the index allows dropping chunks irrelevant to queried time ranges without accessing them directly.
Chunk references within single series must be increasing, and chunk references for `series_(N+1)` must be higher than chunk references for `series_N`.
This property guarantees that chunks that belong to the same series are grouped together in the segment files.
Furthermore chunk `mint` must be less or equal than `maxt`, and subsequent chunks within single series must have increasing `mint` and `maxt` and not overlap.
`mint` of the first chunk is stored, it's `maxt` is stored as a delta and the `mint` and `maxt` are encoded as deltas to the previous time for subsequent chunks. Similarly, the reference of the first chunk is stored and the next ref is stored as a delta to the previous one.
```

22
tsdb/exemplar_test.go
View file

@ -40,7 +40,7 @@ func TestValidateExemplar(t *testing.T) {
l := labels.FromStrings("service", "asdf")
e := exemplar.Exemplar{
Labels: labels.FromStrings("traceID", "qwerty"),
Labels: labels.FromStrings("trace_id", "qwerty"),
Value: 0.1,
Ts: 1,
}
@ -49,7 +49,7 @@ func TestValidateExemplar(t *testing.T) {
require.NoError(t, es.AddExemplar(l, e))
e2 := exemplar.Exemplar{
Labels: labels.FromStrings("traceID", "zxcvb"),
Labels: labels.FromStrings("trace_id", "zxcvb"),
Value: 0.1,
Ts: 2,
}
@ -82,7 +82,7 @@ func TestAddExemplar(t *testing.T) {
l := labels.FromStrings("service", "asdf")
e := exemplar.Exemplar{
Labels: labels.FromStrings("traceID", "qwerty"),
Labels: labels.FromStrings("trace_id", "qwerty"),
Value: 0.1,
Ts: 1,
}
@ -91,7 +91,7 @@ func TestAddExemplar(t *testing.T) {
require.Equal(t, 0, es.index[string(l.Bytes(nil))].newest, "exemplar was not stored correctly")
e2 := exemplar.Exemplar{
Labels: labels.FromStrings("traceID", "zxcvb"),
Labels: labels.FromStrings("trace_id", "zxcvb"),
Value: 0.1,
Ts: 2,
}
@ -132,7 +132,7 @@ func TestStorageOverflow(t *testing.T) {
var eList []exemplar.Exemplar
for i := 0; i < len(es.exemplars)+1; i++ {
e := exemplar.Exemplar{
Labels: labels.FromStrings("traceID", "a"),
Labels: labels.FromStrings("trace_id", "a"),
Value: float64(i+1) / 10,
Ts: int64(101 + i),
}
@ -158,7 +158,7 @@ func TestSelectExemplar(t *testing.T) {
lName, lValue := "service", "asdf"
l := labels.FromStrings(lName, lValue)
e := exemplar.Exemplar{
Labels: labels.FromStrings("traceID", "querty"),
Labels: labels.FromStrings("trace_id", "querty"),
Value: 0.1,
Ts: 12,
}
@ -189,7 +189,7 @@ func TestSelectExemplar_MultiSeries(t *testing.T) {
for i := 0; i < len(es.exemplars); i++ {
e1 := exemplar.Exemplar{
Labels: labels.FromStrings("traceID", "a"),
Labels: labels.FromStrings("trace_id", "a"),
Value: float64(i+1) / 10,
Ts: int64(101 + i),
}
@ -197,7 +197,7 @@ func TestSelectExemplar_MultiSeries(t *testing.T) {
require.NoError(t, err)
e2 := exemplar.Exemplar{
Labels: labels.FromStrings("traceID", "b"),
Labels: labels.FromStrings("trace_id", "b"),
Value: float64(i+1) / 10,
Ts: int64(101 + i),
}
@ -231,7 +231,7 @@ func TestSelectExemplar_TimeRange(t *testing.T) {
for i := 0; int64(i) < lenEs; i++ {
err := es.AddExemplar(l, exemplar.Exemplar{
Labels: labels.FromStrings("traceID", strconv.Itoa(i)),
Labels: labels.FromStrings("trace_id", strconv.Itoa(i)),
Value: 0.1,
Ts: int64(101 + i),
})
@ -255,7 +255,7 @@ func TestSelectExemplar_DuplicateSeries(t *testing.T) {
es := exs.(*CircularExemplarStorage)
e := exemplar.Exemplar{
Labels: labels.FromStrings("traceID", "qwerty"),
Labels: labels.FromStrings("trace_id", "qwerty"),
Value: 0.1,
Ts: 12,
}
@ -413,7 +413,7 @@ func TestResize(t *testing.T) {
func BenchmarkAddExemplar(b *testing.B) {
// We need to include these labels since we do length calculation
// before adding.
exLabels := labels.FromStrings("traceID", "89620921")
exLabels := labels.FromStrings("trace_id", "89620921")
for _, n := range []int{10000, 100000, 1000000} {
b.Run(fmt.Sprintf("%d", n), func(b *testing.B) {

55
tsdb/fileutil/flock_test.go
View file

@ -18,6 +18,8 @@ import (
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/util/testutil"
)
@ -27,54 +29,35 @@ func TestLocking(t *testing.T) {
fileName := filepath.Join(dir.Path(), "LOCK")
if _, err := os.Stat(fileName); err == nil {
t.Fatalf("File %q unexpectedly exists.", fileName)
}
_, err := os.Stat(fileName)
require.Error(t, err, "File %q unexpectedly exists.", fileName)
lock, existed, err := Flock(fileName)
if err != nil {
t.Fatalf("Error locking file %q: %s", fileName, err)
}
if existed {
t.Errorf("File %q reported as existing during locking.", fileName)
}
require.NoError(t, err, "Error locking file %q", fileName)
require.False(t, existed, "File %q reported as existing during locking.", fileName)
// File must now exist.
if _, err = os.Stat(fileName); err != nil {
t.Errorf("Could not stat file %q expected to exist: %s", fileName, err)
}
_, err = os.Stat(fileName)
require.NoError(t, err, "Could not stat file %q expected to exist", fileName)
// Try to lock again.
lockedAgain, existed, err := Flock(fileName)
if err == nil {
t.Fatalf("File %q locked twice.", fileName)
}
if lockedAgain != nil {
t.Error("Unsuccessful locking did not return nil.")
}
if !existed {
t.Errorf("Existing file %q not recognized.", fileName)
}
require.Error(t, err, "File %q locked twice.", fileName)
require.Nil(t, lockedAgain, "Unsuccessful locking did not return nil.")
require.True(t, existed, "Existing file %q not recognized.", fileName)
if err := lock.Release(); err != nil {
t.Errorf("Error releasing lock for file %q: %s", fileName, err)
}
err = lock.Release()
require.NoError(t, err, "Error releasing lock for file %q", fileName)
// File must still exist.
if _, err = os.Stat(fileName); err != nil {
t.Errorf("Could not stat file %q expected to exist: %s", fileName, err)
}
_, err = os.Stat(fileName)
require.NoError(t, err, "Could not stat file %q expected to exist", fileName)
// Lock existing file.
lock, existed, err = Flock(fileName)
if err != nil {
t.Fatalf("Error locking file %q: %s", fileName, err)
}
if !existed {
t.Errorf("Existing file %q not recognized.", fileName)
}
require.NoError(t, err, "Error locking file %q", fileName)
require.True(t, existed, "Existing file %q not recognized.", fileName)
if err := lock.Release(); err != nil {
t.Errorf("Error releasing lock for file %q: %s", fileName, err)
}
err = lock.Release()
require.NoError(t, err, "Error releasing lock for file %q", fileName)
}

30
tsdb/head_test.go
View file

@ -30,6 +30,7 @@ import (
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/prometheus/client_golang/prometheus"
prom_testutil "github.com/prometheus/client_golang/prometheus/testutil"
"github.com/prometheus/common/model"
@ -50,6 +51,7 @@ import (
"github.com/prometheus/prometheus/tsdb/tombstones"
"github.com/prometheus/prometheus/tsdb/tsdbutil"
"github.com/prometheus/prometheus/tsdb/wlog"
"github.com/prometheus/prometheus/util/testutil"
)
// newTestHeadDefaultOptions returns the HeadOptions that should be used by default in unit tests.
@ -206,7 +208,7 @@ func readTestWAL(t testing.TB, dir string) (recs []interface{}) {
require.NoError(t, err)
recs = append(recs, exemplars)
default:
t.Fatalf("unknown record type")
require.Fail(t, "unknown record type")
}
}
require.NoError(t, r.Err())
@ -373,7 +375,7 @@ func BenchmarkLoadWLs(b *testing.B) {
Ref: chunks.HeadSeriesRef(k) * 101,
T: int64(i) * 10,
V: float64(i) * 100,
Labels: labels.FromStrings("traceID", fmt.Sprintf("trace-%d", i)),
Labels: labels.FromStrings("trace_id", fmt.Sprintf("trace-%d", i)),
})
}
populateTestWL(b, wal, []interface{}{refExemplars})
@ -658,7 +660,7 @@ func TestHead_ReadWAL(t *testing.T) {
{Ref: 0, Intervals: []tombstones.Interval{{Mint: 99, Maxt: 101}}},
},
[]record.RefExemplar{
{Ref: 10, T: 100, V: 1, Labels: labels.FromStrings("traceID", "asdf")},
{Ref: 10, T: 100, V: 1, Labels: labels.FromStrings("trace_id", "asdf")},
},
}
@ -677,10 +679,10 @@ func TestHead_ReadWAL(t *testing.T) {
s50 := head.series.getByID(50)
s100 := head.series.getByID(100)
require.Equal(t, labels.FromStrings("a", "1"), s10.lset)
require.Equal(t, (*memSeries)(nil), s11) // Series without samples should be garbage collected at head.Init().
require.Equal(t, labels.FromStrings("a", "4"), s50.lset)
require.Equal(t, labels.FromStrings("a", "3"), s100.lset)
testutil.RequireEqual(t, labels.FromStrings("a", "1"), s10.lset)
require.Nil(t, s11) // Series without samples should be garbage collected at head.Init().
testutil.RequireEqual(t, labels.FromStrings("a", "4"), s50.lset)
testutil.RequireEqual(t, labels.FromStrings("a", "3"), s100.lset)
expandChunk := func(c chunkenc.Iterator) (x []sample) {
for c.Next() == chunkenc.ValFloat {
@ -707,7 +709,7 @@ func TestHead_ReadWAL(t *testing.T) {
require.NoError(t, err)
e, err := q.Select(0, 1000, []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "a", "1")})
require.NoError(t, err)
require.Equal(t, exemplar.Exemplar{Ts: 100, Value: 1, Labels: labels.FromStrings("traceID", "asdf")}, e[0].Exemplars[0])
require.True(t, exemplar.Exemplar{Ts: 100, Value: 1, Labels: labels.FromStrings("trace_id", "asdf")}.Equals(e[0].Exemplars[0]))
})
}
}
@ -1371,7 +1373,7 @@ func TestDeletedSamplesAndSeriesStillInWALAfterCheckpoint(t *testing.T) {
case []record.RefMetadata:
metadata++
default:
t.Fatalf("unknown record type")
require.Fail(t, "unknown record type")
}
}
require.Equal(t, 1, series)
@ -1620,9 +1622,7 @@ func TestComputeChunkEndTime(t *testing.T) {
for testName, tc := range cases {
t.Run(testName, func(t *testing.T) {
got := computeChunkEndTime(tc.start, tc.cur, tc.max, tc.ratioToFull)
if got != tc.res {
t.Errorf("expected %d for (start: %d, cur: %d, max: %d, ratioToFull: %f), got %d", tc.res, tc.start, tc.cur, tc.max, tc.ratioToFull, got)
}
require.Equal(t, tc.res, got, "(start: %d, cur: %d, max: %d)", tc.start, tc.cur, tc.max)
})
}
}
@ -3049,7 +3049,7 @@ func TestHeadExemplars(t *testing.T) {
head, _ := newTestHead(t, chunkRange, wlog.CompressionNone, false)
app := head.Appender(context.Background())
l := labels.FromStrings("traceId", "123")
l := labels.FromStrings("trace_id", "123")
// It is perfectly valid to add Exemplars before the current start time -
// histogram buckets that haven't been update in a while could still be
// exported exemplars from an hour ago.
@ -3694,7 +3694,7 @@ func TestChunkSnapshot(t *testing.T) {
e := ex{
seriesLabels: lbls,
e: exemplar.Exemplar{
Labels: labels.FromStrings("traceID", fmt.Sprintf("%d", rand.Int())),
Labels: labels.FromStrings("trace_id", fmt.Sprintf("%d", rand.Int())),
Value: rand.Float64(),
Ts: ts,
},
@ -3745,7 +3745,7 @@ func TestChunkSnapshot(t *testing.T) {
})
require.NoError(t, err)
// Verifies both existence of right exemplars and order of exemplars in the buffer.
require.Equal(t, expExemplars, actExemplars)
testutil.RequireEqualWithOptions(t, expExemplars, actExemplars, []cmp.Option{cmp.AllowUnexported(ex{})})
}
var (

62
tsdb/index/index.go
View file

@ -146,8 +146,11 @@ type Writer struct {
labelNames map[string]uint64 // Label names, and their usage.
// Hold last series to validate that clients insert new series in order.
lastSeries labels.Labels
lastRef storage.SeriesRef
lastSeries labels.Labels
lastSeriesRef storage.SeriesRef
// Hold last added chunk reference to make sure that chunks are ordered properly.
lastChunkRef chunks.ChunkRef
crc32 hash.Hash
@ -433,9 +436,27 @@ func (w *Writer) AddSeries(ref storage.SeriesRef, lset labels.Labels, chunks ...
return fmt.Errorf("out-of-order series added with label set %q", lset)
}
if ref < w.lastRef && !w.lastSeries.IsEmpty() {
if ref < w.lastSeriesRef && !w.lastSeries.IsEmpty() {
return fmt.Errorf("series with reference greater than %d already added", ref)
}
lastChunkRef := w.lastChunkRef
lastMaxT := int64(0)
for ix, c := range chunks {
if c.Ref < lastChunkRef {
return fmt.Errorf("unsorted chunk reference: %d, previous: %d", c.Ref, lastChunkRef)
}
lastChunkRef = c.Ref
if ix > 0 && c.MinTime <= lastMaxT {
return fmt.Errorf("chunk minT %d is not higher than previous chunk maxT %d", c.MinTime, lastMaxT)
}
if c.MaxTime < c.MinTime {
return fmt.Errorf("chunk maxT %d is less than minT %d", c.MaxTime, c.MinTime)
}
lastMaxT = c.MaxTime
}
// We add padding to 16 bytes to increase the addressable space we get through 4 byte
// series references.
if err := w.addPadding(seriesByteAlign); err != nil {
@ -510,7 +531,8 @@ func (w *Writer) AddSeries(ref storage.SeriesRef, lset labels.Labels, chunks ...
}
w.lastSeries.CopyFrom(lset)
w.lastRef = ref
w.lastSeriesRef = ref
w.lastChunkRef = lastChunkRef
return nil
}
@ -715,17 +737,11 @@ func (w *Writer) writeLabelIndexesOffsetTable() error {
}
// Write out the length.
w.buf1.Reset()
l := w.f.pos - startPos - 4
if l > math.MaxUint32 {
return fmt.Errorf("label indexes offset table size exceeds 4 bytes: %d", l)
err := w.writeLengthAndHash(startPos)
if err != nil {
return fmt.Errorf("label indexes offset table length/crc32 write error: %w", err)
}
w.buf1.PutBE32int(int(l))
if err := w.writeAt(w.buf1.Get(), startPos); err != nil {
return err
}
return w.writeLenghtAndHash(startPos)
return nil
}
// writePostingsOffsetTable writes the postings offset table.
@ -793,25 +809,31 @@ func (w *Writer) writePostingsOffsetTable() error {
}
w.fPO = nil
return w.writeLenghtAndHash(startPos)
err = w.writeLengthAndHash(startPos)
if err != nil {
return fmt.Errorf("postings offset table length/crc32 write error: %w", err)
}
return nil
}
func (w *Writer) writeLenghtAndHash(startPos uint64) error {
// Write out the length.
func (w *Writer) writeLengthAndHash(startPos uint64) error {
w.buf1.Reset()
l := w.f.pos - startPos - 4
if l > math.MaxUint32 {
return fmt.Errorf("postings offset table size exceeds 4 bytes: %d", l)
return fmt.Errorf("length size exceeds 4 bytes: %d", l)
}
w.buf1.PutBE32int(int(l))
if err := w.writeAt(w.buf1.Get(), startPos); err != nil {
return err
return fmt.Errorf("write length from buffer error: %w", err)
}
// Write out the hash.
w.buf1.Reset()
w.buf1.PutHashSum(w.crc32)
return w.write(w.buf1.Get())
if err := w.write(w.buf1.Get()); err != nil {
return fmt.Errorf("write buffer's crc32 error: %w", err)
}
return nil
}
const indexTOCLen = 6*8 + crc32.Size

59
tsdb/index/index_test.go
View file

@ -18,7 +18,6 @@ import (
"errors"
"fmt"
"hash/crc32"
"math/rand"
"os"
"path/filepath"
"sort"
@ -206,7 +205,7 @@ func TestIndexRW_Postings(t *testing.T) {
require.NoError(t, err)
require.Empty(t, c)
require.Equal(t, series[i], builder.Labels())
testutil.RequireEqual(t, series[i], builder.Labels())
}
require.NoError(t, p.Err())
@ -407,15 +406,17 @@ func TestPersistence_index_e2e(t *testing.T) {
var input indexWriterSeriesSlice
ref := uint64(0)
// Generate ChunkMetas for every label set.
for i, lset := range lbls {
var metas []chunks.Meta
for j := 0; j <= (i % 20); j++ {
ref++
metas = append(metas, chunks.Meta{
MinTime: int64(j * 10000),
MaxTime: int64((j + 1) * 10000),
Ref: chunks.ChunkRef(rand.Uint64()),
MaxTime: int64((j+1)*10000) - 1,
Ref: chunks.ChunkRef(ref),
Chunk: chunkenc.NewXORChunk(),
})
}
@ -487,7 +488,7 @@ func TestPersistence_index_e2e(t *testing.T) {
err = mi.Series(expp.At(), &eBuilder, &expchks)
require.NoError(t, err)
require.Equal(t, eBuilder.Labels(), builder.Labels())
testutil.RequireEqual(t, eBuilder.Labels(), builder.Labels())
require.Equal(t, expchks, chks)
}
require.False(t, expp.Next(), "Expected no more postings for %q=%q", p.Name, p.Value)
@ -670,3 +671,51 @@ func TestDecoder_Postings_WrongInput(t *testing.T) {
_, _, err := (&Decoder{}).Postings([]byte("the cake is a lie"))
require.Error(t, err)
}
func TestChunksRefOrdering(t *testing.T) {
dir := t.TempDir()
idx, err := NewWriter(context.Background(), filepath.Join(dir, "index"))
require.NoError(t, err)
require.NoError(t, idx.AddSymbol("1"))
require.NoError(t, idx.AddSymbol("2"))
require.NoError(t, idx.AddSymbol("__name__"))
c50 := chunks.Meta{Ref: 50}
c100 := chunks.Meta{Ref: 100}
c200 := chunks.Meta{Ref: 200}
require.NoError(t, idx.AddSeries(1, labels.FromStrings("__name__", "1"), c100))
require.EqualError(t, idx.AddSeries(2, labels.FromStrings("__name__", "2"), c50), "unsorted chunk reference: 50, previous: 100")
require.NoError(t, idx.AddSeries(2, labels.FromStrings("__name__", "2"), c200))
require.NoError(t, idx.Close())
}
func TestChunksTimeOrdering(t *testing.T) {
dir := t.TempDir()
idx, err := NewWriter(context.Background(), filepath.Join(dir, "index"))
require.NoError(t, err)
require.NoError(t, idx.AddSymbol("1"))
require.NoError(t, idx.AddSymbol("2"))
require.NoError(t, idx.AddSymbol("__name__"))
require.NoError(t, idx.AddSeries(1, labels.FromStrings("__name__", "1"),
chunks.Meta{Ref: 1, MinTime: 0, MaxTime: 10}, // Also checks that first chunk can have MinTime: 0.
chunks.Meta{Ref: 2, MinTime: 11, MaxTime: 20},
chunks.Meta{Ref: 3, MinTime: 21, MaxTime: 30},
))
require.EqualError(t, idx.AddSeries(1, labels.FromStrings("__name__", "2"),
chunks.Meta{Ref: 10, MinTime: 0, MaxTime: 10},
chunks.Meta{Ref: 20, MinTime: 10, MaxTime: 20},
), "chunk minT 10 is not higher than previous chunk maxT 10")
require.EqualError(t, idx.AddSeries(1, labels.FromStrings("__name__", "2"),
chunks.Meta{Ref: 10, MinTime: 100, MaxTime: 30},
), "chunk maxT 30 is less than minT 100")
require.NoError(t, idx.Close())
}

28
tsdb/index/postings_test.go
View file

@ -61,9 +61,7 @@ func TestMemPostings_ensureOrder(t *testing.T) {
ok := sort.SliceIsSorted(l, func(i, j int) bool {
return l[i] < l[j]
})
if !ok {
t.Fatalf("postings list %v is not sorted", l)
}
require.True(t, ok, "postings list %v is not sorted", l)
}
}
}
@ -214,9 +212,7 @@ func TestIntersect(t *testing.T) {
for _, c := range cases {
t.Run("", func(t *testing.T) {
if c.res == nil {
t.Fatal("intersect result expectancy cannot be nil")
}
require.NotNil(t, c.res, "intersect result expectancy cannot be nil")
expected, err := ExpandPostings(c.res)
require.NoError(t, err)
@ -228,9 +224,7 @@ func TestIntersect(t *testing.T) {
return
}
if i == EmptyPostings() {
t.Fatal("intersect unexpected result: EmptyPostings sentinel")
}
require.NotEqual(t, EmptyPostings(), i, "intersect unexpected result: EmptyPostings sentinel")
res, err := ExpandPostings(i)
require.NoError(t, err)
@ -501,9 +495,7 @@ func TestMergedPostings(t *testing.T) {
for _, c := range cases {
t.Run("", func(t *testing.T) {
if c.res == nil {
t.Fatal("merge result expectancy cannot be nil")
}
require.NotNil(t, c.res, "merge result expectancy cannot be nil")
ctx := context.Background()
@ -517,9 +509,7 @@ func TestMergedPostings(t *testing.T) {
return
}
if m == EmptyPostings() {
t.Fatal("merge unexpected result: EmptyPostings sentinel")
}
require.NotEqual(t, EmptyPostings(), m, "merge unexpected result: EmptyPostings sentinel")
res, err := ExpandPostings(m)
require.NoError(t, err)
@ -897,9 +887,7 @@ func TestWithoutPostings(t *testing.T) {
for _, c := range cases {
t.Run("", func(t *testing.T) {
if c.res == nil {
t.Fatal("without result expectancy cannot be nil")
}
require.NotNil(t, c.res, "without result expectancy cannot be nil")
expected, err := ExpandPostings(c.res)
require.NoError(t, err)
@ -911,9 +899,7 @@ func TestWithoutPostings(t *testing.T) {
return
}
if w == EmptyPostings() {
t.Fatal("without unexpected result: EmptyPostings sentinel")
}
require.NotEqual(t, EmptyPostings(), w, "without unexpected result: EmptyPostings sentinel")
res, err := ExpandPostings(w)
require.NoError(t, err)

25
tsdb/querier_test.go
View file

@ -2702,22 +2702,7 @@ func TestFindSetMatches(t *testing.T) {
}
for _, c := range cases {
matches := findSetMatches(c.pattern)
if len(c.exp) == 0 {
if len(matches) != 0 {
t.Errorf("Evaluating %s, unexpected result %v", c.pattern, matches)
}
} else {
if len(matches) != len(c.exp) {
t.Errorf("Evaluating %s, length of result not equal to exp", c.pattern)
} else {
for i := 0; i < len(c.exp); i++ {
if c.exp[i] != matches[i] {
t.Errorf("Evaluating %s, unexpected result %s", c.pattern, matches[i])
}
}
}
}
require.Equal(t, c.exp, findSetMatches(c.pattern), "Evaluating %s, unexpected result.", c.pattern)
}
}
@ -3016,9 +3001,7 @@ func TestPostingsForMatchers(t *testing.T) {
}
}
require.NoError(t, p.Err())
if len(exp) != 0 {
t.Errorf("Evaluating %v, missing results %+v", c.matchers, exp)
}
require.Empty(t, exp, "Evaluating %v", c.matchers)
})
}
}
@ -3101,9 +3084,7 @@ func TestClose(t *testing.T) {
createBlock(t, dir, genSeries(1, 1, 10, 20))
db, err := Open(dir, nil, nil, DefaultOptions(), nil)
if err != nil {
t.Fatalf("Opening test storage failed: %s", err)
}
require.NoError(t, err, "Opening test storage failed: %s")
defer func() {
require.NoError(t, db.Close())
}()

13
tsdb/record/record_test.go
View file

@ -24,6 +24,7 @@ import (
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/tsdb/encoding"
"github.com/prometheus/prometheus/tsdb/tombstones"
"github.com/prometheus/prometheus/util/testutil"
)
func TestRecord_EncodeDecode(t *testing.T) {
@ -44,7 +45,7 @@ func TestRecord_EncodeDecode(t *testing.T) {
}
decSeries, err := dec.Series(enc.Series(series, nil), nil)
require.NoError(t, err)
require.Equal(t, series, decSeries)
testutil.RequireEqual(t, series, decSeries)
metadata := []RefMetadata{
{
@ -101,13 +102,13 @@ func TestRecord_EncodeDecode(t *testing.T) {
}, decTstones)
exemplars := []RefExemplar{
{Ref: 0, T: 12423423, V: 1.2345, Labels: labels.FromStrings("traceID", "qwerty")},
{Ref: 123, T: -1231, V: -123, Labels: labels.FromStrings("traceID", "asdf")},
{Ref: 2, T: 0, V: 99999, Labels: labels.FromStrings("traceID", "zxcv")},
{Ref: 0, T: 12423423, V: 1.2345, Labels: labels.FromStrings("trace_id", "qwerty")},
{Ref: 123, T: -1231, V: -123, Labels: labels.FromStrings("trace_id", "asdf")},
{Ref: 2, T: 0, V: 99999, Labels: labels.FromStrings("trace_id", "zxcv")},
}
decExemplars, err := dec.Exemplars(enc.Exemplars(exemplars, nil), nil)
require.NoError(t, err)
require.Equal(t, exemplars, decExemplars)
testutil.RequireEqual(t, exemplars, decExemplars)
histograms := []RefHistogramSample{
{
@ -226,7 +227,7 @@ func TestRecord_Corrupted(t *testing.T) {
t.Run("Test corrupted exemplar record", func(t *testing.T) {
exemplars := []RefExemplar{
{Ref: 0, T: 12423423, V: 1.2345, Labels: labels.FromStrings("traceID", "asdf")},
{Ref: 0, T: 12423423, V: 1.2345, Labels: labels.FromStrings("trace_id", "asdf")},
}
corrupted := enc.Exemplars(exemplars, nil)[:8]

3
tsdb/repair_test.go
View file

@ -25,6 +25,7 @@ import (
"github.com/prometheus/prometheus/tsdb/chunks"
"github.com/prometheus/prometheus/tsdb/fileutil"
"github.com/prometheus/prometheus/tsdb/index"
"github.com/prometheus/prometheus/util/testutil"
)
func TestRepairBadIndexVersion(t *testing.T) {
@ -112,7 +113,7 @@ func TestRepairBadIndexVersion(t *testing.T) {
}
require.NoError(t, p.Err())
require.Equal(t, []labels.Labels{
testutil.RequireEqual(t, []labels.Labels{
labels.FromStrings("a", "1", "b", "1"),
labels.FromStrings("a", "2", "b", "1"),
}, res)

13
tsdb/wal_test.go
View file

@ -34,6 +34,7 @@ import (
"github.com/prometheus/prometheus/tsdb/record"
"github.com/prometheus/prometheus/tsdb/tombstones"
"github.com/prometheus/prometheus/tsdb/wlog"
"github.com/prometheus/prometheus/util/testutil"
)
func TestSegmentWAL_cut(t *testing.T) {
@ -147,7 +148,7 @@ func TestSegmentWAL_Truncate(t *testing.T) {
readSeries = append(readSeries, s...)
}, nil, nil))
require.Equal(t, expected, readSeries)
testutil.RequireEqual(t, expected, readSeries)
}
// Symmetrical test of reading and writing to the WAL via its main interface.
@ -213,9 +214,9 @@ func TestSegmentWAL_Log_Restore(t *testing.T) {
require.NoError(t, r.Read(serf, smplf, delf))
require.Equal(t, recordedSamples, resultSamples)
require.Equal(t, recordedSeries, resultSeries)
require.Equal(t, recordedDeletes, resultDeletes)
testutil.RequireEqual(t, recordedSamples, resultSamples)
testutil.RequireEqual(t, recordedSeries, resultSeries)
testutil.RequireEqual(t, recordedDeletes, resultDeletes)
series := series[k : k+(numMetrics/iterations)]
@ -528,12 +529,12 @@ func TestMigrateWAL_Fuzz(t *testing.T) {
require.NoError(t, err)
res = append(res, s)
default:
t.Fatalf("unknown record type %d", dec.Type(rec))
require.Fail(t, "unknown record type %d", dec.Type(rec))
}
}
require.NoError(t, r.Err())
require.Equal(t, []interface{}{
testutil.RequireEqual(t, []interface{}{
[]record.RefSeries{
{Ref: 100, Labels: labels.FromStrings("abc", "def", "123", "456")},
{Ref: 1, Labels: labels.FromStrings("abc", "def2", "1234", "4567")},

5
tsdb/wlog/checkpoint_test.go
View file

@ -29,6 +29,7 @@ import (
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/tsdb/chunks"
"github.com/prometheus/prometheus/tsdb/record"
"github.com/prometheus/prometheus/util/testutil"
)
func TestLastCheckpoint(t *testing.T) {
@ -201,7 +202,7 @@ func TestCheckpoint(t *testing.T) {
histogramsInWAL += 4
b = enc.Exemplars([]record.RefExemplar{
{Ref: 1, T: last, V: float64(i), Labels: labels.FromStrings("traceID", fmt.Sprintf("trace-%d", i))},
{Ref: 1, T: last, V: float64(i), Labels: labels.FromStrings("trace_id", fmt.Sprintf("trace-%d", i))},
}, nil)
require.NoError(t, w.Log(b))
@ -286,7 +287,7 @@ func TestCheckpoint(t *testing.T) {
{Ref: 2, Labels: labels.FromStrings("a", "b", "c", "2")},
{Ref: 4, Labels: labels.FromStrings("a", "b", "c", "4")},
}
require.Equal(t, expectedRefSeries, series)
testutil.RequireEqual(t, expectedRefSeries, series)
expectedRefMetadata := []record.RefMetadata{
{Ref: 0, Unit: fmt.Sprintf("%d", last-100), Help: fmt.Sprintf("%d", last-100)},

13
tsdb/wlog/reader_test.go
View file

@ -182,16 +182,13 @@ func TestReader(t *testing.T) {
t.Logf("record %d", j)
rec := r.Record()
if j >= len(c.exp) {
t.Fatal("received more records than expected")
}
require.Less(t, j, len(c.exp), "received more records than expected")
require.Equal(t, c.exp[j], rec, "Bytes within record did not match expected Bytes")
}
if !c.fail && r.Err() != nil {
t.Fatalf("unexpected error: %s", r.Err())
}
if c.fail && r.Err() == nil {
t.Fatalf("expected error but got none")
if !c.fail {
require.NoError(t, r.Err())
} else {
require.Error(t, r.Err())
}
})
}

2
tsdb/wlog/watcher_test.go
View file

@ -171,7 +171,7 @@ func TestTailSamples(t *testing.T) {
Ref: chunks.HeadSeriesRef(inner),
T: now.UnixNano() + 1,
V: float64(i),
Labels: labels.FromStrings("traceID", fmt.Sprintf("trace-%d", inner)),
Labels: labels.FromStrings("trace_id", fmt.Sprintf("trace-%d", inner)),
},
}, nil)
require.NoError(t, w.Log(exemplar))

4
tsdb/wlog/wlog_test.go
View file

@ -192,9 +192,7 @@ func TestWALRepair_ReadingError(t *testing.T) {
require.Len(t, result, test.intactRecs, "Wrong number of intact records")
for i, r := range result {
if !bytes.Equal(records[i], r) {
t.Fatalf("record %d diverges: want %x, got %x", i, records[i][:10], r[:10])
}
require.True(t, bytes.Equal(records[i], r), "record %d diverges: want %x, got %x", i, records[i][:10], r[:10])
}
// Make sure there is a new 0 size Segment after the corrupted Segment.

3
util/runtime/statfs_default.go
View file

@ -71,7 +71,8 @@ func Statfs(path string) string {
var fs syscall.Statfs_t
err := syscall.Statfs(path, &fs)
//nolint:unconvert // This ensure Type format on all Platforms
// nolintlint might cry out depending on the architecture (e.g. ARM64), so ignore it.
//nolint:unconvert,nolintlint // This ensures Type format on all Platforms.
localType := int64(fs.Type)
if err != nil {
return strconv.FormatInt(localType, 16)

43
util/testutil/cmp.go Normal file
View file

@ -0,0 +1,43 @@
// Copyright 2023 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package testutil
import (
"fmt"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/model/labels"
)
// Replacement for require.Equal using go-cmp adapted for Prometheus data structures, instead of DeepEqual.
func RequireEqual(t testing.TB, expected, actual interface{}, msgAndArgs ...interface{}) {
t.Helper()
RequireEqualWithOptions(t, expected, actual, nil, msgAndArgs...)
}
// As RequireEqual but allows extra cmp.Options.
func RequireEqualWithOptions(t testing.TB, expected, actual interface{}, extra []cmp.Option, msgAndArgs ...interface{}) {
t.Helper()
options := append([]cmp.Option{cmp.Comparer(labels.Equal)}, extra...)
if cmp.Equal(expected, actual, options...) {
return
}
diff := cmp.Diff(expected, actual, options...)
require.Fail(t, fmt.Sprintf("Not equal: \n"+
"expected: %s\n"+
"actual : %s%s", expected, actual, diff), msgAndArgs...)
}

134
web/api/v1/api_test.go
View file

@ -32,6 +32,7 @@ import (
"github.com/prometheus/prometheus/prompb"
"github.com/prometheus/prometheus/util/stats"
"github.com/prometheus/prometheus/util/testutil"
"github.com/go-kit/log"
"github.com/prometheus/client_golang/prometheus"
@ -216,18 +217,11 @@ type rulesRetrieverMock struct {
func (m *rulesRetrieverMock) CreateAlertingRules() {
expr1, err := parser.ParseExpr(`absent(test_metric3) != 1`)
if err != nil {
m.testing.Fatalf("unable to parse alert expression: %s", err)
}
require.NoError(m.testing, err)
expr2, err := parser.ParseExpr(`up == 1`)
if err != nil {
m.testing.Fatalf("Unable to parse alert expression: %s", err)
}
require.NoError(m.testing, err)
expr3, err := parser.ParseExpr(`vector(1)`)
if err != nil {
m.testing.Fatalf("Unable to parse alert expression: %s", err)
}
require.NoError(m.testing, err)
rule1 := rules.NewAlertingRule(
"test_metric3",
@ -302,9 +296,7 @@ func (m *rulesRetrieverMock) CreateRuleGroups() {
}
recordingExpr, err := parser.ParseExpr(`vector(1)`)
if err != nil {
m.testing.Fatalf("unable to parse alert expression: %s", err)
}
require.NoError(m.testing, err, "unable to parse alert expression")
recordingRule := rules.NewRecordingRule("recording-rule-1", recordingExpr, labels.Labels{})
r = append(r, recordingRule)
@ -607,7 +599,7 @@ func TestGetSeries(t *testing.T) {
r := res.data.([]labels.Labels)
sort.Sort(byLabels(tc.expected))
sort.Sort(byLabels(r))
require.Equal(t, tc.expected, r)
testutil.RequireEqual(t, tc.expected, r)
}
})
}
@ -715,9 +707,7 @@ func TestQueryExemplars(t *testing.T) {
for _, te := range tc.exemplars {
for _, e := range te.Exemplars {
_, err := es.AppendExemplar(0, te.SeriesLabels, e)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
}
}
@ -2833,9 +2823,7 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
}
req, err := request(method, test.query)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
tr.ResetMetadataStore()
for _, tm := range test.metadata {
@ -2845,9 +2833,7 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
for _, te := range test.exemplars {
for _, e := range te.Exemplars {
_, err := es.AppendExemplar(0, te.SeriesLabels, e)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
}
}
@ -2883,37 +2869,25 @@ func describeAPIFunc(f apiFunc) string {
func assertAPIError(t *testing.T, got *apiError, exp errorType) {
t.Helper()
if got != nil {
if exp == errorNone {
t.Fatalf("Unexpected error: %s", got)
}
if exp != got.typ {
t.Fatalf("Expected error of type %q but got type %q (%q)", exp, got.typ, got)
}
return
}
if exp != errorNone {
t.Fatalf("Expected error of type %q but got none", exp)
if exp == errorNone {
require.Nil(t, got)
} else {
require.NotNil(t, got)
require.Equal(t, exp, got.typ, "(%q)", got)
}
}
func assertAPIResponse(t *testing.T, got, exp interface{}) {
t.Helper()
require.Equal(t, exp, got)
testutil.RequireEqual(t, exp, got)
}
func assertAPIResponseLength(t *testing.T, got interface{}, expLen int) {
t.Helper()
gotLen := reflect.ValueOf(got).Len()
if gotLen != expLen {
t.Fatalf(
"Response length does not match, expected:\n%d\ngot:\n%d",
expLen,
gotLen,
)
}
require.Equal(t, expLen, gotLen, "Response length does not match")
}
func assertAPIResponseMetadataLen(t *testing.T, got interface{}, expLen int) {
@ -2925,13 +2899,7 @@ func assertAPIResponseMetadataLen(t *testing.T, got interface{}, expLen int) {
gotLen += len(m)
}
if gotLen != expLen {
t.Fatalf(
"Amount of metadata in the response does not match, expected:\n%d\ngot:\n%d",
expLen,
gotLen,
)
}
require.Equal(t, expLen, gotLen, "Amount of metadata in the response does not match")
}
type fakeDB struct {
@ -3272,26 +3240,18 @@ func TestRespondError(t *testing.T) {
defer s.Close()
resp, err := http.Get(s.URL)
if err != nil {
t.Fatalf("Error on test request: %s", err)
}
require.NoError(t, err, "Error on test request")
body, err := io.ReadAll(resp.Body)
defer resp.Body.Close()
if err != nil {
t.Fatalf("Error reading response body: %s", err)
}
if want, have := http.StatusServiceUnavailable, resp.StatusCode; want != have {
t.Fatalf("Return code %d expected in error response but got %d", want, have)
}
if h := resp.Header.Get("Content-Type"); h != "application/json" {
t.Fatalf("Expected Content-Type %q but got %q", "application/json", h)
}
require.NoError(t, err, "Error reading response body")
want, have := http.StatusServiceUnavailable, resp.StatusCode
require.Equal(t, want, have, "Return code %d expected in error response but got %d", want, have)
h := resp.Header.Get("Content-Type")
require.Equal(t, "application/json", h, "Expected Content-Type %q but got %q", "application/json", h)
var res Response
if err = json.Unmarshal(body, &res); err != nil {
t.Fatalf("Error unmarshaling JSON body: %s", err)
}
err = json.Unmarshal(body, &res)
require.NoError(t, err, "Error unmarshaling JSON body")
exp := &Response{
Status: statusError,
@ -3420,17 +3380,13 @@ func TestParseTime(t *testing.T) {
for _, test := range tests {
ts, err := parseTime(test.input)
if err != nil && !test.fail {
t.Errorf("Unexpected error for %q: %s", test.input, err)
if !test.fail {
require.NoError(t, err, "Unexpected error for %q", test.input)
require.NotNil(t, ts)
require.True(t, ts.Equal(test.result), "Expected time %v for input %q but got %v", test.result, test.input, ts)
continue
}
if err == nil && test.fail {
t.Errorf("Expected error for %q but got none", test.input)
continue
}
if !test.fail && !ts.Equal(test.result) {
t.Errorf("Expected time %v for input %q but got %v", test.result, test.input, ts)
}
require.Error(t, err, "Expected error for %q but got none", test.input)
}
}
@ -3474,17 +3430,12 @@ func TestParseDuration(t *testing.T) {
for _, test := range tests {
d, err := parseDuration(test.input)
if err != nil && !test.fail {
t.Errorf("Unexpected error for %q: %s", test.input, err)
if !test.fail {
require.NoError(t, err, "Unexpected error for %q", test.input)
require.Equal(t, test.result, d, "Expected duration %v for input %q but got %v", test.result, test.input, d)
continue
}
if err == nil && test.fail {
t.Errorf("Expected error for %q but got none", test.input)
continue
}
if !test.fail && d != test.result {
t.Errorf("Expected duration %v for input %q but got %v", test.result, test.input, d)
}
require.Error(t, err, "Expected error for %q but got none", test.input)
}
}
@ -3497,18 +3448,11 @@ func TestOptionsMethod(t *testing.T) {
defer s.Close()
req, err := http.NewRequest("OPTIONS", s.URL+"/any_path", nil)
if err != nil {
t.Fatalf("Error creating OPTIONS request: %s", err)
}
require.NoError(t, err, "Error creating OPTIONS request")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
t.Fatalf("Error executing OPTIONS request: %s", err)
}
if resp.StatusCode != http.StatusNoContent {
t.Fatalf("Expected status %d, got %d", http.StatusNoContent, resp.StatusCode)
}
require.NoError(t, err, "Error executing OPTIONS request")
require.Equal(t, http.StatusNoContent, resp.StatusCode)
}
func TestTSDBStatus(t *testing.T) {
@ -3547,9 +3491,7 @@ func TestTSDBStatus(t *testing.T) {
api := &API{db: tc.db, gatherer: prometheus.DefaultGatherer}
endpoint := tc.endpoint(api)
req, err := http.NewRequest(tc.method, fmt.Sprintf("?%s", tc.values.Encode()), nil)
if err != nil {
t.Fatalf("Error when creating test request: %s", err)
}
require.NoError(t, err, "Error when creating test request")
res := endpoint(req)
assertAPIError(t, res.err, tc.errType)
})

8
web/api/v1/json_codec_test.go
View file

@ -134,14 +134,14 @@ func TestJsonCodec_Encode(t *testing.T) {
SeriesLabels: labels.FromStrings("foo", "bar"),
Exemplars: []exemplar.Exemplar{
{
Labels: labels.FromStrings("traceID", "abc"),
Labels: labels.FromStrings("trace_id", "abc"),
Value: 100.123,
Ts: 1234,
},
},
},
},
expected: `{"status":"success","data":[{"seriesLabels":{"foo":"bar"},"exemplars":[{"labels":{"traceID":"abc"},"value":"100.123","timestamp":1.234}]}]}`,
expected: `{"status":"success","data":[{"seriesLabels":{"foo":"bar"},"exemplars":[{"labels":{"trace_id":"abc"},"value":"100.123","timestamp":1.234}]}]}`,
},
{
response: []exemplar.QueryResult{
@ -149,14 +149,14 @@ func TestJsonCodec_Encode(t *testing.T) {
SeriesLabels: labels.FromStrings("foo", "bar"),
Exemplars: []exemplar.Exemplar{
{
Labels: labels.FromStrings("traceID", "abc"),
Labels: labels.FromStrings("trace_id", "abc"),
Value: math.Inf(1),
Ts: 1234,
},
},
},
},
expected: `{"status":"success","data":[{"seriesLabels":{"foo":"bar"},"exemplars":[{"labels":{"traceID":"abc"},"value":"+Inf","timestamp":1.234}]}]}`,
expected: `{"status":"success","data":[{"seriesLabels":{"foo":"bar"},"exemplars":[{"labels":{"trace_id":"abc"},"value":"+Inf","timestamp":1.234}]}]}`,
},
}

3
web/federate.go
View file

@ -189,8 +189,9 @@ Loop:
)
for _, s := range vec {
isHistogram := s.H != nil
formatType := format.FormatType()
if isHistogram &&
format != expfmt.FmtProtoDelim && format != expfmt.FmtProtoText && format != expfmt.FmtProtoCompact {
formatType != expfmt.TypeProtoDelim && formatType != expfmt.TypeProtoText && formatType != expfmt.TypeProtoCompact {
// Can't serve the native histogram.
// TODO(codesome): Serve them when other protocols get the native histogram support.
continue

3
web/federate_test.go
View file

@ -37,6 +37,7 @@ import (
"github.com/prometheus/prometheus/storage"
"github.com/prometheus/prometheus/tsdb"
"github.com/prometheus/prometheus/util/teststorage"
"github.com/prometheus/prometheus/util/testutil"
)
var scenarios = map[string]struct {
@ -427,5 +428,5 @@ func TestFederationWithNativeHistograms(t *testing.T) {
// TODO(codesome): Once PromQL is able to set the CounterResetHint on histograms,
// test it with switching histogram types for metric families.
require.Equal(t, 4, metricFamilies)
require.Equal(t, expVec, actVec)
testutil.RequireEqual(t, expVec, actVec)
}

6
web/ui/module/codemirror-promql/src/complete/promql.terms.ts
View file

@ -215,6 +215,12 @@ export const functionIdentifierTerms = [
info: 'Round down values of input series to nearest integer',
type: 'function',
},
{
label: 'histogram_avg',
detail: 'function',
info: 'Return the average of observations from a native histogram (experimental feature)',
type: 'function',
},
{
label: 'histogram_count',
detail: 'function',

12
web/ui/module/codemirror-promql/src/parser/parser.test.ts
View file

@ -757,6 +757,18 @@ describe('promql operations', () => {
expectedValueType: ValueType.vector,
expectedDiag: [],
},
{
expr:
'histogram_avg( # Root of the query, final result, returns the average of observations.\n' +
' sum by(method, path) ( # Argument to histogram_avg(), an aggregated histogram.\n' +
' rate( # Argument to sum(), the per-second increase of a histogram over 5m.\n' +
' demo_api_request_duration_seconds{job="demo"}[5m] # Argument to rate(), a vector of sparse histogram series over the last 5m.\n' +
' )\n' +
' )\n' +
')',
expectedValueType: ValueType.vector,
expectedDiag: [],
},
{
expr:
'histogram_stddev( # Root of the query, final result, returns the standard deviation of observations.\n' +

7
web/ui/module/codemirror-promql/src/types/function.ts
View file

@ -39,6 +39,7 @@ import {
Deriv,
Exp,
Floor,
HistogramAvg,
HistogramCount,
HistogramFraction,
HistogramQuantile,
@ -269,6 +270,12 @@ const promqlFunctions: { [key: number]: PromQLFunction } = {
variadic: 0,
returnType: ValueType.vector,
},
[HistogramAvg]: {
name: 'histogram_avg',
argTypes: [ValueType.vector],
variadic: 0,
returnType: ValueType.vector,
},
[HistogramCount]: {
name: 'histogram_count',
argTypes: [ValueType.vector],

2
web/ui/module/lezer-promql/src/highlight.js
View file

@ -20,7 +20,7 @@ export const promQLHighLight = styleTags({
NumberLiteral: tags.number,
Duration: tags.number,
Identifier: tags.variableName,
'Abs Absent AbsentOverTime Acos Acosh Asin Asinh Atan Atanh AvgOverTime Ceil Changes Clamp ClampMax ClampMin Cos Cosh CountOverTime DaysInMonth DayOfMonth DayOfWeek DayOfYear Deg Delta Deriv Exp Floor HistogramCount HistogramFraction HistogramQuantile HistogramSum HoltWinters Hour Idelta Increase Irate LabelReplace LabelJoin LastOverTime Ln Log10 Log2 MaxOverTime MinOverTime Minute Month Pi PredictLinear PresentOverTime QuantileOverTime Rad Rate Resets Round Scalar Sgn Sin Sinh Sort SortDesc SortByLabel SortByLabelDesc Sqrt StddevOverTime StdvarOverTime SumOverTime Tan Tanh Time Timestamp Vector Year':
'Abs Absent AbsentOverTime Acos Acosh Asin Asinh Atan Atanh AvgOverTime Ceil Changes Clamp ClampMax ClampMin Cos Cosh CountOverTime DaysInMonth DayOfMonth DayOfWeek DayOfYear Deg Delta Deriv Exp Floor HistogramAvg HistogramCount HistogramFraction HistogramQuantile HistogramSum HoltWinters Hour Idelta Increase Irate LabelReplace LabelJoin LastOverTime Ln Log10 Log2 MaxOverTime MinOverTime Minute Month Pi PredictLinear PresentOverTime QuantileOverTime Rad Rate Resets Round Scalar Sgn Sin Sinh Sort SortDesc SortByLabel SortByLabelDesc Sqrt StddevOverTime StdvarOverTime SumOverTime Tan Tanh Time Timestamp Vector Year':
tags.function(tags.variableName),
'Avg Bottomk Count Count_values Group Max Min Quantile Stddev Stdvar Sum Topk': tags.operatorKeyword,
'By Without Bool On Ignoring GroupLeft GroupRight Offset Start End': tags.modifier,

2
web/ui/module/lezer-promql/src/promql.grammar
View file

@ -138,6 +138,7 @@ FunctionIdentifier {
HistogramStdDev |
HistogramStdVar |
HistogramSum |
HistogramAvg |
HoltWinters |
Hour |
Idelta |
@ -364,6 +365,7 @@ NumberLiteral {
Deriv { condFn<"deriv"> }
Exp { condFn<"exp"> }
Floor { condFn<"floor"> }
HistogramAvg { condFn<"histogram_avg"> }
HistogramCount { condFn<"histogram_count"> }
HistogramFraction { condFn<"histogram_fraction"> }
HistogramQuantile { condFn<"histogram_quantile"> }

2
web/ui/react-app/src/pages/graph/Graph.test.tsx
View file

@ -80,7 +80,7 @@ describe('Graph', () => {
exemplars: [
{
labels: {
traceID: '12345',
trace_id: '12345',
},
timestamp: 1572130580,
value: '9',

4
web/web_test.go
View file

@ -440,7 +440,7 @@ func TestShutdownWithStaleConnection(t *testing.T) {
select {
case <-closed:
case <-time.After(timeout + 5*time.Second):
t.Fatalf("Server still running after read timeout.")
require.FailNow(t, "Server still running after read timeout.")
}
}
@ -502,7 +502,7 @@ func TestHandleMultipleQuitRequests(t *testing.T) {
select {
case <-closed:
case <-time.After(5 * time.Second):
t.Fatalf("Server still running after 5 seconds.")
require.FailNow(t, "Server still running after 5 seconds.")
}
}