mirror of
https://github.com/prometheus/prometheus.git
synced 2024-09-19 23:37:31 -07:00
d121db7a65
In most cases, there is no sample at `maxt`, so `PeekBack` has to be used. So far, `PeekBack` did not return a float histogram, and we disregarded even any returned normal histogram. This fixes both, and also tweaks the unit test to discover the problem (by using an earlier timestamp than "now" for the samples in the TSDB). Signed-off-by: beorn7 <beorn@grafana.com>
417 lines
12 KiB
Go
417 lines
12 KiB
Go
// Copyright 2016 The Prometheus Authors
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package web
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/pkg/errors"
|
|
"github.com/prometheus/common/model"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/prometheus/prometheus/config"
|
|
"github.com/prometheus/prometheus/model/histogram"
|
|
"github.com/prometheus/prometheus/model/labels"
|
|
"github.com/prometheus/prometheus/model/textparse"
|
|
"github.com/prometheus/prometheus/promql"
|
|
"github.com/prometheus/prometheus/storage"
|
|
"github.com/prometheus/prometheus/tsdb"
|
|
)
|
|
|
|
var scenarios = map[string]struct {
|
|
params string
|
|
externalLabels labels.Labels
|
|
code int
|
|
body string
|
|
}{
|
|
"empty": {
|
|
params: "",
|
|
code: 200,
|
|
body: ``,
|
|
},
|
|
"match nothing": {
|
|
params: "match[]=does_not_match_anything",
|
|
code: 200,
|
|
body: ``,
|
|
},
|
|
"invalid params from the beginning": {
|
|
params: "match[]=-not-a-valid-metric-name",
|
|
code: 400,
|
|
body: `1:1: parse error: unexpected <op:->
|
|
`,
|
|
},
|
|
"invalid params somewhere in the middle": {
|
|
params: "match[]=not-a-valid-metric-name",
|
|
code: 400,
|
|
body: `1:4: parse error: unexpected <op:->
|
|
`,
|
|
},
|
|
"test_metric1": {
|
|
params: "match[]=test_metric1",
|
|
code: 200,
|
|
body: `# TYPE test_metric1 untyped
|
|
test_metric1{foo="bar",instance="i"} 10000 6000000
|
|
test_metric1{foo="boo",instance="i"} 1 6000000
|
|
`,
|
|
},
|
|
"test_metric2": {
|
|
params: "match[]=test_metric2",
|
|
code: 200,
|
|
body: `# TYPE test_metric2 untyped
|
|
test_metric2{foo="boo",instance="i"} 1 6000000
|
|
`,
|
|
},
|
|
"test_metric_without_labels": {
|
|
params: "match[]=test_metric_without_labels",
|
|
code: 200,
|
|
body: `# TYPE test_metric_without_labels untyped
|
|
test_metric_without_labels{instance=""} 1001 6000000
|
|
`,
|
|
},
|
|
"test_stale_metric": {
|
|
params: "match[]=test_metric_stale",
|
|
code: 200,
|
|
body: ``,
|
|
},
|
|
"test_old_metric": {
|
|
params: "match[]=test_metric_old",
|
|
code: 200,
|
|
body: `# TYPE test_metric_old untyped
|
|
test_metric_old{instance=""} 981 5880000
|
|
`,
|
|
},
|
|
"{foo='boo'}": {
|
|
params: "match[]={foo='boo'}",
|
|
code: 200,
|
|
body: `# TYPE test_metric1 untyped
|
|
test_metric1{foo="boo",instance="i"} 1 6000000
|
|
# TYPE test_metric2 untyped
|
|
test_metric2{foo="boo",instance="i"} 1 6000000
|
|
`,
|
|
},
|
|
"two matchers": {
|
|
params: "match[]=test_metric1&match[]=test_metric2",
|
|
code: 200,
|
|
body: `# TYPE test_metric1 untyped
|
|
test_metric1{foo="bar",instance="i"} 10000 6000000
|
|
test_metric1{foo="boo",instance="i"} 1 6000000
|
|
# TYPE test_metric2 untyped
|
|
test_metric2{foo="boo",instance="i"} 1 6000000
|
|
`,
|
|
},
|
|
"two matchers with overlap": {
|
|
params: "match[]={__name__=~'test_metric1'}&match[]={foo='bar'}",
|
|
code: 200,
|
|
body: `# TYPE test_metric1 untyped
|
|
test_metric1{foo="bar",instance="i"} 10000 6000000
|
|
test_metric1{foo="boo",instance="i"} 1 6000000
|
|
`,
|
|
},
|
|
"everything": {
|
|
params: "match[]={__name__=~'.%2b'}", // '%2b' is an URL-encoded '+'.
|
|
code: 200,
|
|
body: `# TYPE test_metric1 untyped
|
|
test_metric1{foo="bar",instance="i"} 10000 6000000
|
|
test_metric1{foo="boo",instance="i"} 1 6000000
|
|
# TYPE test_metric2 untyped
|
|
test_metric2{foo="boo",instance="i"} 1 6000000
|
|
# TYPE test_metric_old untyped
|
|
test_metric_old{instance=""} 981 5880000
|
|
# TYPE test_metric_without_labels untyped
|
|
test_metric_without_labels{instance=""} 1001 6000000
|
|
`,
|
|
},
|
|
"empty label value matches everything that doesn't have that label": {
|
|
params: "match[]={foo='',__name__=~'.%2b'}",
|
|
code: 200,
|
|
body: `# TYPE test_metric_old untyped
|
|
test_metric_old{instance=""} 981 5880000
|
|
# TYPE test_metric_without_labels untyped
|
|
test_metric_without_labels{instance=""} 1001 6000000
|
|
`,
|
|
},
|
|
"empty label value for a label that doesn't exist at all, matches everything": {
|
|
params: "match[]={bar='',__name__=~'.%2b'}",
|
|
code: 200,
|
|
body: `# TYPE test_metric1 untyped
|
|
test_metric1{foo="bar",instance="i"} 10000 6000000
|
|
test_metric1{foo="boo",instance="i"} 1 6000000
|
|
# TYPE test_metric2 untyped
|
|
test_metric2{foo="boo",instance="i"} 1 6000000
|
|
# TYPE test_metric_old untyped
|
|
test_metric_old{instance=""} 981 5880000
|
|
# TYPE test_metric_without_labels untyped
|
|
test_metric_without_labels{instance=""} 1001 6000000
|
|
`,
|
|
},
|
|
"external labels are added if not already present": {
|
|
params: "match[]={__name__=~'.%2b'}", // '%2b' is an URL-encoded '+'.
|
|
externalLabels: labels.FromStrings("foo", "baz", "zone", "ie"),
|
|
code: 200,
|
|
body: `# TYPE test_metric1 untyped
|
|
test_metric1{foo="bar",instance="i",zone="ie"} 10000 6000000
|
|
test_metric1{foo="boo",instance="i",zone="ie"} 1 6000000
|
|
# TYPE test_metric2 untyped
|
|
test_metric2{foo="boo",instance="i",zone="ie"} 1 6000000
|
|
# TYPE test_metric_old untyped
|
|
test_metric_old{foo="baz",instance="",zone="ie"} 981 5880000
|
|
# TYPE test_metric_without_labels untyped
|
|
test_metric_without_labels{foo="baz",instance="",zone="ie"} 1001 6000000
|
|
`,
|
|
},
|
|
"instance is an external label": {
|
|
// This makes no sense as a configuration, but we should
|
|
// know what it does anyway.
|
|
params: "match[]={__name__=~'.%2b'}", // '%2b' is an URL-encoded '+'.
|
|
externalLabels: labels.FromStrings("instance", "baz"),
|
|
code: 200,
|
|
body: `# TYPE test_metric1 untyped
|
|
test_metric1{foo="bar",instance="i"} 10000 6000000
|
|
test_metric1{foo="boo",instance="i"} 1 6000000
|
|
# TYPE test_metric2 untyped
|
|
test_metric2{foo="boo",instance="i"} 1 6000000
|
|
# TYPE test_metric_old untyped
|
|
test_metric_old{instance="baz"} 981 5880000
|
|
# TYPE test_metric_without_labels untyped
|
|
test_metric_without_labels{instance="baz"} 1001 6000000
|
|
`,
|
|
},
|
|
}
|
|
|
|
func TestFederation(t *testing.T) {
|
|
suite, err := promql.NewTest(t, `
|
|
load 1m
|
|
test_metric1{foo="bar",instance="i"} 0+100x100
|
|
test_metric1{foo="boo",instance="i"} 1+0x100
|
|
test_metric2{foo="boo",instance="i"} 1+0x100
|
|
test_metric_without_labels 1+10x100
|
|
test_metric_stale 1+10x99 stale
|
|
test_metric_old 1+10x98
|
|
`)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer suite.Close()
|
|
|
|
if err := suite.Run(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
h := &Handler{
|
|
localStorage: &dbAdapter{suite.TSDB()},
|
|
lookbackDelta: 5 * time.Minute,
|
|
now: func() model.Time { return 101 * 60 * 1000 }, // 101min after epoch.
|
|
config: &config.Config{
|
|
GlobalConfig: config.GlobalConfig{},
|
|
},
|
|
}
|
|
|
|
for name, scenario := range scenarios {
|
|
t.Run(name, func(t *testing.T) {
|
|
h.config.GlobalConfig.ExternalLabels = scenario.externalLabels
|
|
req := httptest.NewRequest("GET", "http://example.org/federate?"+scenario.params, nil)
|
|
res := httptest.NewRecorder()
|
|
|
|
h.federation(res, req)
|
|
require.Equal(t, scenario.code, res.Code)
|
|
require.Equal(t, scenario.body, normalizeBody(res.Body))
|
|
})
|
|
}
|
|
}
|
|
|
|
type notReadyReadStorage struct {
|
|
LocalStorage
|
|
}
|
|
|
|
func (notReadyReadStorage) Querier(context.Context, int64, int64) (storage.Querier, error) {
|
|
return nil, errors.Wrap(tsdb.ErrNotReady, "wrap")
|
|
}
|
|
|
|
func (notReadyReadStorage) StartTime() (int64, error) {
|
|
return 0, errors.Wrap(tsdb.ErrNotReady, "wrap")
|
|
}
|
|
|
|
func (notReadyReadStorage) Stats(string) (*tsdb.Stats, error) {
|
|
return nil, errors.Wrap(tsdb.ErrNotReady, "wrap")
|
|
}
|
|
|
|
// Regression test for https://github.com/prometheus/prometheus/issues/7181.
|
|
func TestFederation_NotReady(t *testing.T) {
|
|
for name, scenario := range scenarios {
|
|
t.Run(name, func(t *testing.T) {
|
|
h := &Handler{
|
|
localStorage: notReadyReadStorage{},
|
|
lookbackDelta: 5 * time.Minute,
|
|
now: func() model.Time { return 101 * 60 * 1000 }, // 101min after epoch.
|
|
config: &config.Config{
|
|
GlobalConfig: config.GlobalConfig{
|
|
ExternalLabels: scenario.externalLabels,
|
|
},
|
|
},
|
|
}
|
|
|
|
req := httptest.NewRequest("GET", "http://example.org/federate?"+scenario.params, nil)
|
|
res := httptest.NewRecorder()
|
|
|
|
h.federation(res, req)
|
|
if scenario.code == http.StatusBadRequest {
|
|
// Request are expected to be checked before DB readiness.
|
|
require.Equal(t, http.StatusBadRequest, res.Code)
|
|
return
|
|
}
|
|
require.Equal(t, http.StatusServiceUnavailable, res.Code)
|
|
})
|
|
}
|
|
}
|
|
|
|
// normalizeBody sorts the lines within a metric to make it easy to verify the body.
|
|
// (Federation is not taking care of sorting within a metric family.)
|
|
func normalizeBody(body *bytes.Buffer) string {
|
|
var (
|
|
lines []string
|
|
lastHash int
|
|
)
|
|
for line, err := body.ReadString('\n'); err == nil; line, err = body.ReadString('\n') {
|
|
if line[0] == '#' && len(lines) > 0 {
|
|
sort.Strings(lines[lastHash+1:])
|
|
lastHash = len(lines)
|
|
}
|
|
lines = append(lines, line)
|
|
}
|
|
if len(lines) > 0 {
|
|
sort.Strings(lines[lastHash+1:])
|
|
}
|
|
return strings.Join(lines, "")
|
|
}
|
|
|
|
func TestFederationWithNativeHistograms(t *testing.T) {
|
|
suite, err := promql.NewTest(t, "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer suite.Close()
|
|
|
|
if err := suite.Run(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
var expVec promql.Vector
|
|
|
|
db := suite.TSDB()
|
|
hist := &histogram.Histogram{
|
|
Count: 10,
|
|
ZeroCount: 2,
|
|
ZeroThreshold: 0.001,
|
|
Sum: 39.4,
|
|
Schema: 1,
|
|
PositiveSpans: []histogram.Span{
|
|
{Offset: 0, Length: 2},
|
|
{Offset: 1, Length: 2},
|
|
},
|
|
PositiveBuckets: []int64{1, 1, -1, 0},
|
|
NegativeSpans: []histogram.Span{
|
|
{Offset: 0, Length: 2},
|
|
{Offset: 1, Length: 2},
|
|
},
|
|
NegativeBuckets: []int64{1, 1, -1, 0},
|
|
}
|
|
app := db.Appender(context.Background())
|
|
for i := 0; i < 6; i++ {
|
|
l := labels.FromStrings("__name__", "test_metric", "foo", fmt.Sprintf("%d", i))
|
|
expL := labels.FromStrings("__name__", "test_metric", "instance", "", "foo", fmt.Sprintf("%d", i))
|
|
if i%3 == 0 {
|
|
_, err = app.Append(0, l, 100*60*1000, float64(i*100))
|
|
expVec = append(expVec, promql.Sample{
|
|
Point: promql.Point{T: 100 * 60 * 1000, V: float64(i * 100)},
|
|
Metric: expL,
|
|
})
|
|
} else {
|
|
hist.ZeroCount++
|
|
_, err = app.AppendHistogram(0, l, 100*60*1000, hist.Copy(), nil)
|
|
expVec = append(expVec, promql.Sample{
|
|
Point: promql.Point{T: 100 * 60 * 1000, H: hist.ToFloat()},
|
|
Metric: expL,
|
|
})
|
|
}
|
|
require.NoError(t, err)
|
|
}
|
|
require.NoError(t, app.Commit())
|
|
|
|
h := &Handler{
|
|
localStorage: &dbAdapter{suite.TSDB()},
|
|
lookbackDelta: 5 * time.Minute,
|
|
now: func() model.Time { return 101 * 60 * 1000 }, // 101min after epoch.
|
|
config: &config.Config{
|
|
GlobalConfig: config.GlobalConfig{},
|
|
},
|
|
}
|
|
|
|
req := httptest.NewRequest("GET", "http://example.org/federate?match[]=test_metric", nil)
|
|
req.Header.Add("Accept", `application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited,application/openmetrics-text;version=1.0.0;q=0.8,application/openmetrics-text;version=0.0.1;q=0.75,text/plain;version=0.0.4;q=0.5,*/*;q=0.1`)
|
|
res := httptest.NewRecorder()
|
|
|
|
h.federation(res, req)
|
|
|
|
require.Equal(t, http.StatusOK, res.Code)
|
|
body, err := io.ReadAll(res.Body)
|
|
require.NoError(t, err)
|
|
|
|
p := textparse.NewProtobufParser(body)
|
|
var actVec promql.Vector
|
|
metricFamilies := 0
|
|
for {
|
|
et, err := p.Next()
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
require.NoError(t, err)
|
|
if et == textparse.EntryHelp {
|
|
metricFamilies++
|
|
}
|
|
if et == textparse.EntryHistogram || et == textparse.EntrySeries {
|
|
l := labels.Labels{}
|
|
p.Metric(&l)
|
|
actVec = append(actVec, promql.Sample{Metric: l})
|
|
}
|
|
if et == textparse.EntryHistogram {
|
|
_, parsedTimestamp, h, fh := p.Histogram()
|
|
require.Nil(t, h)
|
|
actVec[len(actVec)-1].Point = promql.Point{
|
|
T: *parsedTimestamp,
|
|
H: fh,
|
|
}
|
|
} else if et == textparse.EntrySeries {
|
|
_, parsedTimestamp, v := p.Series()
|
|
actVec[len(actVec)-1].Point = promql.Point{
|
|
T: *parsedTimestamp,
|
|
V: v,
|
|
}
|
|
}
|
|
}
|
|
|
|
// TODO(codesome): Once PromQL is able to set the CounterResetHint on histograms,
|
|
// test it with switching histogram types for metric families.
|
|
require.Equal(t, 4, metricFamilies)
|
|
require.Equal(t, expVec, actVec)
|
|
}
|