mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2024-11-10 07:34:04 -08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Merge pull request #1774 from prometheus/beorn7/index

Browse source 
storage: improve index lookups
This commit is contained in:
Björn Rabenstein 2016-07-20 17:38:04 +02:00 committed by GitHub
parent 40f8da699e fc6737b7fb
commit 5fab430e73
7 changed files with 349 additions and 187 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
promql/parse.go
View file

@ -953,11 +953,11 @@ func (p *parser) vectorSelector(name string) *VectorSelector {
} }
} }
// Set name label matching. // Set name label matching.
matchers = append(matchers, &metric.LabelMatcher{ m, err := metric.NewLabelMatcher(metric.Equal, model.MetricNameLabel, model.LabelValue(name))
Type: metric.Equal, if err != nil {
Name: model.MetricNameLabel, panic(err) // Must not happen with metric.Equal.
Value: model.LabelValue(name), }
}) matchers = append(matchers, m)
} }
if len(matchers) == 0 { if len(matchers) == 0 {
@ -967,14 +967,7 @@ func (p *parser) vectorSelector(name string) *VectorSelector {
// implicit selection of all metrics (e.g. by a typo). // implicit selection of all metrics (e.g. by a typo).
notEmpty := false notEmpty := false
for _, lm := range matchers { for _, lm := range matchers {
// Matching changes the inner state of the regex and causes reflect.DeepEqual if !lm.MatchesEmptyString() {
// to return false, which break tests.
// Thus, we create a new label matcher for this testing.
lm, err := metric.NewLabelMatcher(lm.Type, lm.Name, lm.Value)
if err != nil {
p.error(err)
}
if !lm.Match("") {
notEmpty = true notEmpty = true
break break
} }

182
promql/parse_test.go
View file

@ -144,7 +144,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
}, },
@ -154,7 +154,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
}, },
@ -263,13 +263,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{Card: CardOneToOne}, VectorMatching: &VectorMatching{Card: CardOneToOne},
@ -281,7 +281,7 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &NumberLiteral{1}, RHS: &NumberLiteral{1},
@ -293,7 +293,7 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &NumberLiteral{1}, RHS: &NumberLiteral{1},
@ -307,7 +307,7 @@ var testExpr = []struct {
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
}, },
@ -318,13 +318,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{Card: CardManyToMany}, VectorMatching: &VectorMatching{Card: CardManyToMany},
@ -336,13 +336,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{Card: CardManyToMany}, VectorMatching: &VectorMatching{Card: CardManyToMany},
@ -354,13 +354,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{Card: CardManyToMany}, VectorMatching: &VectorMatching{Card: CardManyToMany},
@ -375,13 +375,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{Card: CardOneToOne}, VectorMatching: &VectorMatching{Card: CardOneToOne},
@ -391,13 +391,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "bla", Name: "bla",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bla"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bla"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "blub", Name: "blub",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "blub"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "blub"),
}, },
}, },
VectorMatching: &VectorMatching{Card: CardManyToMany}, VectorMatching: &VectorMatching{Card: CardManyToMany},
@ -416,13 +416,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{Card: CardManyToMany}, VectorMatching: &VectorMatching{Card: CardManyToMany},
@ -430,7 +430,7 @@ var testExpr = []struct {
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "baz", Name: "baz",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "baz"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "baz"),
}, },
}, },
VectorMatching: &VectorMatching{Card: CardManyToMany}, VectorMatching: &VectorMatching{Card: CardManyToMany},
@ -438,7 +438,7 @@ var testExpr = []struct {
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "qux", Name: "qux",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "qux"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "qux"),
}, },
}, },
VectorMatching: &VectorMatching{Card: CardManyToMany}, VectorMatching: &VectorMatching{Card: CardManyToMany},
@ -451,7 +451,7 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
RHS: &BinaryExpr{ RHS: &BinaryExpr{
@ -459,13 +459,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "bla", Name: "bla",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bla"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bla"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "blub", Name: "blub",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "blub"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "blub"),
}, },
}, },
VectorMatching: &VectorMatching{ VectorMatching: &VectorMatching{
@ -488,13 +488,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{ VectorMatching: &VectorMatching{
@ -510,13 +510,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{ VectorMatching: &VectorMatching{
@ -532,13 +532,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{ VectorMatching: &VectorMatching{
@ -554,13 +554,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{ VectorMatching: &VectorMatching{
@ -576,13 +576,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{ VectorMatching: &VectorMatching{
@ -597,13 +597,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{ VectorMatching: &VectorMatching{
@ -618,13 +618,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "baz", Name: "baz",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "baz"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "baz"),
}, },
}, },
VectorMatching: &VectorMatching{ VectorMatching: &VectorMatching{
@ -640,13 +640,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{ VectorMatching: &VectorMatching{
@ -663,13 +663,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{ VectorMatching: &VectorMatching{
@ -685,13 +685,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{ VectorMatching: &VectorMatching{
@ -707,13 +707,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{ VectorMatching: &VectorMatching{
@ -730,13 +730,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{ VectorMatching: &VectorMatching{
@ -825,7 +825,7 @@ var testExpr = []struct {
Name: "foo", Name: "foo",
Offset: 0, Offset: 0,
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
}, { }, {
@ -834,7 +834,7 @@ var testExpr = []struct {
Name: "foo", Name: "foo",
Offset: 5 * time.Minute, Offset: 5 * time.Minute,
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
}, { }, {
@ -843,8 +843,8 @@ var testExpr = []struct {
Name: "foo:bar", Name: "foo:bar",
Offset: 0, Offset: 0,
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: "a", Value: "bc"}, mustLabelMatcher(metric.Equal, "a", "bc"),
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo:bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo:bar"),
}, },
}, },
}, { }, {
@ -853,8 +853,8 @@ var testExpr = []struct {
Name: "foo", Name: "foo",
Offset: 0, Offset: 0,
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: "NaN", Value: "bc"}, mustLabelMatcher(metric.Equal, "NaN", "bc"),
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
}, { }, {
@ -863,11 +863,11 @@ var testExpr = []struct {
Name: "foo", Name: "foo",
Offset: 0, Offset: 0,
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: "a", Value: "b"}, mustLabelMatcher(metric.Equal, "a", "b"),
{Type: metric.NotEqual, Name: "foo", Value: "bar"}, mustLabelMatcher(metric.NotEqual, "foo", "bar"),
mustLabelMatcher(metric.RegexMatch, "test", "test"), mustLabelMatcher(metric.RegexMatch, "test", "test"),
mustLabelMatcher(metric.RegexNoMatch, "bar", "baz"), mustLabelMatcher(metric.RegexNoMatch, "bar", "baz"),
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
}, { }, {
@ -949,7 +949,7 @@ var testExpr = []struct {
Offset: 0, Offset: 0,
Range: 5 * time.Second, Range: 5 * time.Second,
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "test"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "test"),
}, },
}, },
}, { }, {
@ -959,7 +959,7 @@ var testExpr = []struct {
Offset: 0, Offset: 0,
Range: 5 * time.Minute, Range: 5 * time.Minute,
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "test"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "test"),
}, },
}, },
}, { }, {
@ -969,7 +969,7 @@ var testExpr = []struct {
Offset: 5 * time.Minute, Offset: 5 * time.Minute,
Range: 5 * time.Hour, Range: 5 * time.Hour,
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "test"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "test"),
}, },
}, },
}, { }, {
@ -979,7 +979,7 @@ var testExpr = []struct {
Offset: 10 * time.Second, Offset: 10 * time.Second,
Range: 5 * 24 * time.Hour, Range: 5 * 24 * time.Hour,
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "test"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "test"),
}, },
}, },
}, { }, {
@ -989,7 +989,7 @@ var testExpr = []struct {
Offset: 14 * 24 * time.Hour, Offset: 14 * 24 * time.Hour,
Range: 5 * 7 * 24 * time.Hour, Range: 5 * 7 * 24 * time.Hour,
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "test"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "test"),
}, },
}, },
}, { }, {
@ -999,8 +999,8 @@ var testExpr = []struct {
Offset: 3 * 24 * time.Hour, Offset: 3 * 24 * time.Hour,
Range: 5 * 365 * 24 * time.Hour, Range: 5 * 365 * 24 * time.Hour,
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: "a", Value: "b"}, mustLabelMatcher(metric.Equal, "a", "b"),
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "test"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "test"),
}, },
}, },
}, { }, {
@ -1059,7 +1059,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
Grouping: model.LabelNames{"foo"}, Grouping: model.LabelNames{"foo"},
@ -1072,7 +1072,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
Grouping: model.LabelNames{"foo"}, Grouping: model.LabelNames{"foo"},
@ -1085,7 +1085,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
Grouping: model.LabelNames{"foo", "bar"}, Grouping: model.LabelNames{"foo", "bar"},
@ -1097,7 +1097,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
Grouping: model.LabelNames{"foo"}, Grouping: model.LabelNames{"foo"},
@ -1109,7 +1109,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
Grouping: model.LabelNames{"foo"}, Grouping: model.LabelNames{"foo"},
@ -1122,7 +1122,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
Grouping: model.LabelNames{"foo"}, Grouping: model.LabelNames{"foo"},
@ -1135,7 +1135,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
Grouping: model.LabelNames{"foo"}, Grouping: model.LabelNames{"foo"},
@ -1148,7 +1148,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
Grouping: model.LabelNames{"foo"}, Grouping: model.LabelNames{"foo"},
@ -1161,7 +1161,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
Grouping: model.LabelNames{"foo"}, Grouping: model.LabelNames{"foo"},
@ -1173,7 +1173,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
}, },
@ -1184,7 +1184,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
Grouping: model.LabelNames{"foo"}, Grouping: model.LabelNames{"foo"},
@ -1196,7 +1196,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
Grouping: model.LabelNames{}, Grouping: model.LabelNames{},
@ -1208,7 +1208,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
Param: &NumberLiteral{5}, Param: &NumberLiteral{5},
@ -1220,7 +1220,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
Param: &StringLiteral{"value"}, Param: &StringLiteral{"value"},
@ -1288,8 +1288,8 @@ var testExpr = []struct {
&VectorSelector{ &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.NotEqual, Name: "foo", Value: "bar"}, mustLabelMatcher(metric.NotEqual, "foo", "bar"),
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
}, },
@ -1302,7 +1302,7 @@ var testExpr = []struct {
&MatrixSelector{ &MatrixSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
Range: 5 * time.Minute, Range: 5 * time.Minute,
}, },
@ -1316,7 +1316,7 @@ var testExpr = []struct {
&VectorSelector{ &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
}, },
@ -1329,7 +1329,7 @@ var testExpr = []struct {
&VectorSelector{ &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
&NumberLiteral{5}, &NumberLiteral{5},
@ -1537,7 +1537,7 @@ var testStatement = []struct {
&MatrixSelector{ &MatrixSelector{
Name: "http_request_count", Name: "http_request_count",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "http_request_count"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "http_request_count"),
}, },
Range: 5 * time.Minute, Range: 5 * time.Minute,
}, },
@ -1553,7 +1553,7 @@ var testStatement = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "dc:http_request:rate5m", Name: "dc:http_request:rate5m",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "dc:http_request:rate5m"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "dc:http_request:rate5m"),
}, },
}, },
RHS: &NumberLiteral{10000}, RHS: &NumberLiteral{10000},
@ -1570,8 +1570,8 @@ var testStatement = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: "label1", Value: "value1"}, mustLabelMatcher(metric.Equal, "label1", "value1"),
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
Labels: nil, Labels: nil,
@ -1583,7 +1583,7 @@ var testStatement = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &NumberLiteral{10}, RHS: &NumberLiteral{10},
@ -1604,9 +1604,9 @@ var testStatement = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: "a", Value: "b"}, mustLabelMatcher(metric.Equal, "a", "b"),
mustLabelMatcher(metric.RegexMatch, "x", "y"), mustLabelMatcher(metric.RegexMatch, "x", "y"),
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
Labels: model.LabelSet{"x": "", "a": "z"}, Labels: model.LabelSet{"x": "", "a": "z"},
@ -1628,7 +1628,7 @@ var testStatement = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
RHS: &NumberLiteral{1}, RHS: &NumberLiteral{1},

10
storage/local/interface.go
View file

@ -60,11 +60,11 @@ type Querier interface {
NewPreloader() Preloader NewPreloader() Preloader
// MetricsForLabelMatchers returns the metrics from storage that satisfy // MetricsForLabelMatchers returns the metrics from storage that satisfy
// the given label matchers. At least one label matcher must be // the given label matchers. At least one label matcher must be
// specified that does not match the empty string. The times from and // specified that does not match the empty string, otherwise an empty
// through are hints for the storage to optimize the search. The storage // map is returned. The times from and through are hints for the storage
// MAY exclude metrics that have no samples in the specified interval // to optimize the search. The storage MAY exclude metrics that have no
// from the returned map. In doubt, specify model.Earliest for from and // samples in the specified interval from the returned map. In doubt,
// model.Latest for through. // specify model.Earliest for from and model.Latest for through.
MetricsForLabelMatchers(from, through model.Time, matchers ...*metric.LabelMatcher) map[model.Fingerprint]metric.Metric MetricsForLabelMatchers(from, through model.Time, matchers ...*metric.LabelMatcher) map[model.Fingerprint]metric.Metric
// LastSampleForFingerprint returns the last sample that has been // LastSampleForFingerprint returns the last sample that has been
// ingested for the provided fingerprint. If this instance of the // ingested for the provided fingerprint. If this instance of the

155
storage/local/storage.go
View file

@ -19,6 +19,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"math" "math"
"sort"
"sync" "sync"
"sync/atomic" "sync/atomic"
"time" "time"
@ -59,6 +60,19 @@ const (
// other words: if there are no chunks to persist, it doesn't help chunk // other words: if there are no chunks to persist, it doesn't help chunk
// eviction if we speed up persistence.) // eviction if we speed up persistence.)
factorMinChunksToPersist = 0.2 factorMinChunksToPersist = 0.2
// Threshold for when to stop using LabelMatchers to retrieve and
// intersect fingerprints. The rationale here is that looking up more
// fingerprints has diminishing returns if we already have narrowed down
// the possible fingerprints significantly. It is then easier to simply
// lookup the metrics for all the fingerprints and directly compare them
// to the matchers. Since a fingerprint lookup for an Equal matcher is
// much less expensive, there is a lower threshold for that case.
// TODO(beorn7): These numbers need to be tweaked, probably a bit lower.
// 5x higher numbers have resulted in slightly worse performance in a
// real-life production scenario.
fpEqualMatchThreshold = 1000
fpOtherMatchThreshold = 10000
) )
var ( var (
@ -445,27 +459,29 @@ func (s *MemorySeriesStorage) NewPreloader() Preloader {
} }
} }
// fingerprintsForLabelPairs returns the set of fingerprints that have the given labels. // fingerprintsForLabelPair returns the fingerprints with the given
// This does not work with empty label values. // LabelPair. If intersectWith is non-nil, the method will only return
func (s *MemorySeriesStorage) fingerprintsForLabelPairs(pairs ...model.LabelPair) map[model.Fingerprint]struct{} { // fingerprints that are also contained in intersectsWith. If mergeWith is
var result map[model.Fingerprint]struct{} // non-nil, the found fingerprints are added to the given map. The returned map
for _, pair := range pairs { // is the same as the given one.
intersection := map[model.Fingerprint]struct{}{} func (s *MemorySeriesStorage) fingerprintsForLabelPair(
fps := s.persistence.fingerprintsForLabelPair(pair) pair model.LabelPair,
if len(fps) == 0 { mergeWith map[model.Fingerprint]struct{},
return nil intersectWith map[model.Fingerprint]struct{},
} ) map[model.Fingerprint]struct{} {
for _, fp := range fps { if mergeWith == nil {
if _, ok := result[fp]; ok || result == nil { mergeWith = map[model.Fingerprint]struct{}{}
intersection[fp] = struct{}{}
}
}
if len(intersection) == 0 {
return nil
}
result = intersection
} }
return result for _, fp := range s.persistence.fingerprintsForLabelPair(pair) {
if intersectWith == nil {
mergeWith[fp] = struct{}{}
continue
}
if _, ok := intersectWith[fp]; ok {
mergeWith[fp] = struct{}{}
}
}
return mergeWith
} }
// MetricsForLabelMatchers implements Storage. // MetricsForLabelMatchers implements Storage.
@ -473,68 +489,75 @@ func (s *MemorySeriesStorage) MetricsForLabelMatchers(
from, through model.Time, from, through model.Time,
matchers ...*metric.LabelMatcher, matchers ...*metric.LabelMatcher,
) map[model.Fingerprint]metric.Metric { ) map[model.Fingerprint]metric.Metric {
sort.Sort(metric.LabelMatchers(matchers))
if len(matchers) == 0 || matchers[0].MatchesEmptyString() {
// No matchers at all or even the best matcher matches the empty string.
return nil
}
var ( var (
equals []model.LabelPair matcherIdx int
filters []*metric.LabelMatcher remainingFPs map[model.Fingerprint]struct{}
) )
for _, lm := range matchers {
if lm.Type == metric.Equal && lm.Value != "" { // Equal matchers.
equals = append(equals, model.LabelPair{ for ; matcherIdx < len(matchers) && (remainingFPs == nil || len(remainingFPs) > fpEqualMatchThreshold); matcherIdx++ {
Name: lm.Name, m := matchers[matcherIdx]
Value: lm.Value, if m.Type != metric.Equal || m.MatchesEmptyString() {
}) break
} else { }
filters = append(filters, lm) remainingFPs = s.fingerprintsForLabelPair(
model.LabelPair{
Name: m.Name,
Value: m.Value,
},
nil,
remainingFPs,
)
if len(remainingFPs) == 0 {
return nil
} }
} }
var resFPs map[model.Fingerprint]struct{} // Other matchers.
if len(equals) > 0 { for ; matcherIdx < len(matchers) && (remainingFPs == nil || len(remainingFPs) > fpOtherMatchThreshold); matcherIdx++ {
resFPs = s.fingerprintsForLabelPairs(equals...) m := matchers[matcherIdx]
} else { if m.MatchesEmptyString() {
// If we cannot make a preselection based on equality matchers, expanding the other matchers to labels break
// and intersecting their fingerprints is still likely to be the best choice. }
var remaining metric.LabelMatchers lvs := m.Filter(s.LabelValuesForLabelName(m.Name))
for _, matcher := range filters { if len(lvs) == 0 {
// Equal matches are all empty values. return nil
if matcher.Match("") { }
remaining = append(remaining, matcher) fps := map[model.Fingerprint]struct{}{}
continue for _, lv := range lvs {
} s.fingerprintsForLabelPair(
intersection := map[model.Fingerprint]struct{}{} model.LabelPair{
Name: m.Name,
matches := matcher.Filter(s.LabelValuesForLabelName(matcher.Name)) Value: lv,
if len(matches) == 0 { },
return nil fps,
} remainingFPs,
for _, v := range matches { )
fps := s.fingerprintsForLabelPairs(model.LabelPair{ }
Name: matcher.Name, remainingFPs = fps
Value: v, if len(remainingFPs) == 0 {
}) return nil
for fp := range fps {
if _, ok := resFPs[fp]; ok || resFPs == nil {
intersection[fp] = struct{}{}
}
}
}
resFPs = intersection
} }
// The intersected matchers no longer need to be compared against the actual metrics.
filters = remaining
} }
result := map[model.Fingerprint]metric.Metric{} result := map[model.Fingerprint]metric.Metric{}
for fp := range resFPs { for fp := range remainingFPs {
s.fpLocker.Lock(fp) s.fpLocker.Lock(fp)
if met, _, ok := s.metricForRange(fp, from, through); ok { if met, _, ok := s.metricForRange(fp, from, through); ok {
result[fp] = metric.Metric{Metric: met} result[fp] = metric.Metric{Metric: met}
} }
s.fpLocker.Unlock(fp) s.fpLocker.Unlock(fp)
} }
for _, matcher := range filters { for _, m := range matchers[matcherIdx:] {
for fp, met := range result { for fp, met := range result {
if !matcher.Match(met.Metric[matcher.Name]) { if !m.Match(met.Metric[m.Name]) {
delete(result, fp) delete(result, fp)
} }
} }

29
storage/local/storage_test.go
View file

@ -326,7 +326,10 @@ func TestFingerprintsForLabels(t *testing.T) {
} }
for _, mt := range matcherTests { for _, mt := range matcherTests {
resfps := storage.fingerprintsForLabelPairs(mt.pairs...) var resfps map[model.Fingerprint]struct{}
for _, pair := range mt.pairs {
resfps = storage.fingerprintsForLabelPair(pair, nil, resfps)
}
if len(mt.expected) != len(resfps) { if len(mt.expected) != len(resfps) {
t.Fatalf("expected %d matches for %q, found %d", len(mt.expected), mt.pairs, len(resfps)) t.Fatalf("expected %d matches for %q, found %d", len(mt.expected), mt.pairs, len(resfps))
} }
@ -467,7 +470,9 @@ func TestRetentionCutoff(t *testing.T) {
s.WaitForIndexing() s.WaitForIndexing()
var fp model.Fingerprint var fp model.Fingerprint
for f := range s.fingerprintsForLabelPairs(model.LabelPair{Name: "job", Value: "test"}) { for f := range s.fingerprintsForLabelPair(model.LabelPair{
Name: "job", Value: "test",
}, nil, nil) {
fp = f fp = f
break break
} }
@ -539,7 +544,9 @@ func TestDropMetrics(t *testing.T) {
s.persistence.archiveMetric(fpToBeArchived, m3, 0, insertStart.Add(time.Duration(N-1)*time.Millisecond)) s.persistence.archiveMetric(fpToBeArchived, m3, 0, insertStart.Add(time.Duration(N-1)*time.Millisecond))
s.fpLocker.Unlock(fpToBeArchived) s.fpLocker.Unlock(fpToBeArchived)
fps := s.fingerprintsForLabelPairs(model.LabelPair{Name: model.MetricNameLabel, Value: "test"}) fps := s.fingerprintsForLabelPair(model.LabelPair{
Name: model.MetricNameLabel, Value: "test",
}, nil, nil)
if len(fps) != 3 { if len(fps) != 3 {
t.Errorf("unexpected number of fingerprints: %d", len(fps)) t.Errorf("unexpected number of fingerprints: %d", len(fps))
} }
@ -549,9 +556,9 @@ func TestDropMetrics(t *testing.T) {
s.DropMetricsForFingerprints(fpList[0]) s.DropMetricsForFingerprints(fpList[0])
s.WaitForIndexing() s.WaitForIndexing()
fps2 := s.fingerprintsForLabelPairs(model.LabelPair{ fps2 := s.fingerprintsForLabelPair(model.LabelPair{
Name: model.MetricNameLabel, Value: "test", Name: model.MetricNameLabel, Value: "test",
}) }, nil, nil)
if len(fps2) != 2 { if len(fps2) != 2 {
t.Errorf("unexpected number of fingerprints: %d", len(fps2)) t.Errorf("unexpected number of fingerprints: %d", len(fps2))
} }
@ -576,9 +583,9 @@ func TestDropMetrics(t *testing.T) {
s.DropMetricsForFingerprints(fpList...) s.DropMetricsForFingerprints(fpList...)
s.WaitForIndexing() s.WaitForIndexing()
fps3 := s.fingerprintsForLabelPairs(model.LabelPair{ fps3 := s.fingerprintsForLabelPair(model.LabelPair{
Name: model.MetricNameLabel, Value: "test", Name: model.MetricNameLabel, Value: "test",
}) }, nil, nil)
if len(fps3) != 0 { if len(fps3) != 0 {
t.Errorf("unexpected number of fingerprints: %d", len(fps3)) t.Errorf("unexpected number of fingerprints: %d", len(fps3))
} }
@ -658,7 +665,9 @@ func TestQuarantineMetric(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
fps := s.fingerprintsForLabelPairs(model.LabelPair{Name: model.MetricNameLabel, Value: "test"}) fps := s.fingerprintsForLabelPair(model.LabelPair{
Name: model.MetricNameLabel, Value: "test",
}, nil, nil)
if len(fps) != 3 { if len(fps) != 3 {
t.Errorf("unexpected number of fingerprints: %d", len(fps)) t.Errorf("unexpected number of fingerprints: %d", len(fps))
} }
@ -670,9 +679,9 @@ func TestQuarantineMetric(t *testing.T) {
time.Sleep(time.Second) // Give time to quarantine. TODO(beorn7): Find a better way to wait. time.Sleep(time.Second) // Give time to quarantine. TODO(beorn7): Find a better way to wait.
s.WaitForIndexing() s.WaitForIndexing()
fps2 := s.fingerprintsForLabelPairs(model.LabelPair{ fps2 := s.fingerprintsForLabelPair(model.LabelPair{
Name: model.MetricNameLabel, Value: "test", Name: model.MetricNameLabel, Value: "test",
}) }, nil, nil)
if len(fps2) != 2 { if len(fps2) != 2 {
t.Errorf("unexpected number of fingerprints: %d", len(fps2)) t.Errorf("unexpected number of fingerprints: %d", len(fps2))
} }

100
storage/metric/matcher.go
View file

@ -16,6 +16,7 @@ package metric
import ( import (
"fmt" "fmt"
"regexp" "regexp"
"strings"
"github.com/prometheus/common/model" "github.com/prometheus/common/model"
) )
@ -44,15 +45,23 @@ func (m MatchType) String() string {
panic("unknown match type") panic("unknown match type")
} }
// LabelMatchers is a slice of LabelMatcher objects. // LabelMatchers is a slice of LabelMatcher objects. By implementing the
// sort.Interface, it is sortable by cardinality score, i.e. after sorting, the
// LabelMatcher that is expected to yield the fewest matches is first in the
// slice, and LabelMatchers that match the empty string are last.
type LabelMatchers []*LabelMatcher type LabelMatchers []*LabelMatcher
// LabelMatcher models the matching of a label. func (lms LabelMatchers) Len() int { return len(lms) }
func (lms LabelMatchers) Swap(i, j int) { lms[i], lms[j] = lms[j], lms[i] }
func (lms LabelMatchers) Less(i, j int) bool { return lms[i].score < lms[j].score }
// LabelMatcher models the matching of a label. Create with NewLabelMatcher.
type LabelMatcher struct { type LabelMatcher struct {
Type MatchType Type MatchType
Name model.LabelName Name model.LabelName
Value model.LabelValue Value model.LabelValue
re *regexp.Regexp re *regexp.Regexp
score float64 // Cardinality score, between 0 and 1, 0 is lowest cardinality.
} }
// NewLabelMatcher returns a LabelMatcher object ready to use. // NewLabelMatcher returns a LabelMatcher object ready to use.
@ -69,9 +78,96 @@ func NewLabelMatcher(matchType MatchType, name model.LabelName, value model.Labe
} }
m.re = re m.re = re
} }
m.calculateScore()
return m, nil return m, nil
} }
// calculateScore is a helper method only called in the constructor. It
// calculates the cardinality score upfront, so that sorting by it is faster and
// doesn't change internal state of the matcher.
//
// The score is based on a pretty bad but still quite helpful heuristics for
// now. Note that this is an interim solution until the work in progress to
// properly intersect matchers is complete. We intend to not invest any further
// effort into tweaking the score calculation, as this could easily devolve into
// a rabbit hole.
//
// The heuristics works along the following lines:
//
// - A matcher that is known to match nothing would have a score of 0. (This
// case doesn't happen in the scope of this method.)
//
// - A matcher that matches the empty string has a score of 1.
//
// - Equal matchers have a score <= 0.5. The order in score for other matchers
// are RegexMatch, RegexNoMatch, NotEqual.
//
// - There are a number of score adjustments for known "magic" parts, like
// instance labels, metric names containing a colon (which are probably
// recording rules) and such.
//
// - On top, there is a tiny adjustment for the length of the matcher, following
// the blunt expectation that a long label name and/or value is more specific
// and will therefore have a lower cardinality.
//
// To reiterate on the above: PLEASE RESIST THE TEMPTATION TO TWEAK THIS
// METHOD. IT IS "MAGIC" ENOUGH ALREADY AND WILL GO AWAY WITH THE UPCOMING MORE
// POWERFUL INDEXING.
func (m *LabelMatcher) calculateScore() {
if m.Match("") {
m.score = 1
return
}
// lengthCorrection is between 0 (for length 0) and 0.1 (for length +Inf).
lengthCorrection := 0.1 * (1 - 1/float64(len(m.Name)+len(m.Value)+1))
switch m.Type {
case Equal:
m.score = 0.3 - lengthCorrection
case RegexMatch:
m.score = 0.6 - lengthCorrection
case RegexNoMatch:
m.score = 0.8 + lengthCorrection
case NotEqual:
m.score = 0.9 + lengthCorrection
}
if m.Type != Equal {
// Don't bother anymore in this case.
return
}
switch m.Name {
case model.InstanceLabel:
// Matches only metrics from a single instance, which clearly
// limits the damage.
m.score -= 0.2
case model.JobLabel:
// The usual case is a relatively low number of jobs with many
// metrics each.
m.score += 0.1
case model.BucketLabel, model.QuantileLabel:
// Magic labels for buckets and quantiles will match copiously.
m.score += 0.2
case model.MetricNameLabel:
if strings.Contains(string(m.Value), ":") {
// Probably a recording rule with limited cardinality.
m.score -= 0.1
return
}
if m.Value == "up" || m.Value == "scrape_duration_seconds" {
// Synthetic metrics which are contained in every scrape
// exactly once. There might be less frequent metric
// names, but the worst case is limited here, so give it
// a bump.
m.score -= 0.05
return
}
}
}
// MatchesEmptyString returns true if the LabelMatcher matches the empty string.
func (m *LabelMatcher) MatchesEmptyString() bool {
return m.score >= 1
}
func (m *LabelMatcher) String() string { func (m *LabelMatcher) String() string {
return fmt.Sprintf("%s%s%q", m.Name, m.Type, m.Value) return fmt.Sprintf("%s%s%q", m.Name, m.Type, m.Value)
} }

41
storage/metric/matcher_test.go
View file

@ -14,7 +14,12 @@
package metric package metric
import ( import (
"math/rand"
"reflect"
"sort"
"testing" "testing"
"github.com/prometheus/common/model"
) )
func TestAnchoredMatcher(t *testing.T) { func TestAnchoredMatcher(t *testing.T) {
@ -30,3 +35,39 @@ func TestAnchoredMatcher(t *testing.T) {
t.Errorf("Unexpected match for %q", "fooo") t.Errorf("Unexpected match for %q", "fooo")
} }
} }
func TestLabelMatchersSort(t *testing.T) {
// Line up Matchers in expected order:
want := LabelMatchers{
mustNewLabelMatcher(Equal, model.InstanceLabel, "not empty"),
mustNewLabelMatcher(Equal, model.MetricNameLabel, "a:recording:rule"),
mustNewLabelMatcher(Equal, model.MetricNameLabel, "up"),
mustNewLabelMatcher(Equal, "nothing_special but much longer", "not empty"),
mustNewLabelMatcher(Equal, "nothing_special", "not empty but longer"),
mustNewLabelMatcher(Equal, "nothing_special", "not empty"),
mustNewLabelMatcher(Equal, model.JobLabel, "not empty"),
mustNewLabelMatcher(Equal, model.BucketLabel, "not empty"),
mustNewLabelMatcher(RegexMatch, "irrelevant", "does not match empty string and is longer"),
mustNewLabelMatcher(RegexMatch, "irrelevant", "does not match empty string"),
mustNewLabelMatcher(RegexNoMatch, "irrelevant", "(matches empty string)?"),
mustNewLabelMatcher(RegexNoMatch, "irrelevant", "(matches empty string with a longer expression)?"),
mustNewLabelMatcher(NotEqual, "irrelevant", ""),
mustNewLabelMatcher(Equal, "irrelevant", ""),
}
got := make(LabelMatchers, len(want))
for i, j := range rand.Perm(len(want)) {
got[i] = want[j]
}
sort.Sort(got)
if !reflect.DeepEqual(want, got) {
t.Errorf("unexpected sorting of matchers, got %v, want %v", got, want)
}
}
func mustNewLabelMatcher(mt MatchType, name model.LabelName, val model.LabelValue) *LabelMatcher {
m, err := NewLabelMatcher(mt, name, val)
if err != nil {
panic(err)
}
return m
}