Merge pull request #1774 from prometheus/beorn7/index

storage: improve index lookups
This commit is contained in:
Björn Rabenstein 2016-07-20 17:38:04 +02:00 committed by GitHub
commit 5fab430e73
7 changed files with 349 additions and 187 deletions

View file

@ -953,11 +953,11 @@ func (p *parser) vectorSelector(name string) *VectorSelector {
}
}
// Set name label matching.
matchers = append(matchers, &metric.LabelMatcher{
Type: metric.Equal,
Name: model.MetricNameLabel,
Value: model.LabelValue(name),
})
m, err := metric.NewLabelMatcher(metric.Equal, model.MetricNameLabel, model.LabelValue(name))
if err != nil {
panic(err) // Must not happen with metric.Equal.
}
matchers = append(matchers, m)
}
if len(matchers) == 0 {
@ -967,14 +967,7 @@ func (p *parser) vectorSelector(name string) *VectorSelector {
// implicit selection of all metrics (e.g. by a typo).
notEmpty := false
for _, lm := range matchers {
// Matching changes the inner state of the regex and causes reflect.DeepEqual
// to return false, which break tests.
// Thus, we create a new label matcher for this testing.
lm, err := metric.NewLabelMatcher(lm.Type, lm.Name, lm.Value)
if err != nil {
p.error(err)
}
if !lm.Match("") {
if !lm.MatchesEmptyString() {
notEmpty = true
break
}

View file

@ -144,7 +144,7 @@ var testExpr = []struct {
Expr: &VectorSelector{
Name: "some_metric",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
},
},
},
@ -154,7 +154,7 @@ var testExpr = []struct {
Expr: &VectorSelector{
Name: "some_metric",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
},
},
},
@ -263,13 +263,13 @@ var testExpr = []struct {
LHS: &VectorSelector{
Name: "foo",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
},
},
RHS: &VectorSelector{
Name: "bar",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
},
},
VectorMatching: &VectorMatching{Card: CardOneToOne},
@ -281,7 +281,7 @@ var testExpr = []struct {
LHS: &VectorSelector{
Name: "foo",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
},
},
RHS: &NumberLiteral{1},
@ -293,7 +293,7 @@ var testExpr = []struct {
LHS: &VectorSelector{
Name: "foo",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
},
},
RHS: &NumberLiteral{1},
@ -307,7 +307,7 @@ var testExpr = []struct {
RHS: &VectorSelector{
Name: "bar",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
},
},
},
@ -318,13 +318,13 @@ var testExpr = []struct {
LHS: &VectorSelector{
Name: "foo",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
},
},
RHS: &VectorSelector{
Name: "bar",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
},
},
VectorMatching: &VectorMatching{Card: CardManyToMany},
@ -336,13 +336,13 @@ var testExpr = []struct {
LHS: &VectorSelector{
Name: "foo",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
},
},
RHS: &VectorSelector{
Name: "bar",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
},
},
VectorMatching: &VectorMatching{Card: CardManyToMany},
@ -354,13 +354,13 @@ var testExpr = []struct {
LHS: &VectorSelector{
Name: "foo",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
},
},
RHS: &VectorSelector{
Name: "bar",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
},
},
VectorMatching: &VectorMatching{Card: CardManyToMany},
@ -375,13 +375,13 @@ var testExpr = []struct {
LHS: &VectorSelector{
Name: "foo",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
},
},
RHS: &VectorSelector{
Name: "bar",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
},
},
VectorMatching: &VectorMatching{Card: CardOneToOne},
@ -391,13 +391,13 @@ var testExpr = []struct {
LHS: &VectorSelector{
Name: "bla",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bla"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bla"),
},
},
RHS: &VectorSelector{
Name: "blub",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "blub"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "blub"),
},
},
VectorMatching: &VectorMatching{Card: CardManyToMany},
@ -416,13 +416,13 @@ var testExpr = []struct {
LHS: &VectorSelector{
Name: "foo",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
},
},
RHS: &VectorSelector{
Name: "bar",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
},
},
VectorMatching: &VectorMatching{Card: CardManyToMany},
@ -430,7 +430,7 @@ var testExpr = []struct {
RHS: &VectorSelector{
Name: "baz",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "baz"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "baz"),
},
},
VectorMatching: &VectorMatching{Card: CardManyToMany},
@ -438,7 +438,7 @@ var testExpr = []struct {
RHS: &VectorSelector{
Name: "qux",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "qux"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "qux"),
},
},
VectorMatching: &VectorMatching{Card: CardManyToMany},
@ -451,7 +451,7 @@ var testExpr = []struct {
LHS: &VectorSelector{
Name: "bar",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
},
},
RHS: &BinaryExpr{
@ -459,13 +459,13 @@ var testExpr = []struct {
LHS: &VectorSelector{
Name: "bla",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bla"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bla"),
},
},
RHS: &VectorSelector{
Name: "blub",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "blub"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "blub"),
},
},
VectorMatching: &VectorMatching{
@ -488,13 +488,13 @@ var testExpr = []struct {
LHS: &VectorSelector{
Name: "foo",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
},
},
RHS: &VectorSelector{
Name: "bar",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
},
},
VectorMatching: &VectorMatching{
@ -510,13 +510,13 @@ var testExpr = []struct {
LHS: &VectorSelector{
Name: "foo",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
},
},
RHS: &VectorSelector{
Name: "bar",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
},
},
VectorMatching: &VectorMatching{
@ -532,13 +532,13 @@ var testExpr = []struct {
LHS: &VectorSelector{
Name: "foo",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
},
},
RHS: &VectorSelector{
Name: "bar",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
},
},
VectorMatching: &VectorMatching{
@ -554,13 +554,13 @@ var testExpr = []struct {
LHS: &VectorSelector{
Name: "foo",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
},
},
RHS: &VectorSelector{
Name: "bar",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
},
},
VectorMatching: &VectorMatching{
@ -576,13 +576,13 @@ var testExpr = []struct {
LHS: &VectorSelector{
Name: "foo",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
},
},
RHS: &VectorSelector{
Name: "bar",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
},
},
VectorMatching: &VectorMatching{
@ -597,13 +597,13 @@ var testExpr = []struct {
LHS: &VectorSelector{
Name: "foo",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
},
},
RHS: &VectorSelector{
Name: "bar",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
},
},
VectorMatching: &VectorMatching{
@ -618,13 +618,13 @@ var testExpr = []struct {
LHS: &VectorSelector{
Name: "foo",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
},
},
RHS: &VectorSelector{
Name: "baz",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "baz"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "baz"),
},
},
VectorMatching: &VectorMatching{
@ -640,13 +640,13 @@ var testExpr = []struct {
LHS: &VectorSelector{
Name: "foo",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
},
},
RHS: &VectorSelector{
Name: "bar",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
},
},
VectorMatching: &VectorMatching{
@ -663,13 +663,13 @@ var testExpr = []struct {
LHS: &VectorSelector{
Name: "foo",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
},
},
RHS: &VectorSelector{
Name: "bar",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
},
},
VectorMatching: &VectorMatching{
@ -685,13 +685,13 @@ var testExpr = []struct {
LHS: &VectorSelector{
Name: "foo",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
},
},
RHS: &VectorSelector{
Name: "bar",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
},
},
VectorMatching: &VectorMatching{
@ -707,13 +707,13 @@ var testExpr = []struct {
LHS: &VectorSelector{
Name: "foo",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
},
},
RHS: &VectorSelector{
Name: "bar",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
},
},
VectorMatching: &VectorMatching{
@ -730,13 +730,13 @@ var testExpr = []struct {
LHS: &VectorSelector{
Name: "foo",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
},
},
RHS: &VectorSelector{
Name: "bar",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
},
},
VectorMatching: &VectorMatching{
@ -825,7 +825,7 @@ var testExpr = []struct {
Name: "foo",
Offset: 0,
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
},
},
}, {
@ -834,7 +834,7 @@ var testExpr = []struct {
Name: "foo",
Offset: 5 * time.Minute,
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
},
},
}, {
@ -843,8 +843,8 @@ var testExpr = []struct {
Name: "foo:bar",
Offset: 0,
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: "a", Value: "bc"},
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo:bar"},
mustLabelMatcher(metric.Equal, "a", "bc"),
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo:bar"),
},
},
}, {
@ -853,8 +853,8 @@ var testExpr = []struct {
Name: "foo",
Offset: 0,
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: "NaN", Value: "bc"},
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
mustLabelMatcher(metric.Equal, "NaN", "bc"),
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
},
},
}, {
@ -863,11 +863,11 @@ var testExpr = []struct {
Name: "foo",
Offset: 0,
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: "a", Value: "b"},
{Type: metric.NotEqual, Name: "foo", Value: "bar"},
mustLabelMatcher(metric.Equal, "a", "b"),
mustLabelMatcher(metric.NotEqual, "foo", "bar"),
mustLabelMatcher(metric.RegexMatch, "test", "test"),
mustLabelMatcher(metric.RegexNoMatch, "bar", "baz"),
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
},
},
}, {
@ -949,7 +949,7 @@ var testExpr = []struct {
Offset: 0,
Range: 5 * time.Second,
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "test"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "test"),
},
},
}, {
@ -959,7 +959,7 @@ var testExpr = []struct {
Offset: 0,
Range: 5 * time.Minute,
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "test"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "test"),
},
},
}, {
@ -969,7 +969,7 @@ var testExpr = []struct {
Offset: 5 * time.Minute,
Range: 5 * time.Hour,
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "test"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "test"),
},
},
}, {
@ -979,7 +979,7 @@ var testExpr = []struct {
Offset: 10 * time.Second,
Range: 5 * 24 * time.Hour,
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "test"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "test"),
},
},
}, {
@ -989,7 +989,7 @@ var testExpr = []struct {
Offset: 14 * 24 * time.Hour,
Range: 5 * 7 * 24 * time.Hour,
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "test"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "test"),
},
},
}, {
@ -999,8 +999,8 @@ var testExpr = []struct {
Offset: 3 * 24 * time.Hour,
Range: 5 * 365 * 24 * time.Hour,
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: "a", Value: "b"},
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "test"},
mustLabelMatcher(metric.Equal, "a", "b"),
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "test"),
},
},
}, {
@ -1059,7 +1059,7 @@ var testExpr = []struct {
Expr: &VectorSelector{
Name: "some_metric",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
},
},
Grouping: model.LabelNames{"foo"},
@ -1072,7 +1072,7 @@ var testExpr = []struct {
Expr: &VectorSelector{
Name: "some_metric",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
},
},
Grouping: model.LabelNames{"foo"},
@ -1085,7 +1085,7 @@ var testExpr = []struct {
Expr: &VectorSelector{
Name: "some_metric",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
},
},
Grouping: model.LabelNames{"foo", "bar"},
@ -1097,7 +1097,7 @@ var testExpr = []struct {
Expr: &VectorSelector{
Name: "some_metric",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
},
},
Grouping: model.LabelNames{"foo"},
@ -1109,7 +1109,7 @@ var testExpr = []struct {
Expr: &VectorSelector{
Name: "some_metric",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
},
},
Grouping: model.LabelNames{"foo"},
@ -1122,7 +1122,7 @@ var testExpr = []struct {
Expr: &VectorSelector{
Name: "some_metric",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
},
},
Grouping: model.LabelNames{"foo"},
@ -1135,7 +1135,7 @@ var testExpr = []struct {
Expr: &VectorSelector{
Name: "some_metric",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
},
},
Grouping: model.LabelNames{"foo"},
@ -1148,7 +1148,7 @@ var testExpr = []struct {
Expr: &VectorSelector{
Name: "some_metric",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
},
},
Grouping: model.LabelNames{"foo"},
@ -1161,7 +1161,7 @@ var testExpr = []struct {
Expr: &VectorSelector{
Name: "some_metric",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
},
},
Grouping: model.LabelNames{"foo"},
@ -1173,7 +1173,7 @@ var testExpr = []struct {
Expr: &VectorSelector{
Name: "some_metric",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
},
},
},
@ -1184,7 +1184,7 @@ var testExpr = []struct {
Expr: &VectorSelector{
Name: "some_metric",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
},
},
Grouping: model.LabelNames{"foo"},
@ -1196,7 +1196,7 @@ var testExpr = []struct {
Expr: &VectorSelector{
Name: "some_metric",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
},
},
Grouping: model.LabelNames{},
@ -1208,7 +1208,7 @@ var testExpr = []struct {
Expr: &VectorSelector{
Name: "some_metric",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
},
},
Param: &NumberLiteral{5},
@ -1220,7 +1220,7 @@ var testExpr = []struct {
Expr: &VectorSelector{
Name: "some_metric",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
},
},
Param: &StringLiteral{"value"},
@ -1288,8 +1288,8 @@ var testExpr = []struct {
&VectorSelector{
Name: "some_metric",
LabelMatchers: metric.LabelMatchers{
{Type: metric.NotEqual, Name: "foo", Value: "bar"},
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
mustLabelMatcher(metric.NotEqual, "foo", "bar"),
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
},
},
},
@ -1302,7 +1302,7 @@ var testExpr = []struct {
&MatrixSelector{
Name: "some_metric",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
},
Range: 5 * time.Minute,
},
@ -1316,7 +1316,7 @@ var testExpr = []struct {
&VectorSelector{
Name: "some_metric",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
},
},
},
@ -1329,7 +1329,7 @@ var testExpr = []struct {
&VectorSelector{
Name: "some_metric",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
},
},
&NumberLiteral{5},
@ -1537,7 +1537,7 @@ var testStatement = []struct {
&MatrixSelector{
Name: "http_request_count",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "http_request_count"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "http_request_count"),
},
Range: 5 * time.Minute,
},
@ -1553,7 +1553,7 @@ var testStatement = []struct {
LHS: &VectorSelector{
Name: "dc:http_request:rate5m",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "dc:http_request:rate5m"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "dc:http_request:rate5m"),
},
},
RHS: &NumberLiteral{10000},
@ -1570,8 +1570,8 @@ var testStatement = []struct {
Expr: &VectorSelector{
Name: "bar",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: "label1", Value: "value1"},
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
mustLabelMatcher(metric.Equal, "label1", "value1"),
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
},
},
Labels: nil,
@ -1583,7 +1583,7 @@ var testStatement = []struct {
LHS: &VectorSelector{
Name: "foo",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
},
},
RHS: &NumberLiteral{10},
@ -1604,9 +1604,9 @@ var testStatement = []struct {
Expr: &VectorSelector{
Name: "bar",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: "a", Value: "b"},
mustLabelMatcher(metric.Equal, "a", "b"),
mustLabelMatcher(metric.RegexMatch, "x", "y"),
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
},
},
Labels: model.LabelSet{"x": "", "a": "z"},
@ -1628,7 +1628,7 @@ var testStatement = []struct {
LHS: &VectorSelector{
Name: "some_metric",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
},
},
RHS: &NumberLiteral{1},

View file

@ -60,11 +60,11 @@ type Querier interface {
NewPreloader() Preloader
// MetricsForLabelMatchers returns the metrics from storage that satisfy
// the given label matchers. At least one label matcher must be
// specified that does not match the empty string. The times from and
// through are hints for the storage to optimize the search. The storage
// MAY exclude metrics that have no samples in the specified interval
// from the returned map. In doubt, specify model.Earliest for from and
// model.Latest for through.
// specified that does not match the empty string, otherwise an empty
// map is returned. The times from and through are hints for the storage
// to optimize the search. The storage MAY exclude metrics that have no
// samples in the specified interval from the returned map. In doubt,
// specify model.Earliest for from and model.Latest for through.
MetricsForLabelMatchers(from, through model.Time, matchers ...*metric.LabelMatcher) map[model.Fingerprint]metric.Metric
// LastSampleForFingerprint returns the last sample that has been
// ingested for the provided fingerprint. If this instance of the

View file

@ -19,6 +19,7 @@ import (
"errors"
"fmt"
"math"
"sort"
"sync"
"sync/atomic"
"time"
@ -59,6 +60,19 @@ const (
// other words: if there are no chunks to persist, it doesn't help chunk
// eviction if we speed up persistence.)
factorMinChunksToPersist = 0.2
// Threshold for when to stop using LabelMatchers to retrieve and
// intersect fingerprints. The rationale here is that looking up more
// fingerprints has diminishing returns if we already have narrowed down
// the possible fingerprints significantly. It is then easier to simply
// lookup the metrics for all the fingerprints and directly compare them
// to the matchers. Since a fingerprint lookup for an Equal matcher is
// much less expensive, there is a lower threshold for that case.
// TODO(beorn7): These numbers need to be tweaked, probably a bit lower.
// 5x higher numbers have resulted in slightly worse performance in a
// real-life production scenario.
fpEqualMatchThreshold = 1000
fpOtherMatchThreshold = 10000
)
var (
@ -445,27 +459,29 @@ func (s *MemorySeriesStorage) NewPreloader() Preloader {
}
}
// fingerprintsForLabelPairs returns the set of fingerprints that have the given labels.
// This does not work with empty label values.
func (s *MemorySeriesStorage) fingerprintsForLabelPairs(pairs ...model.LabelPair) map[model.Fingerprint]struct{} {
var result map[model.Fingerprint]struct{}
for _, pair := range pairs {
intersection := map[model.Fingerprint]struct{}{}
fps := s.persistence.fingerprintsForLabelPair(pair)
if len(fps) == 0 {
return nil
// fingerprintsForLabelPair returns the fingerprints with the given
// LabelPair. If intersectWith is non-nil, the method will only return
// fingerprints that are also contained in intersectsWith. If mergeWith is
// non-nil, the found fingerprints are added to the given map. The returned map
// is the same as the given one.
func (s *MemorySeriesStorage) fingerprintsForLabelPair(
pair model.LabelPair,
mergeWith map[model.Fingerprint]struct{},
intersectWith map[model.Fingerprint]struct{},
) map[model.Fingerprint]struct{} {
if mergeWith == nil {
mergeWith = map[model.Fingerprint]struct{}{}
}
for _, fp := range fps {
if _, ok := result[fp]; ok || result == nil {
intersection[fp] = struct{}{}
for _, fp := range s.persistence.fingerprintsForLabelPair(pair) {
if intersectWith == nil {
mergeWith[fp] = struct{}{}
continue
}
if _, ok := intersectWith[fp]; ok {
mergeWith[fp] = struct{}{}
}
}
if len(intersection) == 0 {
return nil
}
result = intersection
}
return result
return mergeWith
}
// MetricsForLabelMatchers implements Storage.
@ -473,68 +489,75 @@ func (s *MemorySeriesStorage) MetricsForLabelMatchers(
from, through model.Time,
matchers ...*metric.LabelMatcher,
) map[model.Fingerprint]metric.Metric {
var (
equals []model.LabelPair
filters []*metric.LabelMatcher
)
for _, lm := range matchers {
if lm.Type == metric.Equal && lm.Value != "" {
equals = append(equals, model.LabelPair{
Name: lm.Name,
Value: lm.Value,
})
} else {
filters = append(filters, lm)
}
}
sort.Sort(metric.LabelMatchers(matchers))
var resFPs map[model.Fingerprint]struct{}
if len(equals) > 0 {
resFPs = s.fingerprintsForLabelPairs(equals...)
} else {
// If we cannot make a preselection based on equality matchers, expanding the other matchers to labels
// and intersecting their fingerprints is still likely to be the best choice.
var remaining metric.LabelMatchers
for _, matcher := range filters {
// Equal matches are all empty values.
if matcher.Match("") {
remaining = append(remaining, matcher)
continue
}
intersection := map[model.Fingerprint]struct{}{}
matches := matcher.Filter(s.LabelValuesForLabelName(matcher.Name))
if len(matches) == 0 {
if len(matchers) == 0 || matchers[0].MatchesEmptyString() {
// No matchers at all or even the best matcher matches the empty string.
return nil
}
for _, v := range matches {
fps := s.fingerprintsForLabelPairs(model.LabelPair{
Name: matcher.Name,
Value: v,
})
for fp := range fps {
if _, ok := resFPs[fp]; ok || resFPs == nil {
intersection[fp] = struct{}{}
var (
matcherIdx int
remainingFPs map[model.Fingerprint]struct{}
)
// Equal matchers.
for ; matcherIdx < len(matchers) && (remainingFPs == nil || len(remainingFPs) > fpEqualMatchThreshold); matcherIdx++ {
m := matchers[matcherIdx]
if m.Type != metric.Equal || m.MatchesEmptyString() {
break
}
remainingFPs = s.fingerprintsForLabelPair(
model.LabelPair{
Name: m.Name,
Value: m.Value,
},
nil,
remainingFPs,
)
if len(remainingFPs) == 0 {
return nil
}
}
// Other matchers.
for ; matcherIdx < len(matchers) && (remainingFPs == nil || len(remainingFPs) > fpOtherMatchThreshold); matcherIdx++ {
m := matchers[matcherIdx]
if m.MatchesEmptyString() {
break
}
resFPs = intersection
lvs := m.Filter(s.LabelValuesForLabelName(m.Name))
if len(lvs) == 0 {
return nil
}
fps := map[model.Fingerprint]struct{}{}
for _, lv := range lvs {
s.fingerprintsForLabelPair(
model.LabelPair{
Name: m.Name,
Value: lv,
},
fps,
remainingFPs,
)
}
remainingFPs = fps
if len(remainingFPs) == 0 {
return nil
}
// The intersected matchers no longer need to be compared against the actual metrics.
filters = remaining
}
result := map[model.Fingerprint]metric.Metric{}
for fp := range resFPs {
for fp := range remainingFPs {
s.fpLocker.Lock(fp)
if met, _, ok := s.metricForRange(fp, from, through); ok {
result[fp] = metric.Metric{Metric: met}
}
s.fpLocker.Unlock(fp)
}
for _, matcher := range filters {
for _, m := range matchers[matcherIdx:] {
for fp, met := range result {
if !matcher.Match(met.Metric[matcher.Name]) {
if !m.Match(met.Metric[m.Name]) {
delete(result, fp)
}
}

View file

@ -326,7 +326,10 @@ func TestFingerprintsForLabels(t *testing.T) {
}
for _, mt := range matcherTests {
resfps := storage.fingerprintsForLabelPairs(mt.pairs...)
var resfps map[model.Fingerprint]struct{}
for _, pair := range mt.pairs {
resfps = storage.fingerprintsForLabelPair(pair, nil, resfps)
}
if len(mt.expected) != len(resfps) {
t.Fatalf("expected %d matches for %q, found %d", len(mt.expected), mt.pairs, len(resfps))
}
@ -467,7 +470,9 @@ func TestRetentionCutoff(t *testing.T) {
s.WaitForIndexing()
var fp model.Fingerprint
for f := range s.fingerprintsForLabelPairs(model.LabelPair{Name: "job", Value: "test"}) {
for f := range s.fingerprintsForLabelPair(model.LabelPair{
Name: "job", Value: "test",
}, nil, nil) {
fp = f
break
}
@ -539,7 +544,9 @@ func TestDropMetrics(t *testing.T) {
s.persistence.archiveMetric(fpToBeArchived, m3, 0, insertStart.Add(time.Duration(N-1)*time.Millisecond))
s.fpLocker.Unlock(fpToBeArchived)
fps := s.fingerprintsForLabelPairs(model.LabelPair{Name: model.MetricNameLabel, Value: "test"})
fps := s.fingerprintsForLabelPair(model.LabelPair{
Name: model.MetricNameLabel, Value: "test",
}, nil, nil)
if len(fps) != 3 {
t.Errorf("unexpected number of fingerprints: %d", len(fps))
}
@ -549,9 +556,9 @@ func TestDropMetrics(t *testing.T) {
s.DropMetricsForFingerprints(fpList[0])
s.WaitForIndexing()
fps2 := s.fingerprintsForLabelPairs(model.LabelPair{
fps2 := s.fingerprintsForLabelPair(model.LabelPair{
Name: model.MetricNameLabel, Value: "test",
})
}, nil, nil)
if len(fps2) != 2 {
t.Errorf("unexpected number of fingerprints: %d", len(fps2))
}
@ -576,9 +583,9 @@ func TestDropMetrics(t *testing.T) {
s.DropMetricsForFingerprints(fpList...)
s.WaitForIndexing()
fps3 := s.fingerprintsForLabelPairs(model.LabelPair{
fps3 := s.fingerprintsForLabelPair(model.LabelPair{
Name: model.MetricNameLabel, Value: "test",
})
}, nil, nil)
if len(fps3) != 0 {
t.Errorf("unexpected number of fingerprints: %d", len(fps3))
}
@ -658,7 +665,9 @@ func TestQuarantineMetric(t *testing.T) {
t.Fatal(err)
}
fps := s.fingerprintsForLabelPairs(model.LabelPair{Name: model.MetricNameLabel, Value: "test"})
fps := s.fingerprintsForLabelPair(model.LabelPair{
Name: model.MetricNameLabel, Value: "test",
}, nil, nil)
if len(fps) != 3 {
t.Errorf("unexpected number of fingerprints: %d", len(fps))
}
@ -670,9 +679,9 @@ func TestQuarantineMetric(t *testing.T) {
time.Sleep(time.Second) // Give time to quarantine. TODO(beorn7): Find a better way to wait.
s.WaitForIndexing()
fps2 := s.fingerprintsForLabelPairs(model.LabelPair{
fps2 := s.fingerprintsForLabelPair(model.LabelPair{
Name: model.MetricNameLabel, Value: "test",
})
}, nil, nil)
if len(fps2) != 2 {
t.Errorf("unexpected number of fingerprints: %d", len(fps2))
}

View file

@ -16,6 +16,7 @@ package metric
import (
"fmt"
"regexp"
"strings"
"github.com/prometheus/common/model"
)
@ -44,15 +45,23 @@ func (m MatchType) String() string {
panic("unknown match type")
}
// LabelMatchers is a slice of LabelMatcher objects.
// LabelMatchers is a slice of LabelMatcher objects. By implementing the
// sort.Interface, it is sortable by cardinality score, i.e. after sorting, the
// LabelMatcher that is expected to yield the fewest matches is first in the
// slice, and LabelMatchers that match the empty string are last.
type LabelMatchers []*LabelMatcher
// LabelMatcher models the matching of a label.
func (lms LabelMatchers) Len() int { return len(lms) }
func (lms LabelMatchers) Swap(i, j int) { lms[i], lms[j] = lms[j], lms[i] }
func (lms LabelMatchers) Less(i, j int) bool { return lms[i].score < lms[j].score }
// LabelMatcher models the matching of a label. Create with NewLabelMatcher.
type LabelMatcher struct {
Type MatchType
Name model.LabelName
Value model.LabelValue
re *regexp.Regexp
score float64 // Cardinality score, between 0 and 1, 0 is lowest cardinality.
}
// NewLabelMatcher returns a LabelMatcher object ready to use.
@ -69,9 +78,96 @@ func NewLabelMatcher(matchType MatchType, name model.LabelName, value model.Labe
}
m.re = re
}
m.calculateScore()
return m, nil
}
// calculateScore is a helper method only called in the constructor. It
// calculates the cardinality score upfront, so that sorting by it is faster and
// doesn't change internal state of the matcher.
//
// The score is based on a pretty bad but still quite helpful heuristics for
// now. Note that this is an interim solution until the work in progress to
// properly intersect matchers is complete. We intend to not invest any further
// effort into tweaking the score calculation, as this could easily devolve into
// a rabbit hole.
//
// The heuristics works along the following lines:
//
// - A matcher that is known to match nothing would have a score of 0. (This
// case doesn't happen in the scope of this method.)
//
// - A matcher that matches the empty string has a score of 1.
//
// - Equal matchers have a score <= 0.5. The order in score for other matchers
// are RegexMatch, RegexNoMatch, NotEqual.
//
// - There are a number of score adjustments for known "magic" parts, like
// instance labels, metric names containing a colon (which are probably
// recording rules) and such.
//
// - On top, there is a tiny adjustment for the length of the matcher, following
// the blunt expectation that a long label name and/or value is more specific
// and will therefore have a lower cardinality.
//
// To reiterate on the above: PLEASE RESIST THE TEMPTATION TO TWEAK THIS
// METHOD. IT IS "MAGIC" ENOUGH ALREADY AND WILL GO AWAY WITH THE UPCOMING MORE
// POWERFUL INDEXING.
func (m *LabelMatcher) calculateScore() {
if m.Match("") {
m.score = 1
return
}
// lengthCorrection is between 0 (for length 0) and 0.1 (for length +Inf).
lengthCorrection := 0.1 * (1 - 1/float64(len(m.Name)+len(m.Value)+1))
switch m.Type {
case Equal:
m.score = 0.3 - lengthCorrection
case RegexMatch:
m.score = 0.6 - lengthCorrection
case RegexNoMatch:
m.score = 0.8 + lengthCorrection
case NotEqual:
m.score = 0.9 + lengthCorrection
}
if m.Type != Equal {
// Don't bother anymore in this case.
return
}
switch m.Name {
case model.InstanceLabel:
// Matches only metrics from a single instance, which clearly
// limits the damage.
m.score -= 0.2
case model.JobLabel:
// The usual case is a relatively low number of jobs with many
// metrics each.
m.score += 0.1
case model.BucketLabel, model.QuantileLabel:
// Magic labels for buckets and quantiles will match copiously.
m.score += 0.2
case model.MetricNameLabel:
if strings.Contains(string(m.Value), ":") {
// Probably a recording rule with limited cardinality.
m.score -= 0.1
return
}
if m.Value == "up" || m.Value == "scrape_duration_seconds" {
// Synthetic metrics which are contained in every scrape
// exactly once. There might be less frequent metric
// names, but the worst case is limited here, so give it
// a bump.
m.score -= 0.05
return
}
}
}
// MatchesEmptyString returns true if the LabelMatcher matches the empty string.
func (m *LabelMatcher) MatchesEmptyString() bool {
return m.score >= 1
}
func (m *LabelMatcher) String() string {
return fmt.Sprintf("%s%s%q", m.Name, m.Type, m.Value)
}

View file

@ -14,7 +14,12 @@
package metric
import (
"math/rand"
"reflect"
"sort"
"testing"
"github.com/prometheus/common/model"
)
func TestAnchoredMatcher(t *testing.T) {
@ -30,3 +35,39 @@ func TestAnchoredMatcher(t *testing.T) {
t.Errorf("Unexpected match for %q", "fooo")
}
}
func TestLabelMatchersSort(t *testing.T) {
// Line up Matchers in expected order:
want := LabelMatchers{
mustNewLabelMatcher(Equal, model.InstanceLabel, "not empty"),
mustNewLabelMatcher(Equal, model.MetricNameLabel, "a:recording:rule"),
mustNewLabelMatcher(Equal, model.MetricNameLabel, "up"),
mustNewLabelMatcher(Equal, "nothing_special but much longer", "not empty"),
mustNewLabelMatcher(Equal, "nothing_special", "not empty but longer"),
mustNewLabelMatcher(Equal, "nothing_special", "not empty"),
mustNewLabelMatcher(Equal, model.JobLabel, "not empty"),
mustNewLabelMatcher(Equal, model.BucketLabel, "not empty"),
mustNewLabelMatcher(RegexMatch, "irrelevant", "does not match empty string and is longer"),
mustNewLabelMatcher(RegexMatch, "irrelevant", "does not match empty string"),
mustNewLabelMatcher(RegexNoMatch, "irrelevant", "(matches empty string)?"),
mustNewLabelMatcher(RegexNoMatch, "irrelevant", "(matches empty string with a longer expression)?"),
mustNewLabelMatcher(NotEqual, "irrelevant", ""),
mustNewLabelMatcher(Equal, "irrelevant", ""),
}
got := make(LabelMatchers, len(want))
for i, j := range rand.Perm(len(want)) {
got[i] = want[j]
}
sort.Sort(got)
if !reflect.DeepEqual(want, got) {
t.Errorf("unexpected sorting of matchers, got %v, want %v", got, want)
}
}
func mustNewLabelMatcher(mt MatchType, name model.LabelName, val model.LabelValue) *LabelMatcher {
m, err := NewLabelMatcher(mt, name, val)
if err != nil {
panic(err)
}
return m
}