From 8cc86f25c0efa7c87fe01571efdb41e630f3e052 Mon Sep 17 00:00:00 2001 From: Tobias Schmidt Date: Sat, 2 Apr 2016 18:52:18 -0400 Subject: [PATCH] Implement relative complement set operator "unless" The `unless` set operator can be used to return all vector elements from the LHS which do not match the elements on the RHS. A use case is to return all metrics for nodes which do not have a specific role: node_load1 unless on(instance) chef_role{role="app"} --- promql/engine.go | 31 +++++++-- promql/lex.go | 19 ++++-- promql/lex_test.go | 3 + promql/parse.go | 14 ++-- promql/parse_test.go | 118 ++++++++++++++++++++++++++++++--- promql/testdata/operators.test | 10 +++ 6 files changed, 169 insertions(+), 26 deletions(-) diff --git a/promql/engine.go b/promql/engine.go index 836710181..5005f6a6f 100644 --- a/promql/engine.go +++ b/promql/engine.go @@ -629,6 +629,8 @@ func (ev *evaluator) eval(expr Expr) model.Value { return ev.vectorAnd(lhs.(vector), rhs.(vector), e.VectorMatching) case itemLOR: return ev.vectorOr(lhs.(vector), rhs.(vector), e.VectorMatching) + case itemLUnless: + return ev.vectorUnless(lhs.(vector), rhs.(vector), e.VectorMatching) default: return ev.vectorBinop(e.Op, lhs.(vector), rhs.(vector), e.VectorMatching, e.ReturnBool) } @@ -724,9 +726,8 @@ func (ev *evaluator) matrixSelector(node *MatrixSelector) matrix { func (ev *evaluator) vectorAnd(lhs, rhs vector, matching *VectorMatching) vector { if matching.Card != CardManyToMany { - panic("logical operations must always be many-to-many matching") + panic("set operations must only use many-to-many matching") } - // If no matching labels are specified, match by all labels. sigf := signatureFunc(matching.On...) var result vector @@ -748,7 +749,7 @@ func (ev *evaluator) vectorAnd(lhs, rhs vector, matching *VectorMatching) vector func (ev *evaluator) vectorOr(lhs, rhs vector, matching *VectorMatching) vector { if matching.Card != CardManyToMany { - panic("logical operations must always be many-to-many matching") + panic("set operations must only use many-to-many matching") } sigf := signatureFunc(matching.On...) @@ -768,10 +769,30 @@ func (ev *evaluator) vectorOr(lhs, rhs vector, matching *VectorMatching) vector return result } -// vectorBinop evaluates a binary operation between two vector, excluding AND and OR. +func (ev *evaluator) vectorUnless(lhs, rhs vector, matching *VectorMatching) vector { + if matching.Card != CardManyToMany { + panic("set operations must only use many-to-many matching") + } + sigf := signatureFunc(matching.On...) + + rightSigs := map[uint64]struct{}{} + for _, rs := range rhs { + rightSigs[sigf(rs.Metric)] = struct{}{} + } + + var result vector + for _, ls := range lhs { + if _, ok := rightSigs[sigf(ls.Metric)]; !ok { + result = append(result, ls) + } + } + return result +} + +// vectorBinop evaluates a binary operation between two vectors, excluding set operators. func (ev *evaluator) vectorBinop(op itemType, lhs, rhs vector, matching *VectorMatching, returnBool bool) vector { if matching.Card == CardManyToMany { - panic("many-to-many only allowed for AND and OR") + panic("many-to-many only allowed for set operators") } var ( result = vector{} diff --git a/promql/lex.go b/promql/lex.go index 743a528d6..a902ea210 100644 --- a/promql/lex.go +++ b/promql/lex.go @@ -48,7 +48,7 @@ func (i item) String() string { return fmt.Sprintf("%q", i.val) } -// isOperator returns true if the item corresponds to a logical or arithmetic operator. +// isOperator returns true if the item corresponds to a arithmetic or set operator. // Returns false otherwise. func (i itemType) isOperator() bool { return i > operatorsStart && i < operatorsEnd } @@ -71,6 +71,15 @@ func (i itemType) isComparisonOperator() bool { } } +// isSetOperator returns whether the item corresponds to a set operator. +func (i itemType) isSetOperator() bool { + switch i { + case itemLAND, itemLOR, itemLUnless: + return true + } + return false +} + // Constants for operator precedence in expressions. // const LowestPrec = 0 // Non-operators. @@ -82,7 +91,7 @@ func (i itemType) precedence() int { switch i { case itemLOR: return 1 - case itemLAND: + case itemLAND, itemLUnless: return 2 case itemEQL, itemNEQ, itemLTE, itemLSS, itemGTE, itemGTR: return 3 @@ -127,6 +136,7 @@ const ( itemDIV itemLAND itemLOR + itemLUnless itemEQL itemNEQ itemLTE @@ -168,8 +178,9 @@ const ( var key = map[string]itemType{ // Operators. - "and": itemLAND, - "or": itemLOR, + "and": itemLAND, + "or": itemLOR, + "unless": itemLUnless, // Aggregators. "sum": itemSum, diff --git a/promql/lex_test.go b/promql/lex_test.go index d516c3d93..549b24f86 100644 --- a/promql/lex_test.go +++ b/promql/lex_test.go @@ -217,6 +217,9 @@ var tests = []struct { }, { input: `or`, expected: []item{{itemLOR, 0, `or`}}, + }, { + input: `unless`, + expected: []item{{itemLUnless, 0, `unless`}}, }, // Test aggregators. { diff --git a/promql/parse.go b/promql/parse.go index c91df6fb3..534526322 100644 --- a/promql/parse.go +++ b/promql/parse.go @@ -447,7 +447,7 @@ func (p *parser) expr() Expr { vecMatching := &VectorMatching{ Card: CardOneToOne, } - if op == itemLAND || op == itemLOR { + if op.isSetOperator() { vecMatching.Card = CardManyToMany } @@ -1042,7 +1042,7 @@ func (p *parser) checkType(node Node) (typ model.ValueType) { rt := p.checkType(n.RHS) if !n.Op.isOperator() { - p.errorf("only logical and arithmetic operators allowed in binary expression, got %q", n.Op) + p.errorf("binary expression does not support operator %q", n.Op) } if (lt != model.ValScalar && lt != model.ValVector) || (rt != model.ValScalar && rt != model.ValVector) { p.errorf("binary expression must contain only scalar and vector types") @@ -1055,18 +1055,18 @@ func (p *parser) checkType(node Node) (typ model.ValueType) { n.VectorMatching = nil } else { // Both operands are vectors. - if n.Op == itemLAND || n.Op == itemLOR { + if n.Op.isSetOperator() { if n.VectorMatching.Card == CardOneToMany || n.VectorMatching.Card == CardManyToOne { - p.errorf("no grouping allowed for AND and OR operations") + p.errorf("no grouping allowed for %q operation", n.Op) } if n.VectorMatching.Card != CardManyToMany { - p.errorf("AND and OR operations must always be many-to-many") + p.errorf("set operations must always be many-to-many") } } } - if (lt == model.ValScalar || rt == model.ValScalar) && (n.Op == itemLAND || n.Op == itemLOR) { - p.errorf("AND and OR not allowed in binary scalar expression") + if (lt == model.ValScalar || rt == model.ValScalar) && n.Op.isSetOperator() { + p.errorf("set operator %q not allowed in binary scalar expression", n.Op) } case *Call: diff --git a/promql/parse_test.go b/promql/parse_test.go index b438d2ec8..c400ba077 100644 --- a/promql/parse_test.go +++ b/promql/parse_test.go @@ -213,7 +213,7 @@ var testExpr = []struct { }, { input: "1 and 1", fail: true, - errMsg: "AND and OR not allowed in binary scalar expression", + errMsg: "set operator \"and\" not allowed in binary scalar expression", }, { input: "1 == 1", fail: true, @@ -221,7 +221,11 @@ var testExpr = []struct { }, { input: "1 or 1", fail: true, - errMsg: "AND and OR not allowed in binary scalar expression", + errMsg: "set operator \"or\" not allowed in binary scalar expression", + }, { + input: "1 unless 1", + fail: true, + errMsg: "set operator \"unless\" not allowed in binary scalar expression", }, { input: "1 !~ 1", fail: true, @@ -339,6 +343,24 @@ var testExpr = []struct { }, VectorMatching: &VectorMatching{Card: CardManyToMany}, }, + }, { + input: "foo unless bar", + expected: &BinaryExpr{ + Op: itemLUnless, + LHS: &VectorSelector{ + Name: "foo", + LabelMatchers: metric.LabelMatchers{ + {Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, + }, + }, + RHS: &VectorSelector{ + Name: "bar", + LabelMatchers: metric.LabelMatchers{ + {Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, + }, + }, + VectorMatching: &VectorMatching{Card: CardManyToMany}, + }, }, { // Test and/or precedence and reassigning of operands. input: "foo + bar or bla and blub", @@ -378,6 +400,45 @@ var testExpr = []struct { }, VectorMatching: &VectorMatching{Card: CardManyToMany}, }, + }, { + // Test and/or/unless precedence. + input: "foo and bar unless baz or qux", + expected: &BinaryExpr{ + Op: itemLOR, + LHS: &BinaryExpr{ + Op: itemLUnless, + LHS: &BinaryExpr{ + Op: itemLAND, + LHS: &VectorSelector{ + Name: "foo", + LabelMatchers: metric.LabelMatchers{ + {Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, + }, + }, + RHS: &VectorSelector{ + Name: "bar", + LabelMatchers: metric.LabelMatchers{ + {Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, + }, + }, + VectorMatching: &VectorMatching{Card: CardManyToMany}, + }, + RHS: &VectorSelector{ + Name: "baz", + LabelMatchers: metric.LabelMatchers{ + {Type: metric.Equal, Name: model.MetricNameLabel, Value: "baz"}, + }, + }, + VectorMatching: &VectorMatching{Card: CardManyToMany}, + }, + RHS: &VectorSelector{ + Name: "qux", + LabelMatchers: metric.LabelMatchers{ + {Type: metric.Equal, Name: model.MetricNameLabel, Value: "qux"}, + }, + }, + VectorMatching: &VectorMatching{Card: CardManyToMany}, + }, }, { // Test precedence and reassigning of operands. input: "bar + on(foo) bla / on(baz, buz) group_right(test) blub", @@ -456,6 +517,27 @@ var testExpr = []struct { On: model.LabelNames{"test", "blub"}, }, }, + }, { + input: "foo unless on(bar) baz", + expected: &BinaryExpr{ + Op: itemLUnless, + LHS: &VectorSelector{ + Name: "foo", + LabelMatchers: metric.LabelMatchers{ + {Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, + }, + }, + RHS: &VectorSelector{ + Name: "baz", + LabelMatchers: metric.LabelMatchers{ + {Type: metric.Equal, Name: model.MetricNameLabel, Value: "baz"}, + }, + }, + VectorMatching: &VectorMatching{ + Card: CardManyToMany, + On: model.LabelNames{"bar"}, + }, + }, }, { input: "foo / on(test,blub) group_left(bar) bar", expected: &BinaryExpr{ @@ -503,19 +585,27 @@ var testExpr = []struct { }, { input: "foo and 1", fail: true, - errMsg: "AND and OR not allowed in binary scalar expression", + errMsg: "set operator \"and\" not allowed in binary scalar expression", }, { input: "1 and foo", fail: true, - errMsg: "AND and OR not allowed in binary scalar expression", + errMsg: "set operator \"and\" not allowed in binary scalar expression", }, { input: "foo or 1", fail: true, - errMsg: "AND and OR not allowed in binary scalar expression", + errMsg: "set operator \"or\" not allowed in binary scalar expression", }, { input: "1 or foo", fail: true, - errMsg: "AND and OR not allowed in binary scalar expression", + errMsg: "set operator \"or\" not allowed in binary scalar expression", + }, { + input: "foo unless 1", + fail: true, + errMsg: "set operator \"unless\" not allowed in binary scalar expression", + }, { + input: "1 unless foo", + fail: true, + errMsg: "set operator \"unless\" not allowed in binary scalar expression", }, { input: "1 or on(bar) foo", fail: true, @@ -527,19 +617,27 @@ var testExpr = []struct { }, { input: "foo and on(bar) group_left(baz) bar", fail: true, - errMsg: "no grouping allowed for AND and OR operations", + errMsg: "no grouping allowed for \"and\" operation", }, { input: "foo and on(bar) group_right(baz) bar", fail: true, - errMsg: "no grouping allowed for AND and OR operations", + errMsg: "no grouping allowed for \"and\" operation", }, { input: "foo or on(bar) group_left(baz) bar", fail: true, - errMsg: "no grouping allowed for AND and OR operations", + errMsg: "no grouping allowed for \"or\" operation", }, { input: "foo or on(bar) group_right(baz) bar", fail: true, - errMsg: "no grouping allowed for AND and OR operations", + errMsg: "no grouping allowed for \"or\" operation", + }, { + input: "foo unless on(bar) group_left(baz) bar", + fail: true, + errMsg: "no grouping allowed for \"unless\" operation", + }, { + input: "foo unless on(bar) group_right(baz) bar", + fail: true, + errMsg: "no grouping allowed for \"unless\" operation", }, { input: `http_requests{group="production"} / on(instance) group_left cpu_count{type="smp"}`, fail: true, diff --git a/promql/testdata/operators.test b/promql/testdata/operators.test index 3e5864fe5..e8d39dc19 100644 --- a/promql/testdata/operators.test +++ b/promql/testdata/operators.test @@ -109,6 +109,16 @@ eval instant at 50m (http_requests{group="canary"} + 1) or on(instance) (http_re vector_matching_a{l="x"} 10 vector_matching_a{l="y"} 20 +eval instant at 50m http_requests{group="canary"} unless http_requests{instance="0"} + http_requests{group="canary", instance="1", job="api-server"} 400 + http_requests{group="canary", instance="1", job="app-server"} 800 + +eval instant at 50m http_requests{group="canary"} unless on(job) http_requests{instance="0"} + +eval instant at 50m http_requests{group="canary"} unless on(job, instance) http_requests{instance="0"} + http_requests{group="canary", instance="1", job="api-server"} 400 + http_requests{group="canary", instance="1", job="app-server"} 800 + eval instant at 50m http_requests{group="canary"} / on(instance,job) http_requests{group="production"} {instance="0", job="api-server"} 3 {instance="0", job="app-server"} 1.4