diff --git a/promql/ast.go b/promql/ast.go index 9606bb2e86..f8eaa3e159 100644 --- a/promql/ast.go +++ b/promql/ast.go @@ -228,11 +228,14 @@ func (vmc VectorMatchCardinality) String() string { type VectorMatching struct { // The cardinality of the two vectors. Card VectorMatchCardinality - // On contains the labels which define equality of a pair - // of elements from the vectors. - On model.LabelNames + // MatchingLabels contains the labels which define equality of a pair of + // elements from the vectors. + MatchingLabels model.LabelNames + // Ignoring excludes the given label names from matching, + // rather than only using them. + Ignoring bool // Include contains additional labels that should be included in - // the result from the side with the higher cardinality. + // the result from the side with the lower cardinality. Include model.LabelNames } diff --git a/promql/engine.go b/promql/engine.go index 5005f6a6f8..b0742ad8d2 100644 --- a/promql/engine.go +++ b/promql/engine.go @@ -728,7 +728,7 @@ func (ev *evaluator) vectorAnd(lhs, rhs vector, matching *VectorMatching) vector if matching.Card != CardManyToMany { panic("set operations must only use many-to-many matching") } - sigf := signatureFunc(matching.On...) + sigf := signatureFunc(matching.Ignoring, matching.MatchingLabels...) var result vector // The set of signatures for the right-hand side vector. @@ -751,7 +751,7 @@ func (ev *evaluator) vectorOr(lhs, rhs vector, matching *VectorMatching) vector if matching.Card != CardManyToMany { panic("set operations must only use many-to-many matching") } - sigf := signatureFunc(matching.On...) + sigf := signatureFunc(matching.Ignoring, matching.MatchingLabels...) var result vector leftSigs := map[uint64]struct{}{} @@ -773,7 +773,7 @@ func (ev *evaluator) vectorUnless(lhs, rhs vector, matching *VectorMatching) vec if matching.Card != CardManyToMany { panic("set operations must only use many-to-many matching") } - sigf := signatureFunc(matching.On...) + sigf := signatureFunc(matching.Ignoring, matching.MatchingLabels...) rightSigs := map[uint64]struct{}{} for _, rs := range rhs { @@ -795,9 +795,8 @@ func (ev *evaluator) vectorBinop(op itemType, lhs, rhs vector, matching *VectorM panic("many-to-many only allowed for set operators") } var ( - result = vector{} - sigf = signatureFunc(matching.On...) - resultLabels = append(matching.On, matching.Include...) + result = vector{} + sigf = signatureFunc(matching.Ignoring, matching.MatchingLabels...) ) // The control flow below handles one-to-one or many-to-one matching. @@ -851,7 +850,7 @@ func (ev *evaluator) vectorBinop(op itemType, lhs, rhs vector, matching *VectorM } else if !keep { continue } - metric := resultMetric(ls.Metric, op, resultLabels...) + metric := resultMetric(ls.Metric, rs.Metric, op, matching) insertedSigs, exists := matchedSigs[sig] if matching.Card == CardOneToOne { @@ -863,7 +862,7 @@ func (ev *evaluator) vectorBinop(op itemType, lhs, rhs vector, matching *VectorM // In many-to-one matching the grouping labels have to ensure a unique metric // for the result vector. Check whether those labels have already been added for // the same matching labels. - insertSig := model.SignatureForLabels(metric.Metric, matching.Include...) + insertSig := uint64(metric.Metric.Fingerprint()) if !exists { insertedSigs = map[uint64]struct{}{} matchedSigs[sig] = insertedSigs @@ -883,12 +882,16 @@ func (ev *evaluator) vectorBinop(op itemType, lhs, rhs vector, matching *VectorM } // signatureFunc returns a function that calculates the signature for a metric -// based on the provided labels. -func signatureFunc(labels ...model.LabelName) func(m metric.Metric) uint64 { - if len(labels) == 0 { +// based on the provided labels. If ignoring, then the given labels are ignored instead. +func signatureFunc(ignoring bool, labels ...model.LabelName) func(m metric.Metric) uint64 { + if len(labels) == 0 || ignoring { return func(m metric.Metric) uint64 { - m.Del(model.MetricNameLabel) - return uint64(m.Metric.Fingerprint()) + tmp := m.Metric.Clone() + for _, l := range labels { + delete(tmp, l) + } + delete(tmp, model.MetricNameLabel) + return uint64(tmp.Fingerprint()) } } return func(m metric.Metric) uint64 { @@ -898,19 +901,49 @@ func signatureFunc(labels ...model.LabelName) func(m metric.Metric) uint64 { // resultMetric returns the metric for the given sample(s) based on the vector // binary operation and the matching options. -func resultMetric(met metric.Metric, op itemType, labels ...model.LabelName) metric.Metric { - if len(labels) == 0 { - if shouldDropMetricName(op) { - met.Del(model.MetricNameLabel) +func resultMetric(lhs, rhs metric.Metric, op itemType, matching *VectorMatching) metric.Metric { + if shouldDropMetricName(op) { + lhs.Del(model.MetricNameLabel) + } + if len(matching.MatchingLabels)+len(matching.Include) == 0 { + return lhs + } + if matching.Ignoring { + if matching.Card == CardOneToOne { + for _, l := range matching.MatchingLabels { + lhs.Del(l) + } } - return met + for _, ln := range matching.Include { + // Included labels from the `group_x` modifier are taken from the "one"-side. + value := rhs.Metric[ln] + if value != "" { + lhs.Set(ln, rhs.Metric[ln]) + } else { + lhs.Del(ln) + } + } + return lhs } // As we definitely write, creating a new metric is the easiest solution. m := model.Metric{} - for _, ln := range labels { - // Included labels from the `group_x` modifier are taken from the "many"-side. - if v, ok := met.Metric[ln]; ok { + if matching.Card == CardOneToOne { + for _, ln := range matching.MatchingLabels { + if v, ok := lhs.Metric[ln]; ok { + m[ln] = v + } + } + } else { + for k, v := range lhs.Metric { + m[k] = v + } + } + for _, ln := range matching.Include { + // Included labels from the `group_x` modifier are taken from the "one"-side . + if v, ok := rhs.Metric[ln]; ok { m[ln] = v + } else { + delete(m, ln) } } return metric.Metric{Metric: m, Copied: false} diff --git a/promql/lex.go b/promql/lex.go index fbd9993520..cbfa877b24 100644 --- a/promql/lex.go +++ b/promql/lex.go @@ -172,6 +172,7 @@ const ( itemBy itemWithout itemOn + itemIgnoring itemGroupLeft itemGroupRight itemBool @@ -209,6 +210,7 @@ var key = map[string]itemType{ "keeping_extra": itemKeepCommon, "keep_common": itemKeepCommon, "on": itemOn, + "ignoring": itemIgnoring, "group_left": itemGroupLeft, "group_right": itemGroupRight, "bool": itemBool, diff --git a/promql/lex_test.go b/promql/lex_test.go index 549b24f865..49ab711f97 100644 --- a/promql/lex_test.go +++ b/promql/lex_test.go @@ -278,6 +278,9 @@ var tests = []struct { }, { input: "on", expected: []item{{itemOn, 0, "on"}}, + }, { + input: "ignoring", + expected: []item{{itemIgnoring, 0, "ignoring"}}, }, { input: "group_left", expected: []item{{itemGroupLeft, 0, "group_left"}}, diff --git a/promql/parse.go b/promql/parse.go index 125e1060b9..38911775f8 100644 --- a/promql/parse.go +++ b/promql/parse.go @@ -461,27 +461,32 @@ func (p *parser) expr() Expr { returnBool = true } - // Parse ON clause. - if p.peek().typ == itemOn { + // Parse ON/IGNORING clause. + if p.peek().typ == itemOn || p.peek().typ == itemIgnoring { + if p.peek().typ == itemIgnoring { + vecMatching.Ignoring = true + } p.next() - vecMatching.On = p.labels() + vecMatching.MatchingLabels = p.labels() // Parse grouping. - if t := p.peek().typ; t == itemGroupLeft { + if t := p.peek().typ; t == itemGroupLeft || t == itemGroupRight { p.next() - vecMatching.Card = CardManyToOne - vecMatching.Include = p.labels() - } else if t == itemGroupRight { - p.next() - vecMatching.Card = CardOneToMany - vecMatching.Include = p.labels() + if t == itemGroupLeft { + vecMatching.Card = CardManyToOne + } else { + vecMatching.Card = CardOneToMany + } + if p.peek().typ == itemLeftParen { + vecMatching.Include = p.labels() + } } } - for _, ln := range vecMatching.On { + for _, ln := range vecMatching.MatchingLabels { for _, ln2 := range vecMatching.Include { - if ln == ln2 { - p.errorf("label %q must not occur in ON and INCLUDE clause at once", ln) + if ln == ln2 && !vecMatching.Ignoring { + p.errorf("label %q must not occur in ON and GROUP clause at once", ln) } } } @@ -1047,7 +1052,7 @@ func (p *parser) checkType(node Node) (typ model.ValueType) { } if (lt != model.ValVector || rt != model.ValVector) && n.VectorMatching != nil { - if len(n.VectorMatching.On) > 0 { + if len(n.VectorMatching.MatchingLabels) > 0 { p.errorf("vector matching only allowed between vectors") } n.VectorMatching = nil diff --git a/promql/parse_test.go b/promql/parse_test.go index c400ba077a..e5d1ea8a37 100644 --- a/promql/parse_test.go +++ b/promql/parse_test.go @@ -250,6 +250,10 @@ var testExpr = []struct { input: "1 offset 1d", fail: true, errMsg: "offset modifier must be preceded by an instant or range selector", + }, { + input: "a - on(b) ignoring(c) d", + fail: true, + errMsg: "parse error at char 11: no valid expression found", }, // Vector binary operations. { @@ -465,14 +469,14 @@ var testExpr = []struct { }, }, VectorMatching: &VectorMatching{ - Card: CardOneToMany, - On: model.LabelNames{"baz", "buz"}, - Include: model.LabelNames{"test"}, + Card: CardOneToMany, + MatchingLabels: model.LabelNames{"baz", "buz"}, + Include: model.LabelNames{"test"}, }, }, VectorMatching: &VectorMatching{ - Card: CardOneToOne, - On: model.LabelNames{"foo"}, + Card: CardOneToOne, + MatchingLabels: model.LabelNames{"foo"}, }, }, }, { @@ -492,8 +496,29 @@ var testExpr = []struct { }, }, VectorMatching: &VectorMatching{ - Card: CardOneToOne, - On: model.LabelNames{"test", "blub"}, + Card: CardOneToOne, + MatchingLabels: model.LabelNames{"test", "blub"}, + }, + }, + }, { + input: "foo * on(test,blub) group_left bar", + expected: &BinaryExpr{ + Op: itemMUL, + LHS: &VectorSelector{ + Name: "foo", + LabelMatchers: metric.LabelMatchers{ + {Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, + }, + }, + RHS: &VectorSelector{ + Name: "bar", + LabelMatchers: metric.LabelMatchers{ + {Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, + }, + }, + VectorMatching: &VectorMatching{ + Card: CardManyToOne, + MatchingLabels: model.LabelNames{"test", "blub"}, }, }, }, { @@ -513,8 +538,30 @@ var testExpr = []struct { }, }, VectorMatching: &VectorMatching{ - Card: CardManyToMany, - On: model.LabelNames{"test", "blub"}, + Card: CardManyToMany, + MatchingLabels: model.LabelNames{"test", "blub"}, + }, + }, + }, { + input: "foo and ignoring(test,blub) bar", + expected: &BinaryExpr{ + Op: itemLAND, + LHS: &VectorSelector{ + Name: "foo", + LabelMatchers: metric.LabelMatchers{ + {Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, + }, + }, + RHS: &VectorSelector{ + Name: "bar", + LabelMatchers: metric.LabelMatchers{ + {Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, + }, + }, + VectorMatching: &VectorMatching{ + Card: CardManyToMany, + MatchingLabels: model.LabelNames{"test", "blub"}, + Ignoring: true, }, }, }, { @@ -534,8 +581,8 @@ var testExpr = []struct { }, }, VectorMatching: &VectorMatching{ - Card: CardManyToMany, - On: model.LabelNames{"bar"}, + Card: CardManyToMany, + MatchingLabels: model.LabelNames{"bar"}, }, }, }, { @@ -555,9 +602,55 @@ var testExpr = []struct { }, }, VectorMatching: &VectorMatching{ - Card: CardManyToOne, - On: model.LabelNames{"test", "blub"}, - Include: model.LabelNames{"bar"}, + Card: CardManyToOne, + MatchingLabels: model.LabelNames{"test", "blub"}, + Include: model.LabelNames{"bar"}, + }, + }, + }, { + input: "foo / ignoring(test,blub) group_left(blub) bar", + expected: &BinaryExpr{ + Op: itemDIV, + LHS: &VectorSelector{ + Name: "foo", + LabelMatchers: metric.LabelMatchers{ + {Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, + }, + }, + RHS: &VectorSelector{ + Name: "bar", + LabelMatchers: metric.LabelMatchers{ + {Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, + }, + }, + VectorMatching: &VectorMatching{ + Card: CardManyToOne, + MatchingLabels: model.LabelNames{"test", "blub"}, + Include: model.LabelNames{"blub"}, + Ignoring: true, + }, + }, + }, { + input: "foo / ignoring(test,blub) group_left(bar) bar", + expected: &BinaryExpr{ + Op: itemDIV, + LHS: &VectorSelector{ + Name: "foo", + LabelMatchers: metric.LabelMatchers{ + {Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, + }, + }, + RHS: &VectorSelector{ + Name: "bar", + LabelMatchers: metric.LabelMatchers{ + {Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, + }, + }, + VectorMatching: &VectorMatching{ + Card: CardManyToOne, + MatchingLabels: model.LabelNames{"test", "blub"}, + Include: model.LabelNames{"bar"}, + Ignoring: true, }, }, }, { @@ -577,9 +670,32 @@ var testExpr = []struct { }, }, VectorMatching: &VectorMatching{ - Card: CardOneToMany, - On: model.LabelNames{"test", "blub"}, - Include: model.LabelNames{"bar", "foo"}, + Card: CardOneToMany, + MatchingLabels: model.LabelNames{"test", "blub"}, + Include: model.LabelNames{"bar", "foo"}, + }, + }, + }, { + input: "foo - ignoring(test,blub) group_right(bar,foo) bar", + expected: &BinaryExpr{ + Op: itemSUB, + LHS: &VectorSelector{ + Name: "foo", + LabelMatchers: metric.LabelMatchers{ + {Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, + }, + }, + RHS: &VectorSelector{ + Name: "bar", + LabelMatchers: metric.LabelMatchers{ + {Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, + }, + }, + VectorMatching: &VectorMatching{ + Card: CardOneToMany, + MatchingLabels: model.LabelNames{"test", "blub"}, + Include: model.LabelNames{"bar", "foo"}, + Ignoring: true, }, }, }, { @@ -638,14 +754,10 @@ var testExpr = []struct { input: "foo unless on(bar) group_right(baz) bar", fail: true, errMsg: "no grouping allowed for \"unless\" operation", - }, { - input: `http_requests{group="production"} / on(instance) group_left cpu_count{type="smp"}`, - fail: true, - errMsg: "unexpected identifier \"cpu_count\" in grouping opts, expected \"(\"", }, { input: `http_requests{group="production"} + on(instance) group_left(job,instance) cpu_count{type="smp"}`, fail: true, - errMsg: "label \"instance\" must not occur in ON and INCLUDE clause at once", + errMsg: "label \"instance\" must not occur in ON and GROUP clause at once", }, { input: "foo + bool bar", fail: true, diff --git a/promql/printer.go b/promql/printer.go index 748d77461e..82ac53812f 100644 --- a/promql/printer.go +++ b/promql/printer.go @@ -159,8 +159,12 @@ func (node *BinaryExpr) String() string { matching := "" vm := node.VectorMatching - if vm != nil && len(vm.On) > 0 { - matching = fmt.Sprintf(" ON(%s)", vm.On) + if vm != nil && len(vm.MatchingLabels) > 0 { + if vm.Ignoring { + matching = fmt.Sprintf(" IGNORING(%s)", vm.MatchingLabels) + } else { + matching = fmt.Sprintf(" ON(%s)", vm.MatchingLabels) + } if vm.Card == CardManyToOne { matching += fmt.Sprintf(" GROUP_LEFT(%s)", vm.Include) } diff --git a/promql/printer_test.go b/promql/printer_test.go index 29b275e0b3..e17774ac8c 100644 --- a/promql/printer_test.go +++ b/promql/printer_test.go @@ -36,6 +36,12 @@ func TestExprString(t *testing.T) { { in: `sum(task:errors:rate10s{job="s"}) WITHOUT (instance)`, }, + { + in: `a - ON(b) c`, + }, + { + in: `a - IGNORING(b) c`, + }, { in: `up > BOOL 0`, }, diff --git a/promql/testdata/operators.test b/promql/testdata/operators.test index e8d39dc195..b070a72815 100644 --- a/promql/testdata/operators.test +++ b/promql/testdata/operators.test @@ -79,6 +79,14 @@ eval instant at 50m (http_requests{group="canary"} + 1) and on(instance) http_re {group="canary", instance="0", job="api-server"} 301 {group="canary", instance="0", job="app-server"} 701 +eval instant at 50m (http_requests{group="canary"} + 1) and ignoring(group) http_requests{instance="0", group="production"} + {group="canary", instance="0", job="api-server"} 301 + {group="canary", instance="0", job="app-server"} 701 + +eval instant at 50m (http_requests{group="canary"} + 1) and ignoring(group, job) http_requests{instance="0", group="production"} + {group="canary", instance="0", job="api-server"} 301 + {group="canary", instance="0", job="app-server"} 701 + eval instant at 50m http_requests{group="canary"} or http_requests{group="production"} http_requests{group="canary", instance="0", job="api-server"} 300 http_requests{group="canary", instance="0", job="app-server"} 700 @@ -109,6 +117,14 @@ eval instant at 50m (http_requests{group="canary"} + 1) or on(instance) (http_re vector_matching_a{l="x"} 10 vector_matching_a{l="y"} 20 +eval instant at 50m (http_requests{group="canary"} + 1) or ignoring(l, group, job) (http_requests or cpu_count or vector_matching_a) + {group="canary", instance="0", job="api-server"} 301 + {group="canary", instance="0", job="app-server"} 701 + {group="canary", instance="1", job="api-server"} 401 + {group="canary", instance="1", job="app-server"} 801 + vector_matching_a{l="x"} 10 + vector_matching_a{l="y"} 20 + eval instant at 50m http_requests{group="canary"} unless http_requests{instance="0"} http_requests{group="canary", instance="1", job="api-server"} 400 http_requests{group="canary", instance="1", job="app-server"} 800 @@ -125,6 +141,18 @@ eval instant at 50m http_requests{group="canary"} / on(instance,job) http_reques {instance="1", job="api-server"} 2 {instance="1", job="app-server"} 1.3333333333333333 +eval instant at 50m http_requests{group="canary"} unless ignoring(group, instance) http_requests{instance="0"} + +eval instant at 50m http_requests{group="canary"} unless ignoring(group) http_requests{instance="0"} + http_requests{group="canary", instance="1", job="api-server"} 400 + http_requests{group="canary", instance="1", job="app-server"} 800 + +eval instant at 50m http_requests{group="canary"} / ignoring(group) http_requests{group="production"} + {instance="0", job="api-server"} 3 + {instance="0", job="app-server"} 1.4 + {instance="1", job="api-server"} 2 + {instance="1", job="app-server"} 1.3333333333333333 + # https://github.com/prometheus/prometheus/issues/1489 eval instant at 50m http_requests AND ON (dummy) vector(1) http_requests{group="canary", instance="0", job="api-server"} 300 @@ -136,6 +164,16 @@ eval instant at 50m http_requests AND ON (dummy) vector(1) http_requests{group="production", instance="1", job="api-server"} 200 http_requests{group="production", instance="1", job="app-server"} 600 +eval instant at 50m http_requests AND IGNORING (group, instance, job) vector(1) + http_requests{group="canary", instance="0", job="api-server"} 300 + http_requests{group="canary", instance="0", job="app-server"} 700 + http_requests{group="canary", instance="1", job="api-server"} 400 + http_requests{group="canary", instance="1", job="app-server"} 800 + http_requests{group="production", instance="0", job="api-server"} 100 + http_requests{group="production", instance="0", job="app-server"} 500 + http_requests{group="production", instance="1", job="api-server"} 200 + http_requests{group="production", instance="1", job="app-server"} 600 + # Comparisons. eval instant at 50m SUM(http_requests) BY (job) > 1000 @@ -170,3 +208,85 @@ eval instant at 50m 0 == bool 1 eval instant at 50m 1 == bool 1 1 + +# group_left/group_right. + +clear + +load 5m + node_var{instance="abc",job="node"} 2 + node_role{instance="abc",job="node",role="prometheus"} 1 + +load 5m + node_cpu{instance="abc",job="node",mode="idle"} 3 + node_cpu{instance="abc",job="node",mode="user"} 1 + node_cpu{instance="def",job="node",mode="idle"} 8 + node_cpu{instance="def",job="node",mode="user"} 2 + +load 5m + random{foo="bar"} 1 + +# Copy machine role to node variable. +eval instant at 5m node_role * on (instance) group_right (role) node_var + {instance="abc",job="node",role="prometheus"} 2 + +eval instant at 5m node_var * on (instance) group_left (role) node_role + {instance="abc",job="node",role="prometheus"} 2 + +eval instant at 5m node_var * ignoring (role) group_left (role) node_role + {instance="abc",job="node",role="prometheus"} 2 + +eval instant at 5m node_role * ignoring (role) group_right (role) node_var + {instance="abc",job="node",role="prometheus"} 2 + +# Copy machine role to node variable with instrumentation labels. +eval instant at 5m node_cpu * ignoring (role, mode) group_left (role) node_role + {instance="abc",job="node",mode="idle",role="prometheus"} 3 + {instance="abc",job="node",mode="user",role="prometheus"} 1 + +eval instant at 5m node_cpu * on (instance) group_left (role) node_role + {instance="abc",job="node",mode="idle",role="prometheus"} 3 + {instance="abc",job="node",mode="user",role="prometheus"} 1 + + +# Ratio of total. +eval instant at 5m node_cpu / on (instance) group_left sum by (instance,job)(node_cpu) + {instance="abc",job="node",mode="idle"} .75 + {instance="abc",job="node",mode="user"} .25 + {instance="def",job="node",mode="idle"} .80 + {instance="def",job="node",mode="user"} .20 + +eval instant at 5m sum by (mode, job)(node_cpu) / on (job) group_left sum by (job)(node_cpu) + {job="node",mode="idle"} 0.7857142857142857 + {job="node",mode="user"} 0.21428571428571427 + +eval instant at 5m sum(sum by (mode, job)(node_cpu) / on (job) group_left sum by (job)(node_cpu)) + {} 1.0 + + +eval instant at 5m node_cpu / ignoring (mode) group_left sum without (mode)(node_cpu) + {instance="abc",job="node",mode="idle"} .75 + {instance="abc",job="node",mode="user"} .25 + {instance="def",job="node",mode="idle"} .80 + {instance="def",job="node",mode="user"} .20 + +eval instant at 5m node_cpu / ignoring (mode) group_left(dummy) sum without (mode)(node_cpu) + {instance="abc",job="node",mode="idle"} .75 + {instance="abc",job="node",mode="user"} .25 + {instance="def",job="node",mode="idle"} .80 + {instance="def",job="node",mode="user"} .20 + +eval instant at 5m sum without (instance)(node_cpu) / ignoring (mode) group_left sum without (instance, mode)(node_cpu) + {job="node",mode="idle"} 0.7857142857142857 + {job="node",mode="user"} 0.21428571428571427 + +eval instant at 5m sum(sum without (instance)(node_cpu) / ignoring (mode) group_left sum without (instance, mode)(node_cpu)) + {} 1.0 + + +# Copy over label from metric with no matching labels, without having to list cross-job target labels ('job' here). +eval instant at 5m node_cpu + on(dummy) group_left(foo) random*0 + {instance="abc",job="node",mode="idle",foo="bar"} 3 + {instance="abc",job="node",mode="user",foo="bar"} 1 + {instance="def",job="node",mode="idle",foo="bar"} 8 + {instance="def",job="node",mode="user",foo="bar"} 2