PromQL: Add position metadata to the AST (#6615)

Signed-off-by: Tobias Guggenmos <tguggenm@redhat.com>
Co-authored-by: Julius Volz <julius.volz@gmail.com>

promql/ast.go

@@ -39,6 +39,9 @@ type Node interface {
 	// String representation of the node that returns the given node when parsed
 	// as part of a valid query.
 	String() string
+
+	// PositionRange returns the position of the AST Node in the query string.
+	PositionRange() PositionRange
 }
 
 // Statement is a generic interface for all statements.
@@ -84,6 +87,7 @@ type AggregateExpr struct {
 	Param    Expr     // Parameter used by some aggregators.
 	Grouping []string // The labels by which to group the Vector.
 	Without  bool     // Whether to drop the given labels rather than keep them.
+	PosRange PositionRange
 }
 
 // BinaryExpr represents a binary expression between two child expressions.
@@ -103,12 +107,16 @@ type BinaryExpr struct {
 type Call struct {
 	Func *Function   // The function that was called.
 	Args Expressions // Arguments used in the call.
+
+	PosRange PositionRange
 }
 
 // MatrixSelector represents a Matrix selection.
 type MatrixSelector struct {
 	VectorSelector *VectorSelector
 	Range          time.Duration
+
+	EndPos Pos
 }
 
 // SubqueryExpr represents a subquery.
@@ -117,22 +125,28 @@ type SubqueryExpr struct {
 	Range  time.Duration
 	Offset time.Duration
 	Step   time.Duration
+
+	EndPos Pos
 }
 
 // NumberLiteral represents a number.
 type NumberLiteral struct {
 	Val float64
+
+	PosRange PositionRange
 }
 
 // ParenExpr wraps an expression so it cannot be disassembled as a consequence
 // of operator precedence.
 type ParenExpr struct {
 	Expr     Expr
+	PosRange PositionRange
 }
 
 // StringLiteral represents a string.
 type StringLiteral struct {
 	Val      string
+	PosRange PositionRange
 }
 
 // UnaryExpr represents a unary operation on another expression.
@@ -140,6 +154,8 @@ type StringLiteral struct {
 type UnaryExpr struct {
 	Op   ItemType
 	Expr Expr
+
+	StartPos Pos
 }
 
 // VectorSelector represents a Vector selection.
@@ -151,6 +167,8 @@ type VectorSelector struct {
 	// The unexpanded seriesSet populated at query preparation time.
 	unexpandedSeriesSet storage.SeriesSet
 	series              []storage.Series
+
+	PosRange PositionRange
 }
 
 func (e *AggregateExpr) Type() ValueType  { return ValueTypeVector }
@@ -319,3 +337,82 @@ func Children(node Node) []Node {
 		panic(errors.Errorf("promql.Children: unhandled node type %T", node))
 	}
 }
+
+// PositionRange describes a position in the input string of the parser.
+type PositionRange struct {
+	Start Pos
+	End   Pos
+}
+
+// mergeRanges is a helper function to merge the PositionRanges of two Nodes.
+// Note that the arguments must be in the same order as they
+// occur in the input string.
+func mergeRanges(first Node, last Node) PositionRange {
+	return PositionRange{
+		Start: first.PositionRange().Start,
+		End:   last.PositionRange().End,
+	}
+}
+
+// Item implements the Node interface.
+// This makes it possible to call mergeRanges on them.
+func (i *Item) PositionRange() PositionRange {
+	return PositionRange{
+		Start: i.Pos,
+		End:   i.Pos + Pos(len(i.Val)),
+	}
+}
+
+func (e *AggregateExpr) PositionRange() PositionRange {
+	return e.PosRange
+}
+func (e *BinaryExpr) PositionRange() PositionRange {
+	return mergeRanges(e.LHS, e.RHS)
+}
+func (e *Call) PositionRange() PositionRange {
+	return e.PosRange
+}
+func (e *EvalStmt) PositionRange() PositionRange {
+	return e.Expr.PositionRange()
+}
+func (e Expressions) PositionRange() PositionRange {
+	if len(e) == 0 {
+		// Position undefined.
+		return PositionRange{
+			Start: -1,
+			End:   -1,
+		}
+	} else {
+		return mergeRanges(e[0], e[len(e)-1])
+	}
+}
+func (e *MatrixSelector) PositionRange() PositionRange {
+	return PositionRange{
+		Start: e.VectorSelector.PositionRange().Start,
+		End:   e.EndPos,
+	}
+}
+func (e *SubqueryExpr) PositionRange() PositionRange {
+	return PositionRange{
+		Start: e.Expr.PositionRange().Start,
+		End:   e.EndPos,
+	}
+}
+func (e *NumberLiteral) PositionRange() PositionRange {
+	return e.PosRange
+}
+func (e *ParenExpr) PositionRange() PositionRange {
+	return e.PosRange
+}
+func (e *StringLiteral) PositionRange() PositionRange {
+	return e.PosRange
+}
+func (e *UnaryExpr) PositionRange() PositionRange {
+	return PositionRange{
+		Start: e.StartPos,
+		End:   e.Expr.PositionRange().End,
+	}
+}
+func (e *VectorSelector) PositionRange() PositionRange {
+	return e.PosRange
+}

promql/engine.go

@@ -406,6 +406,13 @@ type testStmt func(context.Context) error
 func (testStmt) String() string { return "test statement" }
 func (testStmt) stmt()          {}
 
+func (testStmt) PositionRange() PositionRange {
+	return PositionRange{
+		Start: -1,
+		End:   -1,
+	}
+}
+
 func (ng *Engine) newTestQuery(f func(context.Context) error) Query {
 	qry := &query{
 		q:     "test statement",

promql/generated_parser.y

@@ -36,7 +36,6 @@ import (
     series    []sequenceValue
     uint      uint64
     float     float64
-    string    string
     duration  time.Duration
 }
 
@@ -127,7 +126,7 @@ START_METRIC_SELECTOR
 
 
 // Type definitions for grammar rules.
-%type <matchers> label_match_list label_matchers
+%type <matchers> label_match_list
 %type <matcher> label_matcher
 
 %type <item> aggregate_op grouping_label match_op maybe_label metric_identifier unary_op
@@ -138,8 +137,7 @@ START_METRIC_SELECTOR
 %type <series> series_item series_values
 %type <uint> uint
 %type <float> number series_value signed_number
-%type <node> aggregate_expr aggregate_modifier bin_modifier binary_expr bool_modifier expr function_call function_call_args function_call_body group_modifiers matrix_selector number_literal offset_expr on_or_ignoring paren_expr string_literal subquery_expr unary_expr vector_selector
+%type <node> aggregate_expr aggregate_modifier bin_modifier binary_expr bool_modifier expr function_call function_call_args function_call_body group_modifiers label_matchers matrix_selector number_literal offset_expr on_or_ignoring paren_expr string_literal subquery_expr unary_expr vector_selector
-%type <string> string
 %type <duration> duration maybe_duration
 
 %start start
@@ -333,6 +331,10 @@ function_call   : IDENTIFIER function_call_body
                         $$ = &Call{
                                 Func: fn,
                                 Args: $2.(Expressions),
+                                PosRange: PositionRange{
+                                        Start: $1.Pos,
+                                        End:   yylex.(*parser).lastClosing,
+                                },
                         }
                         }
                 ;
@@ -354,7 +356,7 @@ function_call_args: function_call_args COMMA expr
  */
 
 paren_expr      : LEFT_PAREN expr RIGHT_PAREN
-                        { $$ = &ParenExpr{Expr: $2.(Expr)} }
+                        { $$ = &ParenExpr{Expr: $2.(Expr), PosRange: mergeRanges(&$1, &$3)} }
                 ;
 
 /*
@@ -386,6 +388,7 @@ matrix_selector : expr LEFT_BRACKET duration RIGHT_BRACKET
                         $$ = &MatrixSelector{
                                 VectorSelector: vs,
                                 Range: $3,
+                                EndPos: yylex.(*parser).lastClosing,
                         }
                         }
                 ;
@@ -396,6 +399,8 @@ subquery_expr   : expr LEFT_BRACKET duration COLON maybe_duration RIGHT_BRACKET
                                 Expr:  $1.(Expr),
                                 Range: $3,
                                 Step:  $5,
+                                
+                                EndPos: $6.Pos + 1,
                         }
                         }
                 | expr LEFT_BRACKET duration COLON duration error
@@ -420,9 +425,10 @@ unary_expr      :
                                 if $1.Typ == SUB {
                                         nl.Val *= -1
                                 }
+                                nl.PosRange.Start = $1.Pos
                                 $$ = nl
                         } else {
-                                $$ = &UnaryExpr{Op: $1.Typ, Expr: $2.(Expr)}
+                                $$ = &UnaryExpr{Op: $1.Typ, Expr: $2.(Expr), StartPos: $1.Pos}
                         }
                         }
                 ;
@@ -432,20 +438,52 @@ unary_expr      :
  */
 
 vector_selector: metric_identifier label_matchers
-                        { $$ = yylex.(*parser).newVectorSelector($1.Val, $2) }
+                        { 
+                        vs := $2.(*VectorSelector)
+                        vs.PosRange = mergeRanges(&$1, vs) 
+                        vs.Name = $1.Val
+                        yylex.(*parser).assembleVectorSelector(vs)
+                        $$ = vs
+                        }
                 | metric_identifier 
-                        { $$ = yylex.(*parser).newVectorSelector($1.Val, nil) }
+                        { 
+                        vs := &VectorSelector{
+                                Name: $1.Val,
+                                LabelMatchers: []*labels.Matcher{},
+                                PosRange: $1.PositionRange(),
+                        }
+                        yylex.(*parser).assembleVectorSelector(vs)
+                        $$ = vs
+                        }
                 | label_matchers
-                        { $$ = yylex.(*parser).newVectorSelector("", $1) }
+                        { 
+                        vs := $1.(*VectorSelector)
+                        yylex.(*parser).assembleVectorSelector(vs)
+                        $$ = vs
+                        }
                 ;
 
 label_matchers  : LEFT_BRACE label_match_list RIGHT_BRACE
-                        { $$ = $2 }
+                        {
+                        $$ = &VectorSelector{
+                                LabelMatchers: $2,
+                                PosRange: mergeRanges(&$1, &$3),
+                        }
+                        }
                 | LEFT_BRACE label_match_list COMMA RIGHT_BRACE
-                        { $$ = $2 }
+                        {
+                        $$ = &VectorSelector{
+                                LabelMatchers: $2,
+                                PosRange: mergeRanges(&$1, &$4),
+                        }
+                        }
                 | LEFT_BRACE RIGHT_BRACE
-                        { $$ = []*labels.Matcher{} }
+                        {
-
+                        $$ = &VectorSelector{
+                                LabelMatchers: []*labels.Matcher{},
+                                PosRange: mergeRanges(&$1, &$2),
+                        }
+                        }
                 ;
 
 label_match_list: label_match_list COMMA label_matcher
@@ -590,7 +628,14 @@ match_op        : EQL | NEQ | EQL_REGEX | NEQ_REGEX ;
  * Literals.
  */
 
-number_literal  : number { $$ = &NumberLiteral{$1}} ;
+number_literal  : NUMBER 
+                        {
+                        $$ = &NumberLiteral{
+                                Val:           yylex.(*parser).number($1.Val),
+                                PosRange: $1.PositionRange(),
+                        }
+                        }
+                ;
 
 number          : NUMBER { $$ = yylex.(*parser).number($1.Val) } ;
 
@@ -619,9 +664,14 @@ duration        : DURATION
                 ;
 
 
-string_literal  : string { $$ = &StringLiteral{$1}} ;
+string_literal  : STRING
-
+                        {
-string          : STRING { $$ = yylex.(*parser).unquoteString($1.Val) } ;
+                        $$ = &StringLiteral{
+                                Val: yylex.(*parser).unquoteString($1.Val),
+                                PosRange: $1.PositionRange(),
+                        }
+                        }
+                        ;
 
 /*
  * Wrappers for optional arguments.

promql/lex.go

@@ -213,6 +213,7 @@ const eof = -1
 type stateFn func(*Lexer) stateFn
 
 // Pos is the position in a string.
+// Negative numbers indicate undefined positions.
 type Pos int
 
 // Lexer holds the state of the scanner.

promql/parse.go

@@ -41,6 +41,10 @@ type parser struct {
 	inject    ItemType
 	injecting bool
 
+	// Everytime an Item is lexed that could be the end
+	// of certain expressions its end position is stored here.
+	lastClosing Pos
+
 	yyParser yyParserImpl
 
 	generatedParserResult interface{}
@@ -229,13 +233,15 @@ func (p *parser) Lex(lval *yySymType) int {
 		}
 	}
 
-	if typ == ERROR {
+	switch typ {
-		p.errorf("%s", lval.item.Val)
-	}
 
-	if typ == EOF {
+	case ERROR:
+		p.errorf("%s", lval.item.Val)
+	case EOF:
 		lval.item.Typ = EOF
 		p.InjectItem(0)
+	case RIGHT_BRACE, RIGHT_PAREN, RIGHT_BRACKET, DURATION:
+		p.lastClosing = lval.item.Pos + Pos(len(lval.item.Val))
 	}
 
 	return int(typ)
@@ -298,29 +304,26 @@ func (p *parser) newBinaryExpression(lhs Node, op Item, modifiers Node, rhs Node
 	return ret
 }
 
-func (p *parser) newVectorSelector(name string, labelMatchers []*labels.Matcher) *VectorSelector {
+func (p *parser) assembleVectorSelector(vs *VectorSelector) {
-	ret := &VectorSelector{LabelMatchers: labelMatchers}
+	if vs.Name != "" {
 
-	if name != "" {
+		for _, m := range vs.LabelMatchers {
-		ret.Name = name
-
-		for _, m := range ret.LabelMatchers {
 			if m.Name == labels.MetricName {
-				p.errorf("metric name must not be set twice: %q or %q", name, m.Value)
+				p.errorf("metric name must not be set twice: %q or %q", vs.Name, m.Value)
 			}
 		}
 
-		nameMatcher, err := labels.NewMatcher(labels.MatchEqual, labels.MetricName, name)
+		nameMatcher, err := labels.NewMatcher(labels.MatchEqual, labels.MetricName, vs.Name)
 		if err != nil {
 			panic(err) // Must not happen with labels.MatchEqual
 		}
-		ret.LabelMatchers = append(ret.LabelMatchers, nameMatcher)
+		vs.LabelMatchers = append(vs.LabelMatchers, nameMatcher)
 	}
 
 	// A Vector selector must contain at least one non-empty matcher to prevent
 	// implicit selection of all metrics (e.g. by a typo).
 	notEmpty := false
-	for _, lm := range ret.LabelMatchers {
+	for _, lm := range vs.LabelMatchers {
 		if !lm.Matches("") {
 			notEmpty = true
 			break
@@ -329,8 +332,6 @@ func (p *parser) newVectorSelector(name string, labelMatchers []*labels.Matcher)
 	if !notEmpty {
 		p.errorf("vector selector must contain at least one non-empty matcher")
 	}
-
-	return ret
 }
 
 func (p *parser) newAggregateExpr(op Item, modifier Node, args Node) (ret *AggregateExpr) {
@@ -364,6 +365,11 @@ func (p *parser) newAggregateExpr(op Item, modifier Node, args Node) (ret *Aggre
 
 	ret.Expr = arguments[desiredArgs-1]
 
+	ret.PosRange = PositionRange{
+		Start: op.Pos,
+		End:   p.lastClosing,
+	}
+
 	return ret
 }
 
@@ -578,14 +584,18 @@ func (p *parser) newLabelMatcher(label Item, operator Item, value Item) *labels.
 
 func (p *parser) addOffset(e Node, offset time.Duration) {
 	var offsetp *time.Duration
+	var endPosp *Pos
 
 	switch s := e.(type) {
 	case *VectorSelector:
 		offsetp = &s.Offset
+		endPosp = &s.PosRange.End
 	case *MatrixSelector:
 		offsetp = &s.VectorSelector.Offset
+		endPosp = &s.EndPos
 	case *SubqueryExpr:
 		offsetp = &s.Offset
+		endPosp = &s.EndPos
 	default:
 		p.errorf("offset modifier must be preceded by an instant or range selector, but follows a %T instead", e)
 		return
@@ -598,4 +608,6 @@ func (p *parser) addOffset(e Node, offset time.Duration) {
 		*offsetp = offset
 	}
 
+	*endPosp = p.lastClosing
+
 }