UTF-8: updates UI parser to support UTF-8 characters (#13590)

Signed-off-by: Neeraj Gartia <neerajgartia211002@gmail.com>

web/ui/module/codemirror-promql/src/complete/hybrid.test.ts

@@ -251,6 +251,12 @@ describe('analyzeCompletion test', () => {
       pos: 11, // cursor is between the bracket after the string myL
       expectedContext: [{ kind: ContextKind.LabelName }],
     },
+    {
+      title: 'continue to autocomplete QuotedLabelName in aggregate modifier',
+      expr: 'sum by ("myL")',
+      pos: 12, // cursor is between the bracket after the string myL
+      expectedContext: [{ kind: ContextKind.LabelName }],
+    },
     {
       title: 'autocomplete labelName in a list',
       expr: 'sum by (myLabel1,)',
@@ -263,6 +269,12 @@ describe('analyzeCompletion test', () => {
       pos: 23, // cursor is between the bracket after the string myLab
       expectedContext: [{ kind: ContextKind.LabelName }],
     },
+    {
+      title: 'autocomplete labelName in a list 2',
+      expr: 'sum by ("myLabel1", "myLab")',
+      pos: 27, // cursor is between the bracket after the string myLab
+      expectedContext: [{ kind: ContextKind.LabelName }],
+    },
     {
       title: 'autocomplete labelName associated to a metric',
       expr: 'metric_name{}',
@@ -299,6 +311,12 @@ describe('analyzeCompletion test', () => {
       pos: 22, // cursor is between the bracket after the comma
       expectedContext: [{ kind: ContextKind.LabelName, metricName: '' }],
     },
+    {
+      title: 'continue to autocomplete quoted labelName associated to a metric',
+      expr: '{"metric_"}',
+      pos: 10, // cursor is between the bracket after the string metric_
+      expectedContext: [{ kind: ContextKind.MetricName, metricName: 'metric_' }],
+    },
     {
       title: 'autocomplete the labelValue with metricName + labelName',
       expr: 'metric_name{labelName=""}',
@@ -342,6 +360,30 @@ describe('analyzeCompletion test', () => {
         },
       ],
     },
+    {
+      title: 'autocomplete the labelValue with metricName + quoted labelName',
+      expr: 'metric_name{labelName="labelValue", "labelName"!=""}',
+      pos: 50, // cursor is between the quotes
+      expectedContext: [
+        {
+          kind: ContextKind.LabelValue,
+          metricName: 'metric_name',
+          labelName: 'labelName',
+          matchers: [
+            {
+              name: 'labelName',
+              type: Neq,
+              value: '',
+            },
+            {
+              name: 'labelName',
+              type: EqlSingle,
+              value: 'labelValue',
+            },
+          ],
+        },
+      ],
+    },
     {
       title: 'autocomplete the labelValue associated to a labelName',
       expr: '{labelName=""}',
@@ -427,6 +469,12 @@ describe('analyzeCompletion test', () => {
       pos: 22, // cursor is after '!'
       expectedContext: [{ kind: ContextKind.MatchOp }],
     },
+    {
+      title: 'autocomplete matchOp 3',
+      expr: 'metric_name{"labelName"!}',
+      pos: 24, // cursor is after '!'
+      expectedContext: [{ kind: ContextKind.BinOp }],
+    },
     {
       title: 'autocomplete duration with offset',
       expr: 'http_requests_total offset 5',

web/ui/module/codemirror-promql/src/complete/hybrid.ts

@@ -29,7 +29,6 @@ import {
   GroupingLabels,
   Gte,
   Gtr,
-  LabelMatcher,
   LabelMatchers,
   LabelName,
   Lss,
@@ -52,6 +51,9 @@ import {
   SubqueryExpr,
   Unless,
   VectorSelector,
+  UnquotedLabelMatcher,
+  QuotedLabelMatcher,
+  QuotedLabelName,
 } from '@prometheus-io/lezer-promql';
 import { Completion, CompletionContext, CompletionResult } from '@codemirror/autocomplete';
 import { EditorState } from '@codemirror/state';
@@ -181,7 +183,10 @@ export function computeStartCompletePosition(node: SyntaxNode, pos: number): num
   let start = node.from;
   if (node.type.id === LabelMatchers || node.type.id === GroupingLabels) {
     start = computeStartCompleteLabelPositionInLabelMatcherOrInGroupingLabel(node, pos);
-  } else if (node.type.id === FunctionCallBody || (node.type.id === StringLiteral && node.parent?.type.id === LabelMatcher)) {
+  } else if (
+    node.type.id === FunctionCallBody ||
+    (node.type.id === StringLiteral && (node.parent?.type.id === UnquotedLabelMatcher || node.parent?.type.id === QuotedLabelMatcher))
+  ) {
     // When the cursor is between bracket, quote, we need to increment the starting position to avoid to consider the open bracket/ first string.
     start++;
   } else if (
@@ -212,7 +217,7 @@ export function analyzeCompletion(state: EditorState, node: SyntaxNode): Context
         result.push({ kind: ContextKind.Duration });
         break;
       }
-      if (node.parent?.type.id === LabelMatcher) {
+      if (node.parent?.type.id === UnquotedLabelMatcher || node.parent?.type.id === QuotedLabelMatcher) {
         // In this case the current token is not itself a valid match op yet:
         //      metric_name{labelName!}
         result.push({ kind: ContextKind.MatchOp });
@@ -380,7 +385,7 @@ export function analyzeCompletion(state: EditorState, node: SyntaxNode): Context
         //      sum by (myL)
         // So we have to continue to autocomplete any kind of labelName
         result.push({ kind: ContextKind.LabelName });
-      } else if (node.parent?.type.id === LabelMatcher) {
+      } else if (node.parent?.type.id === UnquotedLabelMatcher) {
         // In that case we are in the given situation:
         //       metric_name{myL} or {myL}
         // so we have or to continue to autocomplete any kind of labelName or
@@ -389,9 +394,9 @@ export function analyzeCompletion(state: EditorState, node: SyntaxNode): Context
       }
       break;
     case StringLiteral:
-      if (node.parent?.type.id === LabelMatcher) {
+      if (node.parent?.type.id === UnquotedLabelMatcher || node.parent?.type.id === QuotedLabelMatcher) {
         // In this case we are in the given situation:
-        //      metric_name{labelName=""}
+        //      metric_name{labelName=""} or metric_name{"labelName"=""}
         // So we can autocomplete the labelValue
 
         // Get the labelName.
@@ -399,18 +404,34 @@ export function analyzeCompletion(state: EditorState, node: SyntaxNode): Context
         let labelName = '';
         if (node.parent.firstChild?.type.id === LabelName) {
           labelName = state.sliceDoc(node.parent.firstChild.from, node.parent.firstChild.to);
+        } else if (node.parent.firstChild?.type.id === QuotedLabelName) {
+          labelName = state.sliceDoc(node.parent.firstChild.from, node.parent.firstChild.to).slice(1, -1);
         }
         // then find the metricName if it exists
         const metricName = getMetricNameInVectorSelector(node, state);
         // finally get the full matcher available
         const matcherNode = walkBackward(node, LabelMatchers);
-        const labelMatchers = buildLabelMatchers(matcherNode ? matcherNode.getChildren(LabelMatcher) : [], state);
+        const labelMatcherOpts = [QuotedLabelName, QuotedLabelMatcher, UnquotedLabelMatcher];
+        let labelMatchers: Matcher[] = [];
+        for (const labelMatcherOpt of labelMatcherOpts) {
+          labelMatchers = labelMatchers.concat(buildLabelMatchers(matcherNode ? matcherNode.getChildren(labelMatcherOpt) : [], state));
+        }
         result.push({
           kind: ContextKind.LabelValue,
           metricName: metricName,
           labelName: labelName,
           matchers: labelMatchers,
         });
+      } else if (node.parent?.parent?.type.id === GroupingLabels) {
+        // In this case we are in the given situation:
+        //      sum by ("myL")
+        // So we have to continue to autocomplete any kind of labelName
+        result.push({ kind: ContextKind.LabelName });
+      } else if (node.parent?.parent?.type.id === LabelMatchers) {
+        // In that case we are in the given situation:
+        //       {""} or {"metric_"}
+        // since this is for the QuotedMetricName we need to continue to autocomplete for the metric names
+        result.push({ kind: ContextKind.MetricName, metricName: state.sliceDoc(node.from, node.to).slice(1, -1) });
       }
       break;
     case NumberLiteral:

web/ui/module/codemirror-promql/src/parser/matcher.ts

@@ -12,15 +12,50 @@
 // limitations under the License.
 
 import { SyntaxNode } from '@lezer/common';
-import { EqlRegex, EqlSingle, LabelName, MatchOp, Neq, NeqRegex, StringLiteral } from '@prometheus-io/lezer-promql';
+import {
+  EqlRegex,
+  EqlSingle,
+  LabelName,
+  MatchOp,
+  Neq,
+  NeqRegex,
+  StringLiteral,
+  UnquotedLabelMatcher,
+  QuotedLabelMatcher,
+  QuotedLabelName,
+} from '@prometheus-io/lezer-promql';
 import { EditorState } from '@codemirror/state';
 import { Matcher } from '../types';
 
 function createMatcher(labelMatcher: SyntaxNode, state: EditorState): Matcher {
   const matcher = new Matcher(0, '', '');
   const cursor = labelMatcher.cursor();
+  switch (cursor.type.id) {
+    case QuotedLabelMatcher:
       if (!cursor.next()) {
-    // weird case, that would mean the labelMatcher doesn't have any child.
+        // weird case, that would mean the QuotedLabelMatcher doesn't have any child.
+        return matcher;
+      }
+      do {
+        switch (cursor.type.id) {
+          case QuotedLabelName:
+            matcher.name = state.sliceDoc(cursor.from, cursor.to).slice(1, -1);
+            break;
+          case MatchOp:
+            const ope = cursor.node.firstChild;
+            if (ope) {
+              matcher.type = ope.type.id;
+            }
+            break;
+          case StringLiteral:
+            matcher.value = state.sliceDoc(cursor.from, cursor.to).slice(1, -1);
+            break;
+        }
+      } while (cursor.nextSibling());
+      break;
+    case UnquotedLabelMatcher:
+      if (!cursor.next()) {
+        // weird case, that would mean the UnquotedLabelMatcher doesn't have any child.
         return matcher;
       }
       do {
@@ -39,6 +74,13 @@ function createMatcher(labelMatcher: SyntaxNode, state: EditorState): Matcher {
             break;
         }
       } while (cursor.nextSibling());
+      break;
+    case QuotedLabelName:
+      matcher.name = '__name__';
+      matcher.value = state.sliceDoc(cursor.from, cursor.to).slice(1, -1);
+      matcher.type = EqlSingle;
+      break;
+  }
   return matcher;
 }
 

web/ui/module/codemirror-promql/src/parser/parser.test.ts

@@ -204,6 +204,11 @@ describe('promql operations', () => {
       expectedValueType: ValueType.vector,
       expectedDiag: [] as Diagnostic[],
     },
+    {
+      expr: 'foo and on(test,"blub") bar',
+      expectedValueType: ValueType.vector,
+      expectedDiag: [] as Diagnostic[],
+    },
     {
       expr: 'foo and on() bar',
       expectedValueType: ValueType.vector,
@@ -214,6 +219,11 @@ describe('promql operations', () => {
       expectedValueType: ValueType.vector,
       expectedDiag: [] as Diagnostic[],
     },
+    {
+      expr: 'foo and ignoring(test,"blub") bar',
+      expectedValueType: ValueType.vector,
+      expectedDiag: [] as Diagnostic[],
+    },
     {
       expr: 'foo and ignoring() bar',
       expectedValueType: ValueType.vector,
@@ -229,6 +239,11 @@ describe('promql operations', () => {
       expectedValueType: ValueType.vector,
       expectedDiag: [] as Diagnostic[],
     },
+    {
+      expr: 'foo / on(test,blub) group_left("bar") bar',
+      expectedValueType: ValueType.vector,
+      expectedDiag: [] as Diagnostic[],
+    },
     {
       expr: 'foo / ignoring(test,blub) group_left(blub) bar',
       expectedValueType: ValueType.vector,
@@ -825,6 +840,134 @@ describe('promql operations', () => {
       expectedValueType: ValueType.vector,
       expectedDiag: [],
     },
+    {
+      expr: '{"foo"}',
+      expectedValueType: ValueType.vector,
+      expectedDiag: [],
+    },
+    {
+      // with metric name in the middle
+      expr: '{a="b","foo",c~="d"}',
+      expectedValueType: ValueType.vector,
+      expectedDiag: [],
+    },
+    {
+      expr: '{"foo", a="bc"}',
+      expectedValueType: ValueType.vector,
+      expectedDiag: [],
+    },
+    {
+      expr: '{"colon:in:the:middle"}',
+      expectedValueType: ValueType.vector,
+      expectedDiag: [],
+    },
+    {
+      expr: '{"dot.in.the.middle"}',
+      expectedValueType: ValueType.vector,
+      expectedDiag: [],
+    },
+    {
+      expr: '{"😀 in metric name"}',
+      expectedValueType: ValueType.vector,
+      expectedDiag: [],
+    },
+    {
+      // quotes with escape
+      expr: '{"this is \"foo\" metric"}', // eslint-disable-line
+      expectedValueType: ValueType.vector,
+      expectedDiag: [],
+    },
+    {
+      expr: '{"foo","colon:in:the:middle"="val"}',
+      expectedValueType: ValueType.vector,
+      expectedDiag: [],
+    },
+    {
+      expr: '{"foo","dot.in.the.middle"="val"}',
+      expectedValueType: ValueType.vector,
+      expectedDiag: [],
+    },
+    {
+      expr: '{"foo","😀 in label name"="val"}',
+      expectedValueType: ValueType.vector,
+      expectedDiag: [],
+    },
+    {
+      // quotes with escape
+      expr: '{"foo","this is \"bar\" label"="val"}', // eslint-disable-line
+      expectedValueType: ValueType.vector,
+      expectedDiag: [],
+    },
+    {
+      expr: 'foo{"bar"}',
+      expectedValueType: ValueType.vector,
+      expectedDiag: [
+        {
+          from: 0,
+          message: 'metric name must not be set twice: foo or bar',
+          severity: 'error',
+          to: 10,
+        },
+      ],
+    },
+    {
+      expr: '{"foo", __name__="bar"}',
+      expectedValueType: ValueType.vector,
+      expectedDiag: [
+        {
+          from: 0,
+          message: 'metric name must not be set twice: foo or bar',
+          severity: 'error',
+          to: 23,
+        },
+      ],
+    },
+    {
+      expr: '{"foo", "__name__"="bar"}',
+      expectedValueType: ValueType.vector,
+      expectedDiag: [
+        {
+          from: 0,
+          message: 'metric name must not be set twice: foo or bar',
+          severity: 'error',
+          to: 25,
+        },
+      ],
+    },
+    {
+      expr: '{"__name__"="foo", __name__="bar"}',
+      expectedValueType: ValueType.vector,
+      expectedDiag: [
+        {
+          from: 0,
+          message: 'metric name must not be set twice: foo or bar',
+          severity: 'error',
+          to: 34,
+        },
+      ],
+    },
+    {
+      expr: '{"foo", "bar"}',
+      expectedValueType: ValueType.vector,
+      expectedDiag: [
+        {
+          from: 0,
+          to: 14,
+          message: 'metric name must not be set twice: foo or bar',
+          severity: 'error',
+        },
+      ],
+    },
+    {
+      expr: `{'foo\`metric':'bar'}`, // eslint-disable-line
+      expectedValueType: ValueType.vector,
+      expectedDiag: [],
+    },
+    {
+      expr: '{`foo\"metric`=`bar`}', // eslint-disable-line
+      expectedValueType: ValueType.vector,
+      expectedDiag: [],
+    },
   ];
   testCases.forEach((value) => {
     const state = createEditorState(value.expr);

web/ui/module/codemirror-promql/src/parser/parser.ts

@@ -27,7 +27,6 @@ import {
   Gte,
   Gtr,
   Identifier,
-  LabelMatcher,
   LabelMatchers,
   Lss,
   Lte,
@@ -36,11 +35,14 @@ import {
   Or,
   ParenExpr,
   Quantile,
+  QuotedLabelMatcher,
+  QuotedLabelName,
   StepInvariantExpr,
   SubqueryExpr,
   Topk,
   UnaryExpr,
   Unless,
+  UnquotedLabelMatcher,
   VectorSelector,
 } from '@prometheus-io/lezer-promql';
 import { containsAtLeastOneChild } from './path-finder';
@@ -282,7 +284,11 @@ export class Parser {
 
   private checkVectorSelector(node: SyntaxNode): void {
     const matchList = node.getChild(LabelMatchers);
-    const labelMatchers = buildLabelMatchers(matchList ? matchList.getChildren(LabelMatcher) : [], this.state);
+    const labelMatcherOpts = [QuotedLabelName, QuotedLabelMatcher, UnquotedLabelMatcher];
+    let labelMatchers: Matcher[] = [];
+    for (const labelMatcherOpt of labelMatcherOpts) {
+      labelMatchers = labelMatchers.concat(buildLabelMatchers(matchList ? matchList.getChildren(labelMatcherOpt) : [], this.state));
+    }
     let vectorSelectorName = '';
     // VectorSelector ( Identifier )
     // https://github.com/promlabs/lezer-promql/blob/71e2f9fa5ae6f5c5547d5738966cd2512e6b99a8/src/promql.grammar#L200
@@ -301,6 +307,14 @@ export class Parser {
       // adding the metric name as a Matcher to avoid a false positive for this kind of expression:
       // foo{bare=''}
       labelMatchers.push(new Matcher(EqlSingle, '__name__', vectorSelectorName));
+    } else {
+      // In this case when metric name is not set outside the braces
+      // It is checking whether metric name is set twice like in :
+      // {__name__:"foo", "foo"}, {"foo", "bar"}
+      const labelMatchersMetricName = labelMatchers.filter((lm) => lm.name === '__name__');
+      if (labelMatchersMetricName.length > 1) {
+        this.addDiagnostic(node, `metric name must not be set twice: ${labelMatchersMetricName[0].value} or ${labelMatchersMetricName[1].value}`);
+      }
     }
 
     // A Vector selector must contain at least one non-empty matcher to prevent

web/ui/module/lezer-promql/src/promql.grammar

@@ -97,7 +97,7 @@ binModifiers {
 }
 
 GroupingLabels {
-  "(" (LabelName ("," LabelName)* ","?)? ")"
+  "(" ((LabelName | QuotedLabelName) ("," (LabelName | QuotedLabelName))* ","?)? ")"
 }
 
 FunctionCall {
@@ -220,7 +220,7 @@ VectorSelector {
 }
 
 LabelMatchers {
-  "{" (LabelMatcher ("," LabelMatcher)* ","?)? "}"
+  "{" ((UnquotedLabelMatcher | QuotedLabelMatcher | QuotedLabelName)("," (UnquotedLabelMatcher | QuotedLabelMatcher | QuotedLabelName))* ","?)? "}"
 }
 
 MatchOp {
@@ -230,10 +230,18 @@ MatchOp {
   NeqRegex
 }
 
-LabelMatcher {
+UnquotedLabelMatcher {
     LabelName MatchOp StringLiteral
 }
 
+QuotedLabelMatcher {
+    QuotedLabelName MatchOp StringLiteral
+}
+
+QuotedLabelName {
+    StringLiteral
+}
+
 StepInvariantExpr {
   expr At ( NumberLiteral | AtModifierPreprocessors "(" ")" )
 }

web/ui/module/lezer-promql/test/expression.txt

@@ -112,6 +112,54 @@ PromQL(
     )
 )
 
+# Quoted label name in grouping labels
+
+sum by("job", mode) (test_metric) / on("job") group_left sum by("job")(test_metric)
+
+==>
+
+PromQL(
+    BinaryExpr(
+        AggregateExpr(
+          AggregateOp(Sum),
+          AggregateModifier(
+            By,
+            GroupingLabels(
+                QuotedLabelName(StringLiteral),
+                LabelName
+            )
+          ),
+          FunctionCallBody(
+                VectorSelector(
+                  Identifier
+                )
+          )
+      ),
+      Div,
+        MatchingModifierClause(
+          On,
+          GroupingLabels(
+              QuotedLabelName(StringLiteral)
+          )
+          GroupLeft
+        ),
+        AggregateExpr(
+          AggregateOp(Sum),
+          AggregateModifier(
+            By,
+            GroupingLabels(
+                QuotedLabelName(StringLiteral)
+            )
+          ),
+          FunctionCallBody(
+                VectorSelector(
+                  Identifier
+                )
+          )
+        )
+    )
+)
+
 # Case insensitivity for aggregations and binop modifiers.
 
 SuM BY(testlabel1) (testmetric1) / IGNOring(testlabel2) AVG withOUT(testlabel3) (testmetric2)
@@ -226,22 +274,22 @@ PromQL(
     VectorSelector(
       Identifier,
       LabelMatchers(
-        LabelMatcher(
+        UnquotedLabelMatcher(
             LabelName,
             MatchOp(EqlSingle),
             StringLiteral
         ),
-        LabelMatcher(
+        UnquotedLabelMatcher(
             LabelName,
             MatchOp(Neq),
             StringLiteral
         ),
-        LabelMatcher(
+        UnquotedLabelMatcher(
             LabelName,
             MatchOp(EqlRegex),
             StringLiteral
         ),
-        LabelMatcher(
+        UnquotedLabelMatcher(
             LabelName,
             MatchOp(NeqRegex),
             StringLiteral
@@ -571,14 +619,14 @@ PromQL(NumberLiteral)
 NaN{foo="bar"}
 
 ==>
-PromQL(BinaryExpr(NumberLiteral,⚠,VectorSelector(LabelMatchers(LabelMatcher(LabelName,MatchOp(EqlSingle),StringLiteral)))))
+PromQL(BinaryExpr(NumberLiteral,⚠,VectorSelector(LabelMatchers(UnquotedLabelMatcher(LabelName,MatchOp(EqlSingle),StringLiteral)))))
 
 # Trying to illegally use Inf as a metric name.
 
 Inf{foo="bar"}
 
 ==>
-PromQL(BinaryExpr(NumberLiteral,⚠,VectorSelector(LabelMatchers(LabelMatcher(LabelName,MatchOp(EqlSingle),StringLiteral)))))
+PromQL(BinaryExpr(NumberLiteral,⚠,VectorSelector(LabelMatchers(UnquotedLabelMatcher(LabelName,MatchOp(EqlSingle),StringLiteral)))))
 
 # Negative offset
 
@@ -614,3 +662,24 @@ MetricName(Identifier)
 
 ==>
 PromQL(BinaryExpr(NumberLiteral,Add,BinaryExpr(VectorSelector(Identifier),Atan2,VectorSelector(Identifier))))
+
+# Testing quoted metric name
+
+{"metric_name"}
+
+==>
+PromQL(VectorSelector(LabelMatchers(QuotedLabelName(StringLiteral))))
+
+# Testing quoted label name
+
+{"foo"="bar"}
+
+==>
+PromQL(VectorSelector(LabelMatchers(QuotedLabelMatcher(QuotedLabelName(StringLiteral), MatchOp(EqlSingle), StringLiteral))))
+
+# Testing quoted metric name and label name
+
+{"metric_name", "foo"="bar"}
+
+==>
+PromQL(VectorSelector(LabelMatchers(QuotedLabelName(StringLiteral), QuotedLabelMatcher(QuotedLabelName(StringLiteral), MatchOp(EqlSingle), StringLiteral))))