Support UTF-8 metric names and labels in web UI

Fixes most of https://github.com/prometheus/prometheus/issues/15202

This should address all areas of the UI except for the autocompletion in the
codemirror-promql text editor. The strategy here is that any time we print or
internally serialize (like for the PromLens tree view) either a metric name or
a label name as part of a selector or in other relevant parts of PromQL, we
check whether it contains characters beyond what was previously supported, and
if so, quote and escape it. In the case of metric names, we also have to move
them from the beginning of the selector into the curly braces.

Signed-off-by: Julius Volz <julius.volz@gmail.com>

web/ui/mantine-ui/src/components/LabelBadges.tsx

@@ -2,6 +2,7 @@ import { Badge, BadgeVariant, Group, MantineColor, Stack } from "@mantine/core";
 import { FC } from "react";
 import { escapeString } from "../lib/escapeString";
 import badgeClasses from "../Badge.module.css";
+import { maybeQuoteLabelName } from "../promql/utils";
 
 export interface LabelBadgesProps {
   labels: Record<string, string>;
@@ -30,7 +31,7 @@ export const LabelBadges: FC<LabelBadgesProps> = ({
           }}
           key={k}
         >
-          {k}="{escapeString(v)}"
+          {maybeQuoteLabelName(k)}="{escapeString(v)}"
         </Badge>
       );
     })}

web/ui/mantine-ui/src/lib/formatSeries.ts

@@ -1,12 +1,24 @@
+import {
+  maybeQuoteLabelName,
+  metricContainsExtendedCharset,
+} from "../promql/utils";
 import { escapeString } from "./escapeString";
 
+// TODO: Maybe replace this with the new PromLens-derived serialization code in src/promql/serialize.ts?
 export const formatSeries = (labels: { [key: string]: string }): string => {
   if (labels === null) {
     return "scalar";
   }
 
+  if (metricContainsExtendedCharset(labels.__name__ || "")) {
+    return `{"${escapeString(labels.__name__)}",${Object.entries(labels)
+      .filter(([k]) => k !== "__name__")
+      .map(([k, v]) => `${maybeQuoteLabelName(k)}="${escapeString(v)}"`)
+      .join(", ")}}`;
+  }
+
   return `${labels.__name__ || ""}{${Object.entries(labels)
     .filter(([k]) => k !== "__name__")
-    .map(([k, v]) => `${k}="${escapeString(v)}"`)
+    .map(([k, v]) => `${maybeQuoteLabelName(k)}="${escapeString(v)}"`)
     .join(", ")}}`;
 };

web/ui/mantine-ui/src/pages/query/SeriesName.tsx

@@ -5,6 +5,10 @@ import classes from "./SeriesName.module.css";
 import { escapeString } from "../../lib/escapeString";
 import { useClipboard } from "@mantine/hooks";
 import { notifications } from "@mantine/notifications";
+import {
+  maybeQuoteLabelName,
+  metricContainsExtendedCharset,
+} from "../../promql/utils";
 
 interface SeriesNameProps {
   labels: { [key: string]: string } | null;
@@ -15,8 +19,26 @@ const SeriesName: FC<SeriesNameProps> = ({ labels, format }) => {
   const clipboard = useClipboard();
 
   const renderFormatted = (): React.ReactElement => {
+    const metricExtendedCharset =
+      labels && metricContainsExtendedCharset(labels.__name__ || "");
+
     const labelNodes: React.ReactElement[] = [];
     let first = true;
+
+    // If the metric name uses the extended new charset, we need to escape it,
+    // put it into the label matcher list, and make sure it's the first item.
+    if (metricExtendedCharset) {
+      labelNodes.push(
+        <span key="__name__">
+          <span className={classes.labelValue}>
+            "{escapeString(labels.__name__)}"
+          </span>
+        </span>
+      );
+
+      first = false;
+    }
+
     for (const label in labels) {
       if (label === "__name__") {
         continue;
@@ -37,7 +59,10 @@ const SeriesName: FC<SeriesNameProps> = ({ labels, format }) => {
             }}
             title="Click to copy label matcher"
           >
-            <span className={classes.labelName}>{label}</span>=
+            <span className={classes.labelName}>
+              {maybeQuoteLabelName(label)}
+            </span>
+            =
             <span className={classes.labelValue}>
               "{escapeString(labels[label])}"
             </span>
@@ -52,9 +77,11 @@ const SeriesName: FC<SeriesNameProps> = ({ labels, format }) => {
 
     return (
       <span>
-        <span className={classes.metricName}>
-          {labels ? labels.__name__ : ""}
-        </span>
+        {!metricExtendedCharset && (
+          <span className={classes.metricName}>
+            {labels ? labels.__name__ : ""}
+          </span>
+        )}
         {"{"}
         {labelNodes}
         {"}"}
@@ -62,10 +89,6 @@ const SeriesName: FC<SeriesNameProps> = ({ labels, format }) => {
     );
   };
 
-  if (labels === null) {
-    return <>scalar</>;
-  }
-
   if (format) {
     return renderFormatted();
   }

web/ui/mantine-ui/src/promql/format.tsx

@@ -8,14 +8,21 @@ import ASTNode, {
   MatrixSelector,
 } from "./ast";
 import { formatPrometheusDuration } from "../lib/formatTime";
-import { maybeParenthesizeBinopChild, escapeString } from "./utils";
+import {
+  maybeParenthesizeBinopChild,
+  escapeString,
+  maybeQuoteLabelName,
+  metricContainsExtendedCharset,
+} from "./utils";
 
 export const labelNameList = (labels: string[]): React.ReactNode[] => {
   return labels.map((l, i) => {
     return (
       <span key={i}>
         {i !== 0 && ", "}
-        <span className="promql-code promql-label-name">{l}</span>
+        <span className="promql-code promql-label-name">
+          {maybeQuoteLabelName(l)}
+        </span>
       </span>
     );
   });
@@ -69,27 +76,45 @@ const formatAtAndOffset = (
 const formatSelector = (
   node: VectorSelector | MatrixSelector
 ): ReactElement => {
-  const matchLabels = node.matchers
-    .filter(
-      (m) =>
-        !(
-          m.name === "__name__" &&
-          m.type === matchType.equal &&
-          m.value === node.name
-        )
-    )
-    .map((m, i) => (
-      <span key={i}>
-        {i !== 0 && ","}
-        <span className="promql-label-name">{m.name}</span>
-        {m.type}
-        <span className="promql-string">"{escapeString(m.value)}"</span>
+  const matchLabels: JSX.Element[] = [];
+
+  // If the metric name contains the new extended charset, we need to escape it
+  // and add it at the beginning of the matchers list in the curly braces.
+  const metricName =
+    node.name ||
+    node.matchers.find(
+      (m) => m.name === "__name__" && m.type === matchType.equal
+    )?.value ||
+    "";
+  const metricExtendedCharset = metricContainsExtendedCharset(metricName);
+  if (metricExtendedCharset) {
+    matchLabels.push(
+      <span key="__name__">
+        <span className="promql-string">"{escapeString(metricName)}"</span>
       </span>
-    ));
+    );
+  }
+
+  matchLabels.push(
+    ...node.matchers
+      .filter((m) => !(m.name === "__name__" && m.type === matchType.equal))
+      .map((m, i) => (
+        <span key={i}>
+          {(i !== 0 || metricExtendedCharset) && ","}
+          <span className="promql-label-name">
+            {maybeQuoteLabelName(m.name)}
+          </span>
+          {m.type}
+          <span className="promql-string">"{escapeString(m.value)}"</span>
+        </span>
+      ))
+  );
 
   return (
     <>
-      <span className="promql-metric-name">{node.name}</span>
+      {!metricExtendedCharset && (
+        <span className="promql-metric-name">{metricName}</span>
+      )}
       {matchLabels.length > 0 && (
         <>
           {"{"}

web/ui/mantine-ui/src/promql/serialize.ts

@@ -11,8 +11,14 @@ import {
   aggregatorsWithParam,
   maybeParenthesizeBinopChild,
   escapeString,
+  metricContainsExtendedCharset,
+  maybeQuoteLabelName,
 } from "./utils";
 
+const labelNameList = (labels: string[]): string => {
+  return labels.map((ln) => maybeQuoteLabelName(ln)).join(", ");
+};
+
 const serializeAtAndOffset = (
   timestamp: number | null,
   startOrEnd: StartOrEnd,
@@ -28,15 +34,23 @@ const serializeAtAndOffset = (
 
 const serializeSelector = (node: VectorSelector | MatrixSelector): string => {
   const matchers = node.matchers
-    .filter(
-      (m) =>
-        !(
-          m.name === "__name__" &&
-          m.type === matchType.equal &&
-          m.value === node.name
-        )
-    )
-    .map((m) => `${m.name}${m.type}"${escapeString(m.value)}"`);
+    .filter((m) => !(m.name === "__name__" && m.type === matchType.equal))
+    .map(
+      (m) => `${maybeQuoteLabelName(m.name)}${m.type}"${escapeString(m.value)}"`
+    );
+
+  // If the metric name contains the new extended charset, we need to escape it
+  // and add it at the beginning of the matchers list in the curly braces.
+  const metricName =
+    node.name ||
+    node.matchers.find(
+      (m) => m.name === "__name__" && m.type === matchType.equal
+    )?.value ||
+    "";
+  const metricExtendedCharset = metricContainsExtendedCharset(metricName);
+  if (metricExtendedCharset) {
+    matchers.unshift(`"${escapeString(metricName)}"`);
+  }
 
   const range =
     node.type === nodeType.matrixSelector
@@ -48,7 +62,7 @@ const serializeSelector = (node: VectorSelector | MatrixSelector): string => {
     node.offset
   );
 
-  return `${node.name}${matchers.length > 0 ? `{${matchers.join(",")}}` : ""}${range}${atAndOffset}`;
+  return `${!metricExtendedCharset ? metricName : ""}${matchers.length > 0 ? `{${matchers.join(",")}}` : ""}${range}${atAndOffset}`;
 };
 
 const serializeNode = (
@@ -68,9 +82,9 @@ const serializeNode = (
     case nodeType.aggregation:
       return `${initialInd}${node.op}${
         node.without
-          ? ` without(${node.grouping.join(", ")}) `
+          ? ` without(${labelNameList(node.grouping)}) `
           : node.grouping.length > 0
-            ? ` by(${node.grouping.join(", ")}) `
+            ? ` by(${labelNameList(node.grouping)}) `
             : ""
       }(${childListSeparator}${
         aggregatorsWithParam.includes(node.op) && node.param !== null
@@ -119,16 +133,16 @@ const serializeNode = (
       const vm = node.matching;
       if (vm !== null && (vm.labels.length > 0 || vm.on)) {
         if (vm.on) {
-          matching = ` on(${vm.labels.join(", ")})`;
+          matching = ` on(${labelNameList(vm.labels)})`;
         } else {
-          matching = ` ignoring(${vm.labels.join(", ")})`;
+          matching = ` ignoring(${labelNameList(vm.labels)})`;
         }
 
         if (
           vm.card === vectorMatchCardinality.manyToOne ||
           vm.card === vectorMatchCardinality.oneToMany
         ) {
-          grouping = ` group_${vm.card === vectorMatchCardinality.manyToOne ? "left" : "right"}(${vm.include.join(",")})`;
+          grouping = ` group_${vm.card === vectorMatchCardinality.manyToOne ? "left" : "right"}(${labelNameList(vm.include)})`;
         }
       }
 

web/ui/mantine-ui/src/promql/serializeAndFormat.test.ts

@@ -99,7 +99,7 @@ describe("serializeNode and formatNode", () => {
           timestamp: null,
           startOrEnd: null,
         },
-        output: '{__name__="metric_name"} offset 1m',
+        output: "metric_name offset 1m",
       },
       {
         // Escaping in label values.
@@ -642,6 +642,113 @@ describe("serializeNode and formatNode", () => {
 == bool on(label1, label2) group_right(label3)
   …`,
       },
+      // Test new Prometheus 3.0 UTF-8 support.
+      {
+        node: {
+          bool: false,
+          lhs: {
+            bool: false,
+            lhs: {
+              expr: {
+                matchers: [
+                  {
+                    name: "__name__",
+                    type: matchType.equal,
+                    value: "metric_ä",
+                  },
+                  {
+                    name: "foo",
+                    type: matchType.equal,
+                    value: "bar",
+                  },
+                ],
+                name: "",
+                offset: 0,
+                startOrEnd: null,
+                timestamp: null,
+                type: nodeType.vectorSelector,
+              },
+              grouping: ["a", "ä"],
+              op: aggregationType.sum,
+              param: null,
+              type: nodeType.aggregation,
+              without: false,
+            },
+            matching: {
+              card: vectorMatchCardinality.manyToOne,
+              include: ["c", "ü"],
+              labels: ["b", "ö"],
+              on: true,
+            },
+            op: binaryOperatorType.div,
+            rhs: {
+              expr: {
+                matchers: [
+                  {
+                    name: "__name__",
+                    type: matchType.equal,
+                    value: "metric_ö",
+                  },
+                  {
+                    name: "bar",
+                    type: matchType.equal,
+                    value: "foo",
+                  },
+                ],
+                name: "",
+                offset: 0,
+                startOrEnd: null,
+                timestamp: null,
+                type: nodeType.vectorSelector,
+              },
+              grouping: ["d", "ä"],
+              op: aggregationType.sum,
+              param: null,
+              type: nodeType.aggregation,
+              without: true,
+            },
+            type: nodeType.binaryExpr,
+          },
+          matching: {
+            card: vectorMatchCardinality.oneToOne,
+            include: [],
+            labels: ["e", "ö"],
+            on: false,
+          },
+          op: binaryOperatorType.add,
+          rhs: {
+            expr: {
+              matchers: [
+                {
+                  name: "__name__",
+                  type: matchType.equal,
+                  value: "metric_ü",
+                },
+              ],
+              name: "",
+              offset: 0,
+              startOrEnd: null,
+              timestamp: null,
+              type: nodeType.vectorSelector,
+            },
+            type: nodeType.parenExpr,
+          },
+          type: nodeType.binaryExpr,
+        },
+        output:
+          'sum by(a, "ä") ({"metric_ä",foo="bar"}) / on(b, "ö") group_left(c, "ü") sum without(d, "ä") ({"metric_ö",bar="foo"}) + ignoring(e, "ö") ({"metric_ü"})',
+        prettyOutput: `    sum by(a, "ä") (
+      {"metric_ä",foo="bar"}
+    )
+  / on(b, "ö") group_left(c, "ü")
+    sum without(d, "ä") (
+      {"metric_ö",bar="foo"}
+    )
++ ignoring(e, "ö")
+  (
+    {"metric_ü"}
+  )`,
+      },
     ];
 
     tests.forEach((t) => {

web/ui/mantine-ui/src/promql/utils.ts

@@ -267,6 +267,21 @@ export const humanizedValueType: Record<valueType, string> = {
   [valueType.matrix]: "range vector",
 };
 
+const metricNameRe = /^[a-zA-Z_:][a-zA-Z0-9_:]*$/;
+const labelNameCharsetRe = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
+
+export const metricContainsExtendedCharset = (str: string) => {
+  return !metricNameRe.test(str);
+};
+
+export const labelNameContainsExtendedCharset = (str: string) => {
+  return !labelNameCharsetRe.test(str);
+};
+
 export const escapeString = (str: string) => {
   return str.replace(/([\\"])/g, "\\$1");
 };
+
+export const maybeQuoteLabelName = (str: string) => {
+  return labelNameContainsExtendedCharset(str) ? `"${escapeString(str)}"` : str;
+};