From a2665a9f7384cd34052ede077c036c27fa9a8ce5 Mon Sep 17 00:00:00 2001 From: Augustin Husson <husson.augustin@gmail.com> Date: Sun, 29 Aug 2021 15:16:25 +0200 Subject: [PATCH] add lezer-promql module (#9229) * add lezer-promql module Signed-off-by: Augustin Husson <husson.augustin@gmail.com> * integrate lezer-promql in codemirror-promql Signed-off-by: Augustin Husson <husson.augustin@gmail.com> --- web/ui/module/codemirror-promql/.gitignore | 5 +- web/ui/module/codemirror-promql/build.sh | 0 .../codemirror-promql/package-lock.json | 45 +- web/ui/module/codemirror-promql/package.json | 17 +- .../src/lang-promql/complete/hybrid.test.ts | 2 +- .../src/lang-promql/complete/hybrid.ts | 2 +- .../src/lang-promql/grammar/promql.grammar | 396 ++++++++ .../lang-promql/grammar/test/expression.txt | 842 ++++++++++++++++++ .../grammar/test/test-promql.test.ts | 15 + .../src/lang-promql/grammar/tokens.js | 83 ++ .../src/lang-promql/parser/matcher.test.ts | 2 +- .../src/lang-promql/parser/matcher.ts | 2 +- .../src/lang-promql/parser/parser.ts | 2 +- .../lang-promql/parser/path-finder.test.ts | 2 +- .../src/lang-promql/parser/type.ts | 2 +- .../src/lang-promql/parser/vector.test.ts | 2 +- .../src/lang-promql/parser/vector.ts | 2 +- .../src/lang-promql/promql.ts | 2 +- .../src/lang-promql/types/function.ts | 2 +- .../src/lang-promql/types/matcher.ts | 2 +- .../codemirror-promql/src/test/utils.ts | 2 +- web/ui/module/codemirror-promql/tsconfig.json | 3 +- 22 files changed, 1394 insertions(+), 38 deletions(-) mode change 100644 => 100755 web/ui/module/codemirror-promql/build.sh create mode 100644 web/ui/module/codemirror-promql/src/lang-promql/grammar/promql.grammar create mode 100644 web/ui/module/codemirror-promql/src/lang-promql/grammar/test/expression.txt create mode 100644 web/ui/module/codemirror-promql/src/lang-promql/grammar/test/test-promql.test.ts create mode 100644 web/ui/module/codemirror-promql/src/lang-promql/grammar/tokens.js diff --git a/web/ui/module/codemirror-promql/.gitignore b/web/ui/module/codemirror-promql/.gitignore index e83fa0316a..dacf9222ae 100644 --- a/web/ui/module/codemirror-promql/.gitignore +++ b/web/ui/module/codemirror-promql/.gitignore @@ -3,6 +3,9 @@ node_modules/ dist/ lib/ -src/**/codemirror_grammar.js + +src/lang-promql/grammar/**.ts +src/lang-promql/grammar/parser.js +src/lang-promql/grammar/parser.terms.js /.nyc_output diff --git a/web/ui/module/codemirror-promql/build.sh b/web/ui/module/codemirror-promql/build.sh old mode 100644 new mode 100755 diff --git a/web/ui/module/codemirror-promql/package-lock.json b/web/ui/module/codemirror-promql/package-lock.json index 068b256b55..9eec7a216f 100644 --- a/web/ui/module/codemirror-promql/package-lock.json +++ b/web/ui/module/codemirror-promql/package-lock.json @@ -5,10 +5,9 @@ "requires": true, "packages": { "": { - "version": "0.16.0", + "version": "0.17.0", "license": "MIT", "dependencies": { - "lezer-promql": "^0.20.0", "lru-cache": "^6.0.0" }, "devDependencies": { @@ -35,6 +34,8 @@ "eslint-plugin-prettier": "^3.1.4", "html-webpack-plugin": "^4.3.0", "isomorphic-fetch": "^3.0.0", + "lezer": "^0.13.1", + "lezer-generator": "^0.13.1", "mocha": "^8.1.2", "nock": "^13.0.11", "nyc": "^15.1.0", @@ -56,7 +57,8 @@ "@codemirror/language": "^0.18.0", "@codemirror/lint": "^0.18.1", "@codemirror/state": "^0.18.2", - "@codemirror/view": "^0.18.1" + "@codemirror/view": "^0.18.1", + "lezer": "^0.13.0" } }, "node_modules/@babel/code-frame": { @@ -6333,22 +6335,28 @@ "version": "0.13.4", "resolved": "https://registry.npmjs.org/lezer/-/lezer-0.13.4.tgz", "integrity": "sha512-cLQxUVY28VBBqKBt/R8CYeH57KQnIvscAnoahzvhlZTK8qxMkIyGExR6ecEpYYDX06ZhROZrEm1IiPvjLAsTig==", + "dev": true, "dependencies": { "lezer-tree": "^0.13.2" } }, - "node_modules/lezer-promql": { - "version": "0.20.0", - "resolved": "https://registry.npmjs.org/lezer-promql/-/lezer-promql-0.20.0.tgz", - "integrity": "sha512-1CHG77fFghl032FfHT33buGyAHiTaMy2fqicEhcp2wWnbxZxS+Jt6gMzEUaf/TmRTIUJofj9uLar7iL22Jazug==", - "peerDependencies": { - "lezer": "^0.13.0" + "node_modules/lezer-generator": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/lezer-generator/-/lezer-generator-0.13.4.tgz", + "integrity": "sha512-pTWxEgw6U41jM/IwMbhPBPonrcQV5YYL3XoY4QPR7ibOjgo2RaF4wVrdabN1ILtBbGvtHZekTGyrbsqfKnMHMA==", + "dev": true, + "dependencies": { + "lezer": "^0.13.2" + }, + "bin": { + "lezer-generator": "dist/lezer-generator.cjs" } }, "node_modules/lezer-tree": { "version": "0.13.2", "resolved": "https://registry.npmjs.org/lezer-tree/-/lezer-tree-0.13.2.tgz", - "integrity": "sha512-15ZxW8TxVNAOkHIo43Iouv4zbSkQQ5chQHBpwXcD2bBFz46RB4jYLEEww5l1V0xyIx9U2clSyyrLes+hAUFrGQ==" + "integrity": "sha512-15ZxW8TxVNAOkHIo43Iouv4zbSkQQ5chQHBpwXcD2bBFz46RB4jYLEEww5l1V0xyIx9U2clSyyrLes+hAUFrGQ==", + "dev": true }, "node_modules/load-json-file": { "version": "2.0.0", @@ -17369,20 +17377,25 @@ "version": "0.13.4", "resolved": "https://registry.npmjs.org/lezer/-/lezer-0.13.4.tgz", "integrity": "sha512-cLQxUVY28VBBqKBt/R8CYeH57KQnIvscAnoahzvhlZTK8qxMkIyGExR6ecEpYYDX06ZhROZrEm1IiPvjLAsTig==", + "dev": true, "requires": { "lezer-tree": "^0.13.2" } }, - "lezer-promql": { - "version": "0.20.0", - "resolved": "https://registry.npmjs.org/lezer-promql/-/lezer-promql-0.20.0.tgz", - "integrity": "sha512-1CHG77fFghl032FfHT33buGyAHiTaMy2fqicEhcp2wWnbxZxS+Jt6gMzEUaf/TmRTIUJofj9uLar7iL22Jazug==", - "requires": {} + "lezer-generator": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/lezer-generator/-/lezer-generator-0.13.4.tgz", + "integrity": "sha512-pTWxEgw6U41jM/IwMbhPBPonrcQV5YYL3XoY4QPR7ibOjgo2RaF4wVrdabN1ILtBbGvtHZekTGyrbsqfKnMHMA==", + "dev": true, + "requires": { + "lezer": "^0.13.2" + } }, "lezer-tree": { "version": "0.13.2", "resolved": "https://registry.npmjs.org/lezer-tree/-/lezer-tree-0.13.2.tgz", - "integrity": "sha512-15ZxW8TxVNAOkHIo43Iouv4zbSkQQ5chQHBpwXcD2bBFz46RB4jYLEEww5l1V0xyIx9U2clSyyrLes+hAUFrGQ==" + "integrity": "sha512-15ZxW8TxVNAOkHIo43Iouv4zbSkQQ5chQHBpwXcD2bBFz46RB4jYLEEww5l1V0xyIx9U2clSyyrLes+hAUFrGQ==", + "dev": true }, "load-json-file": { "version": "2.0.0", diff --git a/web/ui/module/codemirror-promql/package.json b/web/ui/module/codemirror-promql/package.json index 3166078781..1980eba94d 100644 --- a/web/ui/module/codemirror-promql/package.json +++ b/web/ui/module/codemirror-promql/package.json @@ -6,11 +6,12 @@ "module": "esm/index.js", "scripts": { "start": "webpack-dev-server --config webpack.config.cjs --open", - "build": "npm run build-lib && npm run build-app", - "build-lib": "bash ./build.sh", - "build-app": "webpack --config webpack.config.cjs", - "test": "ts-mocha -p tsconfig.json src/**/*.test.ts", - "test-coverage": "nyc ts-mocha -p ./tsconfig.json ./**/*.test.ts", + "build": "npm run build:grammar && npm run build:lib && npm run build:app", + "build:grammar": "lezer-generator src/lang-promql/grammar/promql.grammar -o src/lang-promql/grammar/parser", + "build:lib": "bash ./build.sh", + "build:app": "webpack --config webpack.config.cjs", + "test": "npm run build:grammar && ts-mocha -p tsconfig.json src/**/*.test.ts", + "test-coverage": "npm run build:grammar && nyc ts-mocha -p ./tsconfig.json ./**/*.test.ts", "codecov": "nyc report --reporter=text-lcov > coverage.lcov && codecov", "lint": "eslint src/ --ext .ts", "lint:fix": "eslint --fix src/ --ext .ts" @@ -32,7 +33,6 @@ }, "homepage": "https://github.com/prometheus-community/codemirror-promql/blob/master/README.md", "dependencies": { - "lezer-promql": "^0.20.0", "lru-cache": "^6.0.0" }, "devDependencies": { @@ -59,6 +59,8 @@ "eslint-plugin-prettier": "^3.1.4", "html-webpack-plugin": "^4.3.0", "isomorphic-fetch": "^3.0.0", + "lezer": "^0.13.1", + "lezer-generator": "^0.13.1", "mocha": "^8.1.2", "nock": "^13.0.11", "nyc": "^15.1.0", @@ -77,7 +79,8 @@ "@codemirror/language": "^0.18.0", "@codemirror/lint": "^0.18.1", "@codemirror/state": "^0.18.2", - "@codemirror/view": "^0.18.1" + "@codemirror/view": "^0.18.1", + "lezer": "^0.13.0" }, "prettier": { "singleQuote": true, diff --git a/web/ui/module/codemirror-promql/src/lang-promql/complete/hybrid.test.ts b/web/ui/module/codemirror-promql/src/lang-promql/complete/hybrid.test.ts index 323c538bb9..c7925463d6 100644 --- a/web/ui/module/codemirror-promql/src/lang-promql/complete/hybrid.test.ts +++ b/web/ui/module/codemirror-promql/src/lang-promql/complete/hybrid.test.ts @@ -27,7 +27,7 @@ import { numberTerms, snippets, } from './promql.terms'; -import { EqlSingle, Neq } from 'lezer-promql'; +import { EqlSingle, Neq } from '../grammar/parser.terms'; import { syntaxTree } from '@codemirror/language'; import { newCompleteStrategy } from './index'; diff --git a/web/ui/module/codemirror-promql/src/lang-promql/complete/hybrid.ts b/web/ui/module/codemirror-promql/src/lang-promql/complete/hybrid.ts index 37de4bb11b..4e7eb7b60b 100644 --- a/web/ui/module/codemirror-promql/src/lang-promql/complete/hybrid.ts +++ b/web/ui/module/codemirror-promql/src/lang-promql/complete/hybrid.ts @@ -58,7 +58,7 @@ import { SubqueryExpr, Unless, VectorSelector, -} from 'lezer-promql'; +} from '../grammar/parser.terms'; import { Completion, CompletionContext, CompletionResult } from '@codemirror/autocomplete'; import { EditorState } from '@codemirror/state'; import { buildLabelMatchers, containsAtLeastOneChild, containsChild, retrieveAllRecursiveNodes, walkBackward, walkThrough } from '../parser'; diff --git a/web/ui/module/codemirror-promql/src/lang-promql/grammar/promql.grammar b/web/ui/module/codemirror-promql/src/lang-promql/grammar/promql.grammar new file mode 100644 index 0000000000..b5dac8bf7c --- /dev/null +++ b/web/ui/module/codemirror-promql/src/lang-promql/grammar/promql.grammar @@ -0,0 +1,396 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +@top PromQL { Expr } +@top MetricName { MetricIdentifier } + +@precedence { + pow @right, + mul @left, + add @left, + eql @left, + and @left, + or @left +} + +Expr { + AggregateExpr | + BinaryExpr | + FunctionCall | + MatrixSelector | + NumberLiteral | + OffsetExpr | + ParenExpr | + StringLiteral | + SubqueryExpr | + UnaryExpr | + VectorSelector | + StepInvariantExpr +} + +AggregateExpr { + AggregateOp AggregateModifier FunctionCallBody | + AggregateOp FunctionCallBody AggregateModifier | + AggregateOp FunctionCallBody +} + +AggregateOp { + Avg | + Bottomk | + Count | + CountValues | + Group | + Max | + Min | + Quantile | + Stddev | + Stdvar | + Sum | + Topk +} + +AggregateModifier { + By GroupingLabels | + Without GroupingLabels +} + +BinaryExpr { + Expr !pow Pow BinModifiers Expr | + Expr !mul Mul BinModifiers Expr | + Expr !mul Div BinModifiers Expr | + Expr !mul Mod BinModifiers Expr | + Expr !add Add BinModifiers Expr | + Expr !add Sub BinModifiers Expr | + Expr !eql Eql BinModifiers Expr | + Expr !eql Gte BinModifiers Expr | + Expr !eql Gtr BinModifiers Expr | + Expr !eql Lte BinModifiers Expr | + Expr !eql Lss BinModifiers Expr | + Expr !eql Neq BinModifiers Expr | + Expr !and And BinModifiers Expr | + Expr !and Unless BinModifiers Expr | + Expr !or Or BinModifiers Expr +} + +OnOrIgnoring { + Ignoring GroupingLabels | + On GroupingLabels +} + +BinModifiers { + Bool? + ( + OnOrIgnoring + ( + (GroupLeft | GroupRight) + (!mul GroupingLabels)? // TODO: Is the "!mul" here correct? Inserted it to resolve a shift/reduce conflict because we always want to count opening parenthesis after this to be counted toward this modifier, not toward a next sub-expression. + )? + )? +} + +GroupingLabels { + "(" GroupingLabelList ")" | + "(" GroupingLabelList "," ")" | + "(" ")" +} + +GroupingLabelList { + GroupingLabelList "," GroupingLabel | + GroupingLabel +} + +GroupingLabel { + LabelName +} + +FunctionCall { + FunctionIdentifier FunctionCallBody +} + +FunctionIdentifier { + AbsentOverTime | + Absent | + Abs | + AvgOverTime | + Ceil | + Changes | + Clamp | + ClampMax | + ClampMin | + CountOverTime | + DaysInMonth | + DayOfMonth | + DayOfWeek | + Delta | + Deriv | + Exp | + Floor | + HistogramQuantile | + HoltWinters | + Hour | + Idelta | + Increase | + Irate | + LabelReplace | + LabelJoin | + LastOverTime | + Ln | + Log10 | + Log2 | + MaxOverTime | + MinOverTime | + Minute | + Month | + PredictLinear | + PresentOverTime | + QuantileOverTime | + Rate | + Resets | + Round | + Scalar | + Sgn | + Sort | + SortDesc | + Sqrt | + StddevOverTime | + StdvarOverTime | + SumOverTime | + Timestamp | + Time | + Vector | + Year +} + +FunctionCallBody { + "(" FunctionCallArgs ")" | + "(" ")" +} + +FunctionCallArgs { + FunctionCallArgs "," Expr | + Expr +} + +ParenExpr { + "(" Expr ")" +} + +OffsetExpr { + Expr Offset Sub? Duration +} + +MatrixSelector { + // TODO: Can this not be more specific than "Expr"? + Expr "[" Duration "]" +} + +SubqueryExpr { + Expr "[" Duration ":" ("" | Duration) "]" +} + +UnaryExpr { + !mul UnaryOp~signed Expr +} + +UnaryOp { + "-" | + "+" +} + +VectorSelector { + MetricIdentifier LabelMatchers | + MetricIdentifier | + LabelMatchers +} + +LabelMatchers { + "{" LabelMatchList "}" | + "{" LabelMatchList "," "}" | + "{" "}" +} + +LabelMatchList { + LabelMatchList "," LabelMatcher | + LabelMatcher +} + +MatchOp { + EqlSingle | + Neq | + EqlRegex | + NeqRegex +} + +LabelMatcher { + LabelName MatchOp StringLiteral +} + +MetricIdentifier { + Identifier +} + +StepInvariantExpr { + Expr At ( NumberLiteral | AtModifierPreprocessors "(" ")" ) +} + +AtModifierPreprocessors { + Start | End +} + +NumberLiteral { + ("-"|"+")?~signed (number | inf | nan) +} + +@skip { whitespace | LineComment } + +@tokens { + whitespace { std.whitespace+ } + LineComment { "#" ![\n]* } + + number { + (std.digit+ ("." std.digit*)? | "." std.digit+) (("e" | "E") ("+" | "-")? std.digit+)? | + "0x" (std.digit | $[a-fA-F])+ + } + StringLiteral { // TODO: This is for JS, make this work for PromQL. + '"' (![\\\n"] | "\\" _)* '"'? | + "'" (![\\\n'] | "\\" _)* "'"? | + "`" ![`]* "`" + } + + Duration { + // Each line below is just the same regex repeated over and over, but each time with one of the units made non-optional, + // to ensure that at least one <number>+<unit> pair is provided and an empty string is not recognized as a valid duration. + ( ( std.digit+ "y" ) ( std.digit+ "w" )? ( std.digit+ "d" )? ( std.digit+ "h" )? ( std.digit+ "m" )? ( std.digit+ "s" )? ( std.digit+ "ms" )? ) | + ( ( std.digit+ "y" )? ( std.digit+ "w" ) ( std.digit+ "d" )? ( std.digit+ "h" )? ( std.digit+ "m" )? ( std.digit+ "s" )? ( std.digit+ "ms" )? ) | + ( ( std.digit+ "y" )? ( std.digit+ "w" )? ( std.digit+ "d" ) ( std.digit+ "h" )? ( std.digit+ "m" )? ( std.digit+ "s" )? ( std.digit+ "ms" )? ) | + ( ( std.digit+ "y" )? ( std.digit+ "w" )? ( std.digit+ "d" )? ( std.digit+ "h" ) ( std.digit+ "m" )? ( std.digit+ "s" )? ( std.digit+ "ms" )? ) | + ( ( std.digit+ "y" )? ( std.digit+ "w" )? ( std.digit+ "d" )? ( std.digit+ "h" )? ( std.digit+ "m" ) ( std.digit+ "s" )? ( std.digit+ "ms" )? ) | + ( ( std.digit+ "y" )? ( std.digit+ "w" )? ( std.digit+ "d" )? ( std.digit+ "h" )? ( std.digit+ "m" )? ( std.digit+ "s" ) ( std.digit+ "ms" )? ) | + ( ( std.digit+ "y" )? ( std.digit+ "w" )? ( std.digit+ "d" )? ( std.digit+ "h" )? ( std.digit+ "m" )? ( std.digit+ "s" )? ( std.digit+ "ms" ) ) + } + Identifier { (std.asciiLetter | "_" | ":") (std.asciiLetter | std.digit | "_" | ":" )*} + LabelName { (std.asciiLetter | "_") (std.asciiLetter | std.digit | "_")* } + + // Operator + Sub { "-" } + Add { "+" } + Mul { "*" } + Mod { "%" } + Div { "/" } + Eql { "==" } + Neq { "!=" } + Lte { "<=" } + Lss { "<" } + Gte { ">=" } + Gtr { ">" } + EqlRegex { "=~" } + EqlSingle { "=" } + NeqRegex { "!~" } + Pow { "^" } + + // Special Modifier + At { "@" } +} + +// Keywords + +@external specialize {Identifier} specializeIdentifier from "./tokens" { + inf, + nan, + Bool, + Ignoring, + On, + GroupLeft, + GroupRight, + Offset +} + +// Contextual keywords + +@external extend {Identifier} extendIdentifier from "./tokens" { + Avg, + Bottomk, + Count, + CountValues, + Group, + Max, + Min, + Quantile, + Stddev, + Stdvar, + Sum, + Topk, + By, + Without, + And, + Or, + Unless, + Start, + End +} + + // FunctionIdentifier definitions + Abs { condFn<"abs"> } + Absent { condFn<"absent"> } + AbsentOverTime { condFn<"absent_over_time"> } + AvgOverTime { condFn<"avg_over_time"> } + Ceil { condFn<"ceil"> } + Changes { condFn<"changes"> } + Clamp { condFn<"clamp"> } + ClampMax { condFn<"clamp_max"> } + ClampMin { condFn<"clamp_min"> } + CountOverTime { condFn<"count_over_time"> } + DaysInMonth { condFn<"days_in_month"> } + DayOfMonth { condFn<"day_of_month"> } + DayOfWeek { condFn<"day_of_week"> } + Delta { condFn<"delta"> } + Deriv { condFn<"deriv"> } + Exp { condFn<"exp"> } + Floor { condFn<"floor"> } + HistogramQuantile { condFn<"histogram_quantile"> } + HoltWinters { condFn<"holt_winters"> } + Hour { condFn<"hour"> } + Idelta { condFn<"idelta"> } + Increase { condFn<"increase"> } + Irate { condFn<"irate"> } + LabelReplace { condFn<"label_replace"> } + LabelJoin { condFn<"label_join"> } + LastOverTime {condFn<"last_over_time">} + Ln { condFn<"ln"> } + Log10 { condFn<"log10"> } + Log2 { condFn<"log2"> } + MaxOverTime { condFn<"max_over_time"> } + MinOverTime { condFn<"min_over_time"> } + Minute { condFn<"minute"> } + Month { condFn<"month"> } + PredictLinear { condFn<"predict_linear"> } + PresentOverTime { condFn<"present_over_time"> } + QuantileOverTime { condFn<"quantile_over_time"> } + Rate { condFn<"rate"> } + Resets { condFn<"resets"> } + Round { condFn<"round"> } + Scalar { condFn<"scalar"> } + Sgn { condFn<"sgn"> } + Sort { condFn<"sort"> } + SortDesc { condFn<"sort_desc"> } + Sqrt { condFn<"sqrt"> } + StddevOverTime { condFn<"stddev_over_time"> } + StdvarOverTime { condFn<"stdvar_over_time"> } + SumOverTime { condFn<"sum_over_time"> } + Time { condFn<"time"> } + Timestamp { condFn<"timestamp"> } + Vector { condFn<"vector"> } + Year { condFn<"year"> } + +// Conditional function names (only parsed as function names when used as such). +condFn<term> { @extend<Identifier, term> } diff --git a/web/ui/module/codemirror-promql/src/lang-promql/grammar/test/expression.txt b/web/ui/module/codemirror-promql/src/lang-promql/grammar/test/expression.txt new file mode 100644 index 0000000000..3e9883155b --- /dev/null +++ b/web/ui/module/codemirror-promql/src/lang-promql/grammar/test/expression.txt @@ -0,0 +1,842 @@ +# Numeric literals + +0.123e3 + +==> + +PromQL(Expr(NumberLiteral)) + +# Double-quoted string literal + +"test string" + +==> + +PromQL(Expr(StringLiteral)) + +# Single-quoted string literal + +'test string' + +==> + +PromQL(Expr(StringLiteral)) + +# Backtick-quoted string literal + +`test string` + +==> + +PromQL(Expr(StringLiteral)) + +# Backtick-quoted multi-line string literal + +`test + +string` + +==> + +PromQL(Expr(StringLiteral)) + +# Addition + +1 + 2 + +==> + +PromQL(Expr(BinaryExpr(Expr(NumberLiteral), Add, BinModifiers, Expr(NumberLiteral)))) + +# Complex expression + +sum by(job, mode) (rate(node_cpu_seconds_total[1m])) / on(job) group_left sum by(job)(rate(node_cpu_seconds_total[1m])) + +==> + +PromQL( + Expr( + BinaryExpr( + Expr( + AggregateExpr( + AggregateOp(Sum), + AggregateModifier( + By, + GroupingLabels( + GroupingLabelList( + GroupingLabelList( + GroupingLabel(LabelName) + ), + GroupingLabel(LabelName) + ) + ) + ), + FunctionCallBody( + FunctionCallArgs( + Expr( + FunctionCall( + FunctionIdentifier(Rate), + FunctionCallBody( + FunctionCallArgs( + Expr( + MatrixSelector( + Expr( + VectorSelector( + MetricIdentifier( + Identifier + ) + ) + ), + Duration + ) + ) + ) + ) + ) + ) + ) + ) + ) + ), + Div, + BinModifiers( + OnOrIgnoring( + On, + GroupingLabels( + GroupingLabelList( + GroupingLabel(LabelName) + ) + ) + ), + GroupLeft + ), + Expr( + AggregateExpr( + AggregateOp(Sum), + AggregateModifier( + By, + GroupingLabels( + GroupingLabelList( + GroupingLabel(LabelName) + ) + ) + ), + FunctionCallBody( + FunctionCallArgs( + Expr( + FunctionCall( + FunctionIdentifier(Rate), + FunctionCallBody( + FunctionCallArgs( + Expr( + MatrixSelector( + Expr( + VectorSelector( + MetricIdentifier( + Identifier + ) + ) + ), + Duration + ) + ) + ) + ) + ) + ) + ) + ) + ) + ) + ) + ) +) + +# Case insensitivity for aggregations and binop modifiers. + +SuM BY(testlabel1) (testmetric1) / IGNOring(testlabel2) AVG withOUT(testlabel3) (testmetric2) + +==> + +PromQL( + Expr( + BinaryExpr( + Expr( + AggregateExpr( + AggregateOp(Sum), + AggregateModifier( + By, + GroupingLabels( + GroupingLabelList( + GroupingLabel(LabelName) + ) + ) + ), + FunctionCallBody( + FunctionCallArgs( + Expr( + VectorSelector( + MetricIdentifier(Identifier) + ) + ) + ) + ) + ) + ), + Div, + BinModifiers( + OnOrIgnoring( + Ignoring, + GroupingLabels( + GroupingLabelList( + GroupingLabel(LabelName) + ) + ) + ) + ), + Expr( + AggregateExpr( + AggregateOp(Avg), + AggregateModifier( + Without, + GroupingLabels( + GroupingLabelList( + GroupingLabel(LabelName) + ) + ) + ), + FunctionCallBody( + FunctionCallArgs( + Expr( + VectorSelector( + MetricIdentifier( + Identifier + ) + ) + ) + ) + ) + ) + ) + ) + ) +) + +# Case insensitivity for set operators + +metric1 and metric2 AND metric3 unless metric4 UNLESS metric5 or metric6 OR metric7 + +==> + +PromQL( + Expr( + BinaryExpr( + Expr( + BinaryExpr( + Expr( + BinaryExpr( + Expr( + BinaryExpr( + Expr( + BinaryExpr( + Expr( + BinaryExpr( + Expr( + VectorSelector( + MetricIdentifier(Identifier) + ) + ), + And, + BinModifiers, + Expr( + VectorSelector( + MetricIdentifier(Identifier) + ) + ) + ) + ), + And, + BinModifiers, + Expr( + VectorSelector( + MetricIdentifier(Identifier) + ) + ) + ) + ), + Unless, + BinModifiers, + Expr( + VectorSelector( + MetricIdentifier(Identifier) + ) + ) + ) + ), + Unless, + BinModifiers, + Expr( + VectorSelector( + MetricIdentifier(Identifier) + ) + ) + ) + ), + Or, + BinModifiers, + Expr( + VectorSelector( + MetricIdentifier(Identifier) + ) + ) + ) + ), + Or, + BinModifiers, + Expr( + VectorSelector( + MetricIdentifier(Identifier) + ) + ) + ) + ) +) +# Duration units + +foo[1y2w3d4h5m6s7ms] + +==> + +PromQL(Expr(MatrixSelector(Expr(VectorSelector(MetricIdentifier(Identifier))),Duration))) + +# Incorrectly ordered duration units + +foo[1m2h] + +==> + +PromQL(Expr(SubqueryExpr(Expr(VectorSelector(MetricIdentifier(Identifier))),Duration,⚠,Duration))) + +# Using a function name as a metric name + +rate + +==> + +PromQL(Expr(VectorSelector(MetricIdentifier(Identifier)))) + +# Match operators + +metric_name{a="1",b!="2",c=~"3",d!~"4"} + +==> + +PromQL( + Expr( + VectorSelector( + MetricIdentifier(Identifier), + LabelMatchers( + LabelMatchList( + LabelMatchList( + LabelMatchList( + LabelMatchList( + LabelMatcher( + LabelName, + MatchOp(EqlSingle), + StringLiteral + ) + ), + LabelMatcher( + LabelName, + MatchOp(Neq), + StringLiteral + ) + ), + LabelMatcher( + LabelName, + MatchOp(EqlRegex), + StringLiteral + ) + ), + LabelMatcher( + LabelName, + MatchOp(NeqRegex), + StringLiteral + ) + ), + ) + ) + ) +) + +# Binary expression with bool modifier + +metric_name > bool 1 + +==> + +PromQL( + Expr( + BinaryExpr( + Expr( + VectorSelector( + MetricIdentifier(Identifier) + ) + ), + Gtr, + BinModifiers(Bool), + Expr(NumberLiteral) + ) + ) +) + +# Binary expression with group_x() labels. + +metric1 + on(foo) group_left(bar, baz) metric2 + +==> + +PromQL( + Expr( + BinaryExpr( + Expr( + VectorSelector( + MetricIdentifier(Identifier) + ) + ), + Add, + BinModifiers( + OnOrIgnoring( + On, + GroupingLabels( + GroupingLabelList( + GroupingLabel(LabelName) + ) + ) + ), + GroupLeft, + GroupingLabels( + GroupingLabelList( + GroupingLabelList( + GroupingLabel(LabelName) + ), + GroupingLabel(LabelName) + ) + ) + ), + Expr( + VectorSelector( + MetricIdentifier( + Identifier + ) + ) + ) + ) + ) +) + +# Function last_over_time + +last_over_time(data[1m]) + +==> +PromQL( + Expr( + FunctionCall( + FunctionIdentifier(LastOverTime), + FunctionCallBody( + FunctionCallArgs( + Expr( + MatrixSelector( + Expr( + VectorSelector( + MetricIdentifier( + Identifier + ) + ) + ), + Duration + ) + ) + ) + ) + ) + ) +) + +# Function sgn + +sgn(data) + +==> +PromQL( + Expr( + FunctionCall( + FunctionIdentifier(Sgn), + FunctionCallBody( + FunctionCallArgs( + Expr( + VectorSelector( + MetricIdentifier( + Identifier + ) + ) + ) + ) + ) + ) + ) +) + +# Function clamp + +clamp(data,0,1) + +==> +PromQL( + Expr( + FunctionCall( + FunctionIdentifier(Clamp), + FunctionCallBody( + FunctionCallArgs( + FunctionCallArgs( + FunctionCallArgs( + Expr( + VectorSelector( + MetricIdentifier( + Identifier + ) + ) + ) + ), + Expr(NumberLiteral) + ), + Expr(NumberLiteral) + ) + ) + ) + ) +) + +# Metric start + +start + +==> +PromQL(Expr(VectorSelector(MetricIdentifier(Identifier)))) + +# Metric end + +end + +==> +PromQL(Expr(VectorSelector(MetricIdentifier(Identifier)))) + +# Simple At start + +foo @ start() + +==> +PromQL( + Expr( + StepInvariantExpr( + Expr( + VectorSelector( + MetricIdentifier( + Identifier + ) + ) + ) + At, + AtModifierPreprocessors(Start), + ) + ) +) + +# Simple At end + +foo @ end() + +==> +PromQL( + Expr( + StepInvariantExpr( + Expr( + VectorSelector( + MetricIdentifier( + Identifier + ) + ) + ) + At, + AtModifierPreprocessors(End), + ) + ) +) + +# Simple At number + +foo @ 1234 + +==> +PromQL( + Expr( + StepInvariantExpr( + Expr( + VectorSelector( + MetricIdentifier( + Identifier + ) + ) + ) + At, + NumberLiteral + ) + ) +) + +# At Modifier with space between bracket + +foo @ start( ) + +==> +PromQL( + Expr( + StepInvariantExpr( + Expr( + VectorSelector( + MetricIdentifier( + Identifier + ) + ) + ) + At, + AtModifierPreprocessors(Start), + ) + ) +) + +# Complex test with At modifier + +rate(process_cpu_seconds_total[1m]) + and +topk(7, rate(process_cpu_seconds_total[1h] @ 1234)) + +==> +PromQL( + Expr( + BinaryExpr( + Expr( + FunctionCall( + FunctionIdentifier(Rate), + FunctionCallBody( + FunctionCallArgs( + Expr( + MatrixSelector( + Expr(VectorSelector(MetricIdentifier(Identifier))), + Duration + ) + ) + ) + ) + ) + ), + And, + BinModifiers, + Expr( + AggregateExpr( + AggregateOp(Topk), + FunctionCallBody( + FunctionCallArgs( + FunctionCallArgs(Expr(NumberLiteral)), + Expr( + FunctionCall( + FunctionIdentifier(Rate), + FunctionCallBody( + FunctionCallArgs( + Expr( + StepInvariantExpr( + Expr( + MatrixSelector( + Expr( + VectorSelector(MetricIdentifier(Identifier)) + ), + Duration + ) + ), + At, + NumberLiteral + ) + ) + ) + ) + ) + ) + ) + ) + ) + ) + ) + ) +) + +# At modifier with negative number + +foo @ - 1234 + +==> +PromQL( + Expr( + StepInvariantExpr( + Expr( + VectorSelector( + MetricIdentifier( + Identifier + ) + ) + ) + At, + NumberLiteral + ) + ) +) + +# At modifier with explicit positive number + +foo @ + 1234 + +==> +PromQL( + Expr( + StepInvariantExpr( + Expr( + VectorSelector( + MetricIdentifier( + Identifier + ) + ) + ) + At, + NumberLiteral + ) + ) +) + +# Metric prefixed by Inf + +infra + +==> +PromQL(Expr(VectorSelector(MetricIdentifier(Identifier)))) + +# Metric prefixed by Nan + +nananere + +==> +PromQL(Expr(VectorSelector(MetricIdentifier(Identifier)))) + +# Mixed-case NaN. + +NaN + +==> +PromQL(Expr(NumberLiteral)) + +# Lower-cased NaN. + +nan + +==> +PromQL(Expr(NumberLiteral)) + +# Inf. + +Inf + +==> +PromQL(Expr(NumberLiteral)) + +# Negative Inf. + +-Inf + +==> +PromQL(Expr(NumberLiteral)) + +# Positive Inf. + ++Inf + +==> +PromQL(Expr(NumberLiteral)) + +# Lower-cased Inf. + +inf + +==> +PromQL(Expr(NumberLiteral)) + +# Upper-cased Inf. + +INF + +==> +PromQL(Expr(NumberLiteral)) + +# Negative number literal. + +-42 + +==> +PromQL(Expr(NumberLiteral)) + +# Explicitly positive number literal. + ++42 + +==> +PromQL(Expr(NumberLiteral)) + +# Trying to illegally use NaN as a metric name. + +NaN{foo="bar"} + +==> +PromQL(Expr(BinaryExpr(Expr(NumberLiteral),⚠,BinModifiers,Expr(VectorSelector(LabelMatchers(LabelMatchList(LabelMatcher(LabelName,MatchOp(EqlSingle),StringLiteral)))))))) + +# Trying to illegally use Inf as a metric name. + +Inf{foo="bar"} + +==> +PromQL(Expr(BinaryExpr(Expr(NumberLiteral),⚠,BinModifiers,Expr(VectorSelector(LabelMatchers(LabelMatchList(LabelMatcher(LabelName,MatchOp(EqlSingle),StringLiteral)))))))) + +# Negative offset + +foo offset -5d + +==> +PromQL(Expr(OffsetExpr(Expr(VectorSelector(MetricIdentifier(Identifier))), Offset, Sub, Duration))) + +# Negative offset with space + +foo offset - 5d + +==> +PromQL(Expr(OffsetExpr(Expr(VectorSelector(MetricIdentifier(Identifier))), Offset, Sub, Duration))) + +# Positive offset + +foo offset 5d + +==> +PromQL(Expr(OffsetExpr(Expr(VectorSelector(MetricIdentifier(Identifier))), Offset, Duration))) + +# Parsing only metric names with alternative @top { "top": "MetricName" } + +sum:my_metric_name:rate5m + +==> +MetricName(MetricIdentifier(Identifier)) diff --git a/web/ui/module/codemirror-promql/src/lang-promql/grammar/test/test-promql.test.ts b/web/ui/module/codemirror-promql/src/lang-promql/grammar/test/test-promql.test.ts new file mode 100644 index 0000000000..a173e5eb35 --- /dev/null +++ b/web/ui/module/codemirror-promql/src/lang-promql/grammar/test/test-promql.test.ts @@ -0,0 +1,15 @@ +import { parser } from '../parser'; +import { fileTests } from 'lezer-generator/dist/test'; + +import * as fs from 'fs'; +import * as path from 'path'; + +const caseDir = './src/lang-promql/grammar/test'; +for (const file of fs.readdirSync(caseDir)) { + if (!/\.txt$/.test(file)) continue; + + const name = /^[^\.]*/.exec(file)[0]; + describe(name, () => { + for (const { name, run } of fileTests(fs.readFileSync(path.join(caseDir, file), 'utf8'), file)) it(name, () => run(parser)); + }); +} diff --git a/web/ui/module/codemirror-promql/src/lang-promql/grammar/tokens.js b/web/ui/module/codemirror-promql/src/lang-promql/grammar/tokens.js new file mode 100644 index 0000000000..c3353485fa --- /dev/null +++ b/web/ui/module/codemirror-promql/src/lang-promql/grammar/tokens.js @@ -0,0 +1,83 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { + And, + Avg, + Bool, + Bottomk, + By, + Count, + CountValues, + End, + Group, + GroupLeft, + GroupRight, + Ignoring, + inf, + Max, + Min, + nan, + Offset, + On, + Or, + Quantile, + Start, + Stddev, + Stdvar, + Sum, + Topk, + Unless, + Without, +} from './parser.terms.js'; + +const keywordTokens = { + inf: inf, + nan: nan, + bool: Bool, + ignoring: Ignoring, + on: On, + group_left: GroupLeft, + group_right: GroupRight, + offset: Offset, +}; + +export const specializeIdentifier = (value, stack) => { + return keywordTokens[value.toLowerCase()] || -1; +}; + +const contextualKeywordTokens = { + avg: Avg, + bottomk: Bottomk, + count: Count, + count_values: CountValues, + group: Group, + max: Max, + min: Min, + quantile: Quantile, + stddev: Stddev, + stdvar: Stdvar, + sum: Sum, + topk: Topk, + by: By, + without: Without, + and: And, + or: Or, + unless: Unless, + start: Start, + end: End, +}; + +export const extendIdentifier = (value, stack) => { + return contextualKeywordTokens[value.toLowerCase()] || -1; +}; diff --git a/web/ui/module/codemirror-promql/src/lang-promql/parser/matcher.test.ts b/web/ui/module/codemirror-promql/src/lang-promql/parser/matcher.test.ts index 538025c68e..f6539ac3bc 100644 --- a/web/ui/module/codemirror-promql/src/lang-promql/parser/matcher.test.ts +++ b/web/ui/module/codemirror-promql/src/lang-promql/parser/matcher.test.ts @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { EqlRegex, EqlSingle, Neq, NeqRegex } from 'lezer-promql'; +import { EqlRegex, EqlSingle, Neq, NeqRegex } from '../grammar/parser.terms'; import { labelMatchersToString } from './matcher'; import { Matcher } from '../types'; import chai from 'chai'; diff --git a/web/ui/module/codemirror-promql/src/lang-promql/parser/matcher.ts b/web/ui/module/codemirror-promql/src/lang-promql/parser/matcher.ts index 6bf03d352d..22421559a1 100644 --- a/web/ui/module/codemirror-promql/src/lang-promql/parser/matcher.ts +++ b/web/ui/module/codemirror-promql/src/lang-promql/parser/matcher.ts @@ -12,7 +12,7 @@ // limitations under the License. import { SyntaxNode } from 'lezer-tree'; -import { EqlRegex, EqlSingle, LabelName, MatchOp, Neq, NeqRegex, StringLiteral } from 'lezer-promql'; +import { EqlRegex, EqlSingle, LabelName, MatchOp, Neq, NeqRegex, StringLiteral } from '../grammar/parser.terms'; import { EditorState } from '@codemirror/state'; import { Matcher } from '../types'; diff --git a/web/ui/module/codemirror-promql/src/lang-promql/parser/parser.ts b/web/ui/module/codemirror-promql/src/lang-promql/parser/parser.ts index 40aeeac9d9..f9a83f3498 100644 --- a/web/ui/module/codemirror-promql/src/lang-promql/parser/parser.ts +++ b/web/ui/module/codemirror-promql/src/lang-promql/parser/parser.ts @@ -47,7 +47,7 @@ import { UnaryExpr, Unless, VectorSelector, -} from 'lezer-promql'; +} from '../grammar/parser.terms'; import { containsAtLeastOneChild, retrieveAllRecursiveNodes, walkThrough } from './path-finder'; import { getType } from './type'; import { buildLabelMatchers } from './matcher'; diff --git a/web/ui/module/codemirror-promql/src/lang-promql/parser/path-finder.test.ts b/web/ui/module/codemirror-promql/src/lang-promql/parser/path-finder.test.ts index 0adb1ac1a2..ff3e06e73e 100644 --- a/web/ui/module/codemirror-promql/src/lang-promql/parser/path-finder.test.ts +++ b/web/ui/module/codemirror-promql/src/lang-promql/parser/path-finder.test.ts @@ -32,7 +32,7 @@ import { NumberLiteral, Sub, VectorSelector, -} from 'lezer-promql'; +} from '../grammar/parser.terms'; import { createEditorState } from '../../test/utils'; import { containsAtLeastOneChild, containsChild, retrieveAllRecursiveNodes, walkBackward, walkThrough } from './path-finder'; import { SyntaxNode } from 'lezer-tree'; diff --git a/web/ui/module/codemirror-promql/src/lang-promql/parser/type.ts b/web/ui/module/codemirror-promql/src/lang-promql/parser/type.ts index 69c920d266..786bb18a30 100644 --- a/web/ui/module/codemirror-promql/src/lang-promql/parser/type.ts +++ b/web/ui/module/codemirror-promql/src/lang-promql/parser/type.ts @@ -26,7 +26,7 @@ import { SubqueryExpr, UnaryExpr, VectorSelector, -} from 'lezer-promql'; +} from '../grammar/parser.terms'; import { walkThrough } from './path-finder'; import { getFunction, ValueType } from '../types'; diff --git a/web/ui/module/codemirror-promql/src/lang-promql/parser/vector.test.ts b/web/ui/module/codemirror-promql/src/lang-promql/parser/vector.test.ts index 5ea22817ef..3f3839df8e 100644 --- a/web/ui/module/codemirror-promql/src/lang-promql/parser/vector.test.ts +++ b/web/ui/module/codemirror-promql/src/lang-promql/parser/vector.test.ts @@ -14,7 +14,7 @@ import { buildVectorMatching } from './vector'; import { createEditorState } from '../../test/utils'; import { walkThrough } from './path-finder'; -import { BinaryExpr, Expr } from 'lezer-promql'; +import { BinaryExpr, Expr } from '../grammar/parser.terms'; import chai from 'chai'; import { syntaxTree } from '@codemirror/language'; import { VectorMatchCardinality } from '../types'; diff --git a/web/ui/module/codemirror-promql/src/lang-promql/parser/vector.ts b/web/ui/module/codemirror-promql/src/lang-promql/parser/vector.ts index bc38921ae9..723dfb5c1c 100644 --- a/web/ui/module/codemirror-promql/src/lang-promql/parser/vector.ts +++ b/web/ui/module/codemirror-promql/src/lang-promql/parser/vector.ts @@ -26,7 +26,7 @@ import { OnOrIgnoring, Or, Unless, -} from 'lezer-promql'; +} from '../grammar/parser.terms'; import { VectorMatchCardinality, VectorMatching } from '../types'; import { containsAtLeastOneChild, retrieveAllRecursiveNodes } from './path-finder'; diff --git a/web/ui/module/codemirror-promql/src/lang-promql/promql.ts b/web/ui/module/codemirror-promql/src/lang-promql/promql.ts index 7d9db2ef7b..7de7206482 100644 --- a/web/ui/module/codemirror-promql/src/lang-promql/promql.ts +++ b/web/ui/module/codemirror-promql/src/lang-promql/promql.ts @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { parser } from 'lezer-promql'; +import { parser } from './grammar/parser'; import { styleTags, tags } from '@codemirror/highlight'; import { Extension } from '@codemirror/state'; import { CompleteConfiguration, CompleteStrategy, newCompleteStrategy } from './complete'; diff --git a/web/ui/module/codemirror-promql/src/lang-promql/types/function.ts b/web/ui/module/codemirror-promql/src/lang-promql/types/function.ts index f30ef41479..890ef243e8 100644 --- a/web/ui/module/codemirror-promql/src/lang-promql/types/function.ts +++ b/web/ui/module/codemirror-promql/src/lang-promql/types/function.ts @@ -63,7 +63,7 @@ import { Timestamp, Vector, Year, -} from 'lezer-promql'; +} from '../grammar/parser.terms'; export enum ValueType { none = 'none', diff --git a/web/ui/module/codemirror-promql/src/lang-promql/types/matcher.ts b/web/ui/module/codemirror-promql/src/lang-promql/types/matcher.ts index be3c06eaf5..ecbe23823f 100644 --- a/web/ui/module/codemirror-promql/src/lang-promql/types/matcher.ts +++ b/web/ui/module/codemirror-promql/src/lang-promql/types/matcher.ts @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { EqlSingle, Neq } from 'lezer-promql'; +import { EqlSingle, Neq } from '../grammar/parser.terms'; export class Matcher { type: number; diff --git a/web/ui/module/codemirror-promql/src/test/utils.ts b/web/ui/module/codemirror-promql/src/test/utils.ts index a4a4c2520f..e17f18e8ff 100644 --- a/web/ui/module/codemirror-promql/src/test/utils.ts +++ b/web/ui/module/codemirror-promql/src/test/utils.ts @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { parser } from 'lezer-promql'; +import { parser } from '../lang-promql/grammar/parser'; import { EditorState } from '@codemirror/state'; import { LezerLanguage } from '@codemirror/language'; import nock from 'nock'; diff --git a/web/ui/module/codemirror-promql/tsconfig.json b/web/ui/module/codemirror-promql/tsconfig.json index 6b36286ce6..78e1eff705 100644 --- a/web/ui/module/codemirror-promql/tsconfig.json +++ b/web/ui/module/codemirror-promql/tsconfig.json @@ -13,7 +13,8 @@ "sourceMap": true, "moduleResolution": "node", "esModuleInterop": true, - "allowSyntheticDefaultImports": true + "allowSyntheticDefaultImports": true, + "allowJs": true }, "include": [ "src/lang-promql"