From 8d4bcd2c773c159ae6e87c9bc7b9517bfb92835d Mon Sep 17 00:00:00 2001 From: Owen Williams Date: Tue, 3 Dec 2024 12:33:10 -0500 Subject: [PATCH] promql: Fix various UTF-8 bugs related to quoting Fixes UTF-8 aggregator label list items getting mutated with quote marks when String-ified. Fixes quoted metric names not supported in metric declarations. Fixes UTF-8 label names not being quoted when String-ified. Fixes https://github.com/prometheus/prometheus/issues/15470 Fixes https://github.com/prometheus/prometheus/issues/15528 Signed-off-by: Owen Williams Co-authored-by: Bryan Boreham --- model/labels/labels_common.go | 6 +- model/labels/labels_test.go | 4 + promql/engine_test.go | 2 +- promql/parser/generated_parser.y | 4 +- promql/parser/generated_parser.y.go | 333 ++++++++++++++-------------- promql/parser/parse.go | 3 +- promql/parser/parse_test.go | 80 ++++++- promql/parser/printer.go | 17 +- promql/promqltest/test.go | 4 + promql/promqltest/test_test.go | 28 ++- storage/remote/codec_test.go | 5 + 11 files changed, 295 insertions(+), 191 deletions(-) diff --git a/model/labels/labels_common.go b/model/labels/labels_common.go index 99529a383..a232eeea5 100644 --- a/model/labels/labels_common.go +++ b/model/labels/labels_common.go @@ -51,7 +51,11 @@ func (ls Labels) String() string { b.WriteByte(',') b.WriteByte(' ') } - b.WriteString(l.Name) + if !model.LabelName(l.Name).IsValidLegacy() { + b.Write(strconv.AppendQuote(b.AvailableBuffer(), l.Name)) + } else { + b.WriteString(l.Name) + } b.WriteByte('=') b.Write(strconv.AppendQuote(b.AvailableBuffer(), l.Value)) i++ diff --git a/model/labels/labels_test.go b/model/labels/labels_test.go index 3a4040776..06802d6f3 100644 --- a/model/labels/labels_test.go +++ b/model/labels/labels_test.go @@ -39,6 +39,10 @@ func TestLabels_String(t *testing.T) { labels: Labels{}, expected: "{}", }, + { + labels: FromStrings("service.name", "t1", "whatever\\whatever", "t2"), + expected: `{"service.name"="t1", "whatever\\whatever"="t2"}`, + }, } for _, c := range cases { str := c.labels.String() diff --git a/promql/engine_test.go b/promql/engine_test.go index 8b8dc3909..20d78a7d8 100644 --- a/promql/engine_test.go +++ b/promql/engine_test.go @@ -751,7 +751,7 @@ load 10s Interval: 5 * time.Second, }, { - Query: `count_values("wrong label!", metric)`, + Query: `count_values("wrong label!\xff", metric)`, ShouldError: true, }, } diff --git a/promql/parser/generated_parser.y b/promql/parser/generated_parser.y index c321a1e97..3865dc654 100644 --- a/promql/parser/generated_parser.y +++ b/promql/parser/generated_parser.y @@ -669,14 +669,14 @@ label_set_item : IDENTIFIER EQL STRING { $$ = labels.Label{Name: $1.Val, Value: yylex.(*parser).unquoteString($3.Val) } } | string_identifier EQL STRING { $$ = labels.Label{Name: $1.Val, Value: yylex.(*parser).unquoteString($3.Val) } } + | string_identifier + { $$ = labels.Label{Name: labels.MetricName, Value: $1.Val} } | IDENTIFIER EQL error { yylex.(*parser).unexpected("label set", "string"); $$ = labels.Label{}} | string_identifier EQL error { yylex.(*parser).unexpected("label set", "string"); $$ = labels.Label{}} | IDENTIFIER error { yylex.(*parser).unexpected("label set", "\"=\""); $$ = labels.Label{}} - | string_identifier error - { yylex.(*parser).unexpected("label set", "\"=\""); $$ = labels.Label{}} | error { yylex.(*parser).unexpected("label set", "identifier or \"}\""); $$ = labels.Label{} } ; diff --git a/promql/parser/generated_parser.y.go b/promql/parser/generated_parser.y.go index 8979410ce..7ff859116 100644 --- a/promql/parser/generated_parser.y.go +++ b/promql/parser/generated_parser.y.go @@ -339,7 +339,7 @@ var yyExca = [...]int16{ 79, 197, 85, 197, -2, 125, - -1, 205, + -1, 204, 9, 246, 12, 246, 13, 246, @@ -371,7 +371,7 @@ var yyExca = [...]int16{ 88, 246, 89, 246, -2, 0, - -1, 206, + -1, 205, 9, 246, 12, 246, 13, 246, @@ -407,139 +407,139 @@ var yyExca = [...]int16{ const yyPrivate = 57344 -const yyLast = 804 +const yyLast = 803 var yyAct = [...]int16{ - 155, 339, 337, 158, 344, 231, 39, 197, 281, 44, - 296, 295, 84, 120, 82, 181, 109, 108, 351, 352, - 353, 354, 107, 111, 203, 136, 204, 159, 154, 112, - 205, 206, 234, 6, 271, 55, 163, 163, 107, 334, - 333, 307, 244, 275, 309, 54, 162, 162, 250, 363, - 91, 272, 330, 131, 362, 233, 60, 270, 276, 110, - 100, 101, 298, 115, 103, 116, 106, 90, 164, 164, - 114, 265, 113, 361, 277, 307, 360, 246, 247, 338, - 103, 248, 106, 153, 165, 165, 264, 316, 201, 261, - 122, 105, 235, 237, 239, 240, 241, 249, 251, 254, - 255, 256, 257, 258, 262, 263, 273, 105, 236, 238, - 242, 243, 245, 252, 253, 152, 117, 166, 259, 260, - 176, 164, 170, 173, 163, 168, 223, 169, 172, 2, - 3, 4, 5, 107, 162, 199, 111, 165, 187, 202, - 189, 171, 112, 269, 207, 208, 209, 210, 211, 212, - 213, 214, 215, 216, 217, 218, 219, 220, 221, 200, - 89, 91, 113, 222, 123, 193, 268, 329, 224, 225, - 183, 100, 101, 191, 121, 103, 104, 106, 90, 7, - 85, 234, 266, 182, 55, 183, 328, 86, 192, 123, - 83, 244, 122, 267, 54, 132, 190, 250, 188, 121, - 345, 230, 105, 86, 233, 77, 35, 119, 304, 10, - 185, 327, 86, 303, 293, 294, 157, 315, 297, 79, - 184, 186, 326, 163, 274, 185, 246, 247, 302, 325, - 248, 324, 314, 162, 323, 184, 186, 299, 261, 313, - 322, 235, 237, 239, 240, 241, 249, 251, 254, 255, - 256, 257, 258, 262, 263, 164, 321, 236, 238, 242, - 243, 245, 252, 253, 180, 126, 320, 259, 260, 179, - 125, 165, 305, 319, 306, 308, 318, 310, 317, 130, - 88, 129, 178, 124, 311, 312, 137, 138, 139, 140, + 154, 338, 336, 157, 343, 230, 39, 196, 280, 44, + 295, 294, 84, 120, 82, 233, 180, 109, 108, 350, + 351, 352, 353, 110, 111, 243, 202, 158, 203, 135, + 112, 249, 361, 6, 333, 329, 113, 332, 232, 204, + 205, 308, 271, 60, 130, 270, 297, 268, 162, 315, + 156, 360, 153, 306, 359, 344, 200, 162, 161, 55, + 245, 246, 222, 115, 247, 116, 107, 161, 269, 54, + 267, 114, 260, 306, 182, 234, 236, 238, 239, 240, + 248, 250, 253, 254, 255, 256, 257, 261, 262, 163, + 122, 235, 237, 241, 242, 244, 251, 252, 192, 328, + 111, 258, 259, 117, 190, 164, 112, 152, 103, 55, + 106, 337, 77, 113, 184, 151, 35, 165, 327, 54, + 175, 191, 169, 172, 183, 185, 167, 189, 168, 2, + 3, 4, 5, 107, 198, 105, 159, 160, 201, 186, + 188, 7, 326, 206, 207, 208, 209, 210, 211, 212, + 213, 214, 215, 216, 217, 218, 219, 220, 199, 194, + 89, 91, 221, 162, 264, 325, 197, 223, 224, 171, + 200, 100, 101, 161, 162, 103, 104, 106, 90, 263, + 233, 324, 170, 162, 161, 323, 362, 322, 321, 274, + 243, 122, 266, 161, 131, 163, 249, 272, 123, 320, + 229, 319, 105, 232, 275, 318, 163, 317, 121, 85, + 316, 164, 163, 292, 293, 163, 265, 296, 129, 83, + 276, 86, 164, 273, 10, 245, 246, 187, 164, 247, + 88, 164, 86, 50, 79, 36, 298, 260, 1, 78, + 234, 236, 238, 239, 240, 248, 250, 253, 254, 255, + 256, 257, 261, 262, 123, 49, 235, 237, 241, 242, + 244, 251, 252, 181, 121, 182, 258, 259, 128, 48, + 127, 304, 119, 305, 307, 59, 309, 86, 9, 9, + 47, 46, 134, 310, 311, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, - 151, 195, 160, 161, 50, 163, 36, 167, 198, 331, - 78, 332, 201, 228, 55, 162, 85, 227, 1, 340, - 341, 342, 336, 49, 54, 343, 83, 347, 346, 349, - 348, 48, 226, 47, 81, 355, 356, 164, 55, 86, - 357, 53, 77, 301, 56, 8, 359, 22, 54, 37, - 55, 175, 46, 165, 57, 128, 135, 127, 45, 43, - 54, 364, 300, 59, 133, 174, 9, 9, 42, 134, - 75, 41, 40, 51, 196, 358, 18, 19, 278, 87, - 20, 194, 229, 80, 350, 156, 76, 58, 232, 52, - 118, 61, 62, 63, 64, 65, 66, 67, 68, 69, - 70, 71, 72, 73, 74, 0, 0, 0, 13, 0, - 0, 0, 24, 0, 30, 0, 0, 31, 32, 55, - 38, 0, 53, 77, 0, 56, 280, 0, 22, 54, - 0, 0, 0, 279, 0, 57, 0, 283, 284, 282, - 289, 291, 288, 290, 285, 286, 287, 292, 0, 0, - 0, 75, 0, 0, 0, 0, 0, 18, 19, 0, - 0, 20, 0, 0, 0, 0, 0, 76, 0, 0, - 0, 0, 61, 62, 63, 64, 65, 66, 67, 68, - 69, 70, 71, 72, 73, 74, 0, 0, 0, 13, - 0, 0, 0, 24, 0, 30, 0, 55, 31, 32, - 53, 77, 0, 56, 335, 0, 22, 54, 0, 0, - 0, 0, 0, 57, 0, 283, 284, 282, 289, 291, - 288, 290, 285, 286, 287, 292, 0, 0, 0, 75, - 0, 0, 0, 0, 0, 18, 19, 0, 0, 20, - 0, 0, 0, 17, 77, 76, 0, 0, 0, 22, + 45, 43, 132, 173, 179, 184, 166, 85, 330, 178, + 331, 42, 133, 55, 41, 183, 185, 83, 339, 340, + 341, 335, 177, 54, 342, 81, 346, 345, 348, 347, + 86, 303, 40, 314, 354, 355, 302, 55, 51, 356, + 53, 77, 300, 56, 195, 358, 22, 54, 313, 55, + 174, 301, 227, 57, 8, 312, 226, 357, 37, 54, + 363, 299, 126, 277, 87, 193, 228, 125, 80, 75, + 349, 225, 155, 58, 231, 18, 19, 52, 118, 20, + 124, 0, 0, 0, 0, 76, 0, 0, 0, 0, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 0, 0, 0, 13, 0, 0, - 0, 24, 0, 30, 0, 0, 31, 32, 18, 19, - 0, 0, 20, 0, 0, 0, 17, 35, 0, 0, - 0, 0, 22, 11, 12, 14, 15, 16, 21, 23, - 25, 26, 27, 28, 29, 33, 34, 0, 0, 0, - 13, 0, 0, 0, 24, 0, 30, 0, 0, 31, - 32, 18, 19, 0, 0, 20, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 11, 12, 14, 15, - 16, 21, 23, 25, 26, 27, 28, 29, 33, 34, - 107, 0, 0, 13, 0, 0, 0, 24, 177, 30, - 0, 0, 31, 32, 0, 0, 0, 0, 0, 107, - 0, 0, 0, 0, 0, 0, 0, 89, 91, 92, - 0, 93, 94, 95, 96, 97, 98, 99, 100, 101, - 102, 0, 103, 104, 106, 90, 89, 91, 92, 0, + 0, 24, 0, 30, 0, 0, 31, 32, 55, 38, + 107, 53, 77, 0, 56, 279, 0, 22, 54, 0, + 0, 0, 278, 0, 57, 0, 282, 283, 281, 288, + 290, 287, 289, 284, 285, 286, 291, 0, 91, 0, + 75, 0, 0, 0, 0, 0, 18, 19, 100, 101, + 20, 0, 103, 0, 106, 90, 76, 0, 0, 0, + 0, 61, 62, 63, 64, 65, 66, 67, 68, 69, + 70, 71, 72, 73, 74, 0, 0, 0, 13, 105, + 0, 0, 24, 0, 30, 0, 55, 31, 32, 53, + 77, 0, 56, 334, 0, 22, 54, 0, 0, 0, + 0, 0, 57, 0, 282, 283, 281, 288, 290, 287, + 289, 284, 285, 286, 291, 0, 0, 0, 75, 0, + 0, 0, 0, 0, 18, 19, 0, 0, 20, 0, + 0, 0, 17, 77, 76, 0, 0, 0, 22, 61, + 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, + 72, 73, 74, 0, 0, 0, 13, 0, 0, 0, + 24, 0, 30, 0, 0, 31, 32, 18, 19, 0, + 0, 20, 0, 0, 0, 17, 35, 0, 0, 0, + 0, 22, 11, 12, 14, 15, 16, 21, 23, 25, + 26, 27, 28, 29, 33, 34, 0, 0, 0, 13, + 0, 0, 0, 24, 0, 30, 0, 0, 31, 32, + 18, 19, 0, 0, 20, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 11, 12, 14, 15, 16, + 21, 23, 25, 26, 27, 28, 29, 33, 34, 107, + 0, 0, 13, 0, 0, 0, 24, 176, 30, 0, + 0, 31, 32, 0, 0, 0, 0, 0, 107, 0, + 0, 0, 0, 0, 0, 0, 89, 91, 92, 0, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, - 0, 103, 104, 106, 90, 107, 0, 0, 0, 105, + 0, 103, 104, 106, 90, 89, 91, 92, 0, 93, + 94, 95, 96, 97, 98, 99, 100, 101, 102, 0, + 103, 104, 106, 90, 107, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 107, 0, 0, 0, 105, 0, - 0, 0, 89, 91, 92, 0, 93, 94, 95, 0, - 97, 98, 99, 100, 101, 102, 0, 103, 104, 106, - 90, 89, 91, 92, 0, 93, 94, 0, 0, 97, - 98, 0, 100, 101, 102, 0, 103, 104, 106, 90, - 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, + 0, 0, 0, 107, 0, 0, 0, 105, 0, 0, + 0, 89, 91, 92, 0, 93, 94, 95, 0, 97, + 98, 99, 100, 101, 102, 0, 103, 104, 106, 90, + 89, 91, 92, 0, 93, 94, 0, 0, 97, 98, + 0, 100, 101, 102, 0, 103, 104, 106, 90, 0, + 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 105, + 0, 0, 105, } var yyPact = [...]int16{ - 31, 169, 574, 574, 410, 531, -1000, -1000, -1000, 193, + 31, 131, 573, 573, 409, 530, -1000, -1000, -1000, 103, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, 314, -1000, 278, -1000, 655, + -1000, -1000, -1000, -1000, -1000, 305, -1000, 228, -1000, 654, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, 57, 147, -1000, -1000, 488, -1000, 488, 192, + -1000, -1000, 21, 98, -1000, -1000, 487, -1000, 487, 99, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, 187, -1000, -1000, - 263, -1000, -1000, 353, 277, -1000, -1000, 29, -1000, -53, - -53, -53, -53, -53, -53, -53, -53, -53, -53, -53, - -53, -53, -53, -53, -53, 26, 214, 305, 147, -56, - -1000, 126, 126, 329, -1000, 636, 24, -1000, 262, -1000, - -1000, 181, 166, -1000, -1000, 178, -1000, 171, -1000, 163, - -1000, 296, 488, -1000, -58, -50, -1000, 488, 488, 488, - 488, 488, 488, 488, 488, 488, 488, 488, 488, 488, - 488, 488, -1000, 175, -1000, -1000, 111, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, 115, 115, 311, -1000, -1000, -1000, - -1000, 179, -1000, -1000, 64, -1000, 655, -1000, -1000, 162, - -1000, 141, -1000, -1000, -1000, -1000, -1000, 32, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, 25, 80, 17, -1000, -1000, - -1000, 409, 8, 126, 126, 126, 126, 24, 24, 119, - 119, 119, 720, 701, 119, 119, 720, 24, 24, 119, - 24, 8, -1000, 40, -1000, -1000, -1000, 341, -1000, 206, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, 252, -1000, -1000, + 360, -1000, -1000, 266, 214, -1000, -1000, 20, -1000, -49, + -49, -49, -49, -49, -49, -49, -49, -49, -49, -49, + -49, -49, -49, -49, -49, 50, 48, 304, 98, -55, + -1000, 167, 167, 328, -1000, 635, 52, -1000, 302, -1000, + -1000, 261, 70, -1000, -1000, 207, -1000, 102, -1000, 96, + 154, 487, -1000, -56, -41, -1000, 487, 487, 487, 487, + 487, 487, 487, 487, 487, 487, 487, 487, 487, 487, + 487, -1000, 100, -1000, -1000, 47, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, 39, 39, 350, -1000, -1000, -1000, -1000, + 178, -1000, -1000, 157, -1000, 654, -1000, -1000, 196, -1000, + 45, -1000, -1000, -1000, -1000, -1000, 43, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, 16, 171, 163, -1000, -1000, -1000, + 408, 406, 167, 167, 167, 167, 52, 52, 119, 119, + 119, 719, 700, 119, 119, 719, 52, 52, 119, 52, + 406, -1000, 24, -1000, -1000, -1000, 340, -1000, 329, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, 488, -1000, -1000, -1000, -1000, - -1000, -1000, 56, 56, 18, 56, 72, 72, 215, 70, - -1000, -1000, 272, 270, 267, 260, 250, 234, 228, 225, - 223, 216, 205, -1000, -1000, -1000, -1000, -1000, -1000, 165, - -1000, -1000, -1000, 30, -1000, 655, -1000, -1000, -1000, 56, - -1000, 14, 13, 487, -1000, -1000, -1000, 22, 27, 27, - 27, 115, 186, 186, 22, 186, 22, -74, -1000, -1000, - -1000, -1000, -1000, 56, 56, -1000, -1000, -1000, 56, -1000, - -1000, -1000, -1000, -1000, -1000, 27, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 52, -1000, - 28, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, 487, -1000, -1000, -1000, -1000, -1000, + -1000, 34, 34, 15, 34, 40, 40, 331, 32, -1000, + -1000, 204, 201, 199, 195, 193, 182, 181, 179, 175, + 159, 136, -1000, -1000, -1000, -1000, -1000, -1000, 97, -1000, + -1000, -1000, 13, -1000, 654, -1000, -1000, -1000, 34, -1000, + 11, 8, 486, -1000, -1000, -1000, 54, 174, 174, 174, + 39, 41, 41, 54, 41, 54, -73, -1000, -1000, -1000, + -1000, -1000, 34, 34, -1000, -1000, -1000, 34, -1000, -1000, + -1000, -1000, -1000, -1000, 174, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, 30, -1000, 165, + -1000, -1000, -1000, -1000, } var yyPgo = [...]int16{ - 0, 390, 13, 389, 5, 15, 388, 363, 387, 385, - 12, 384, 209, 345, 383, 14, 382, 10, 11, 381, - 379, 7, 378, 8, 4, 375, 2, 1, 3, 374, - 27, 0, 373, 372, 17, 195, 371, 369, 6, 368, - 365, 16, 364, 56, 359, 9, 358, 356, 352, 333, - 331, 323, 304, 318, 306, + 0, 378, 13, 377, 5, 16, 374, 275, 373, 372, + 12, 370, 224, 354, 368, 14, 366, 10, 11, 365, + 364, 7, 363, 8, 4, 357, 2, 1, 3, 344, + 27, 0, 338, 332, 18, 194, 314, 312, 6, 311, + 303, 17, 302, 43, 301, 9, 300, 282, 281, 280, + 269, 255, 233, 238, 235, } var yyR1 = [...]int8{ @@ -584,7 +584,7 @@ var yyR2 = [...]int8{ 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 4, 2, 0, - 3, 1, 2, 3, 3, 3, 3, 2, 2, 1, + 3, 1, 2, 3, 3, 1, 3, 3, 2, 1, 2, 0, 3, 2, 1, 1, 3, 1, 3, 4, 1, 3, 5, 5, 1, 1, 1, 4, 3, 3, 2, 3, 1, 2, 3, 3, 3, 3, 3, 3, @@ -612,30 +612,30 @@ var yyChk = [...]int16{ 52, 53, 54, 56, 57, 83, 58, 14, -34, -41, 2, 79, 85, 15, -41, -38, -38, -43, -1, 20, -2, 12, -10, 2, 20, 7, 2, 4, 2, 4, - 2, 24, -35, -42, -37, -47, 78, -35, -35, -35, + 24, -35, -42, -37, -47, 78, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, - -35, -35, -45, 57, 2, -31, -9, 2, -28, -30, - 88, 89, 19, 9, 41, 57, -45, 2, -41, -34, - -17, 15, 2, -17, -40, 22, -38, 22, 20, 7, - 2, -5, 2, 4, 54, 44, 55, -5, 20, -15, - 25, 2, 25, 2, -19, 5, -29, -21, 12, -28, - -30, 16, -38, 82, 84, 80, 81, -38, -38, -38, + -35, -45, 57, 2, -31, -9, 2, -28, -30, 88, + 89, 19, 9, 41, 57, -45, 2, -41, -34, -17, + 15, 2, -17, -40, 22, -38, 22, 20, 7, 2, + -5, 2, 4, 54, 44, 55, -5, 20, -15, 25, + 2, 25, 2, -19, 5, -29, -21, 12, -28, -30, + 16, -38, 82, 84, 80, 81, -38, -38, -38, -38, -38, -38, -38, -38, -38, -38, -38, -38, -38, -38, - -38, -38, -45, 15, -28, -28, 21, 6, 2, -16, - 22, -4, -6, 25, 2, 62, 78, 63, 79, 64, - 65, 66, 80, 81, 12, 82, 47, 48, 51, 67, - 18, 68, 83, 84, 69, 70, 71, 72, 73, 88, - 89, 59, 74, 75, 22, 7, 20, -2, 25, 2, - 25, 2, 26, 26, -30, 26, 41, 57, -22, 24, - 17, -23, 30, 28, 29, 35, 36, 37, 33, 31, - 34, 32, 38, -17, -17, -18, -17, -18, 22, -45, - 21, 2, 22, 7, 2, -38, -27, 19, -27, 26, - -27, -21, -21, 24, 17, 2, 17, 6, 6, 6, - 6, 6, 6, 6, 6, 6, 6, 6, 21, 2, - 22, -4, -27, 26, 26, 17, -23, -26, 57, -27, - -31, -31, -31, -28, -24, 14, -24, -26, -24, -26, - -11, 92, 93, 94, 95, -27, -27, -27, -25, -31, - 24, 21, 2, 21, -31, + -38, -45, 15, -28, -28, 21, 6, 2, -16, 22, + -4, -6, 25, 2, 62, 78, 63, 79, 64, 65, + 66, 80, 81, 12, 82, 47, 48, 51, 67, 18, + 68, 83, 84, 69, 70, 71, 72, 73, 88, 89, + 59, 74, 75, 22, 7, 20, -2, 25, 2, 25, + 2, 26, 26, -30, 26, 41, 57, -22, 24, 17, + -23, 30, 28, 29, 35, 36, 37, 33, 31, 34, + 32, 38, -17, -17, -18, -17, -18, 22, -45, 21, + 2, 22, 7, 2, -38, -27, 19, -27, 26, -27, + -21, -21, 24, 17, 2, 17, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 21, 2, 22, + -4, -27, 26, 26, 17, -23, -26, 57, -27, -31, + -31, -31, -28, -24, 14, -24, -26, -24, -26, -11, + 92, 93, 94, 95, -27, -27, -27, -25, -31, 24, + 21, 2, 21, -31, } var yyDef = [...]int16{ @@ -647,35 +647,35 @@ var yyDef = [...]int16{ 18, 19, 0, 108, 233, 234, 0, 244, 0, 85, 86, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, 227, 228, 0, 5, 100, - 0, 128, 131, 0, 0, 139, 245, 140, 144, 43, + 0, 128, 131, 0, 135, 139, 245, 140, 144, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 0, 0, 0, 0, 22, 23, 0, 0, 0, 61, 0, 83, 84, 0, 89, - 91, 0, 95, 99, 126, 0, 132, 0, 137, 0, - 138, 143, 0, 42, 47, 48, 44, 0, 0, 0, + 91, 0, 95, 99, 126, 0, 132, 0, 138, 0, + 143, 0, 42, 47, 48, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 68, 0, 70, 71, 0, 73, 239, 240, - 74, 75, 235, 236, 0, 0, 0, 82, 20, 21, - 24, 0, 54, 25, 0, 63, 65, 67, 87, 0, - 92, 0, 98, 229, 230, 231, 232, 0, 127, 130, - 133, 135, 134, 136, 142, 145, 147, 150, 154, 155, - 156, 0, 26, 0, 0, -2, -2, 27, 28, 29, - 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, - 40, 41, 69, 0, 237, 238, 76, 0, 81, 0, - 53, 56, 58, 59, 60, 198, 199, 200, 201, 202, - 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, - 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, - 223, 224, 225, 226, 62, 66, 88, 90, 93, 97, - 94, 96, 0, 0, 0, 0, 0, 0, 0, 0, - 160, 162, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 45, 46, 49, 247, 50, 72, 0, - 78, 80, 51, 0, 57, 64, 146, 241, 148, 0, - 151, 0, 0, 0, 158, 163, 159, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 77, 79, - 52, 55, 149, 0, 0, 157, 161, 164, 0, 243, - 165, 166, 167, 168, 169, 0, 170, 171, 172, 173, - 174, 180, 181, 182, 183, 152, 153, 242, 0, 178, - 0, 176, 179, 175, 177, + 0, 68, 0, 70, 71, 0, 73, 239, 240, 74, + 75, 235, 236, 0, 0, 0, 82, 20, 21, 24, + 0, 54, 25, 0, 63, 65, 67, 87, 0, 92, + 0, 98, 229, 230, 231, 232, 0, 127, 130, 133, + 136, 134, 137, 142, 145, 147, 150, 154, 155, 156, + 0, 26, 0, 0, -2, -2, 27, 28, 29, 30, + 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 69, 0, 237, 238, 76, 0, 81, 0, 53, + 56, 58, 59, 60, 198, 199, 200, 201, 202, 203, + 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, + 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, + 224, 225, 226, 62, 66, 88, 90, 93, 97, 94, + 96, 0, 0, 0, 0, 0, 0, 0, 0, 160, + 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 45, 46, 49, 247, 50, 72, 0, 78, + 80, 51, 0, 57, 64, 146, 241, 148, 0, 151, + 0, 0, 0, 158, 163, 159, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 77, 79, 52, + 55, 149, 0, 0, 157, 161, 164, 0, 243, 165, + 166, 167, 168, 169, 0, 170, 171, 172, 173, 174, + 180, 181, 182, 183, 152, 153, 242, 0, 178, 0, + 176, 179, 175, 177, } var yyTok1 = [...]int8{ @@ -1623,10 +1623,9 @@ yydefault: yyVAL.label = labels.Label{Name: yyDollar[1].item.Val, Value: yylex.(*parser).unquoteString(yyDollar[3].item.Val)} } case 135: - yyDollar = yyS[yypt-3 : yypt+1] + yyDollar = yyS[yypt-1 : yypt+1] { - yylex.(*parser).unexpected("label set", "string") - yyVAL.label = labels.Label{} + yyVAL.label = labels.Label{Name: labels.MetricName, Value: yyDollar[1].item.Val} } case 136: yyDollar = yyS[yypt-3 : yypt+1] @@ -1635,9 +1634,9 @@ yydefault: yyVAL.label = labels.Label{} } case 137: - yyDollar = yyS[yypt-2 : yypt+1] + yyDollar = yyS[yypt-3 : yypt+1] { - yylex.(*parser).unexpected("label set", "\"=\"") + yylex.(*parser).unexpected("label set", "string") yyVAL.label = labels.Label{} } case 138: diff --git a/promql/parser/parse.go b/promql/parser/parse.go index 3113d18d5..9bf27264a 100644 --- a/promql/parser/parse.go +++ b/promql/parser/parse.go @@ -244,7 +244,8 @@ type seriesDescription struct { values []SequenceValue } -// ParseSeriesDesc parses the description of a time series. +// ParseSeriesDesc parses the description of a time series. It is only used in +// the PromQL testing framework code. func ParseSeriesDesc(input string) (labels labels.Labels, values []SequenceValue, err error) { p := NewParser(input) p.lex.seriesDesc = true diff --git a/promql/parser/parse_test.go b/promql/parser/parse_test.go index 0e5e2f638..3445fce06 100644 --- a/promql/parser/parse_test.go +++ b/promql/parser/parse_test.go @@ -2398,7 +2398,7 @@ var testExpr = []struct { }, }, { - input: `sum by ("foo")({"some.metric"})`, + input: `sum by ("foo bar")({"some.metric"})`, expected: &AggregateExpr{ Op: SUM, Expr: &VectorSelector{ @@ -2406,14 +2406,14 @@ var testExpr = []struct { MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "some.metric"), }, PosRange: posrange.PositionRange{ - Start: 15, - End: 30, + Start: 19, + End: 34, }, }, - Grouping: []string{"foo"}, + Grouping: []string{"foo bar"}, PosRange: posrange.PositionRange{ Start: 0, - End: 31, + End: 35, }, }, }, @@ -4023,6 +4023,76 @@ func TestParseExpressions(t *testing.T) { } } +func TestParseSeriesDesc(t *testing.T) { + tests := []struct { + name string + input string + expectedLabels labels.Labels + expectedValues []SequenceValue + expectError string + }{ + { + name: "empty string", + expectedLabels: labels.EmptyLabels(), + expectedValues: []SequenceValue{}, + }, + { + name: "simple line", + input: `http_requests{job="api-server", instance="0", group="production"}`, + expectedLabels: labels.FromStrings( + "__name__", "http_requests", + "group", "production", + "instance", "0", + "job", "api-server", + ), + expectedValues: []SequenceValue{}, + }, + { + name: "label name characters that require quoting", + input: `{"http.requests", "service.name"="api-server", instance="0", group="canary"} 0+50x2`, + expectedLabels: labels.FromStrings( + "__name__", "http.requests", + "group", "canary", + "instance", "0", + "service.name", "api-server", + ), + expectedValues: []SequenceValue{ + {Value: 0, Omitted: false, Histogram: (*histogram.FloatHistogram)(nil)}, + {Value: 50, Omitted: false, Histogram: (*histogram.FloatHistogram)(nil)}, + {Value: 100, Omitted: false, Histogram: (*histogram.FloatHistogram)(nil)}, + }, + }, + { + name: "confirm failure on junk after identifier", + input: `{"http.requests"xx} 0+50x2`, + expectError: `parse error: unexpected identifier "xx" in label set, expected "," or "}"`, + }, + { + name: "confirm failure on bare operator after identifier", + input: `{"http.requests"=, x="y"} 0+50x2`, + expectError: `parse error: unexpected "," in label set, expected string`, + }, + { + name: "confirm failure on unterminated string identifier", + input: `{"http.requests} 0+50x2`, + expectError: `parse error: unterminated quoted string`, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + l, v, err := ParseSeriesDesc(tc.input) + if tc.expectError != "" { + require.Contains(t, err.Error(), tc.expectError) + } else { + require.NoError(t, err) + require.True(t, labels.Equal(tc.expectedLabels, l)) + require.Equal(t, tc.expectedValues, v) + } + }) + } +} + // NaN has no equality. Thus, we need a separate test for it. func TestNaNExpression(t *testing.T) { expr, err := ParseExpr("NaN") diff --git a/promql/parser/printer.go b/promql/parser/printer.go index 63b195082..afe755e7d 100644 --- a/promql/parser/printer.go +++ b/promql/parser/printer.go @@ -14,8 +14,10 @@ package parser import ( + "bytes" "fmt" "sort" + "strconv" "strings" "time" @@ -91,13 +93,20 @@ func (node *AggregateExpr) getAggOpStr() string { } func joinLabels(ss []string) string { + var bytea [1024]byte // On stack to avoid memory allocation while building the output. + b := bytes.NewBuffer(bytea[:0]) + for i, s := range ss { - // If the label is already quoted, don't quote it again. - if s[0] != '"' && s[0] != '\'' && s[0] != '`' && !model.IsValidLegacyMetricName(string(model.LabelValue(s))) { - ss[i] = fmt.Sprintf("\"%s\"", s) + if i > 0 { + b.WriteString(", ") + } + if !model.IsValidLegacyMetricName(string(model.LabelValue(s))) { + b.Write(strconv.AppendQuote(b.AvailableBuffer(), s)) + } else { + b.WriteString(s) } } - return strings.Join(ss, ", ") + return b.String() } func (node *BinaryExpr) returnBool() string { diff --git a/promql/promqltest/test.go b/promql/promqltest/test.go index f208b4f31..60df00503 100644 --- a/promql/promqltest/test.go +++ b/promql/promqltest/test.go @@ -56,6 +56,10 @@ const ( DefaultMaxSamplesPerQuery = 10000 ) +func init() { + model.NameValidationScheme = model.UTF8Validation +} + type TBRun interface { testing.TB Run(string, func(*testing.T)) bool diff --git a/promql/promqltest/test_test.go b/promql/promqltest/test_test.go index d0d09271e..327dcd78f 100644 --- a/promql/promqltest/test_test.go +++ b/promql/promqltest/test_test.go @@ -165,6 +165,8 @@ load 5m http_requests{job="api-server", instance="1", group="production"} 0+20x10 http_requests{job="api-server", instance="0", group="canary"} 0+30x10 http_requests{job="api-server", instance="1", group="canary"} 0+40x10 + {"http.requests", "service.name"="api-server", instance="0", group="canary"} 0+50x10 + {"http.requests", "service.name"="api-server", instance="1", group="canary"} 0+60x10 ` testCases := map[string]struct { @@ -176,6 +178,12 @@ load 5m eval instant at 5m sum by (group) (http_requests) {group="production"} 30 {group="canary"} 70 +`, + }, + "instant query on UTF-8 metric with expected float result": { + input: testData + ` +eval instant at 5m sum by ("service.name") ({"http.requests"}) + {"service.name"="api-server"} 110 `, }, "instant query with unexpected float result": { @@ -184,7 +192,7 @@ eval instant at 5m sum by (group) (http_requests) {group="production"} 30 {group="canary"} 80 `, - expectedError: `error in eval sum by (group) (http_requests) (line 8): expected 80 for {group="canary"} but got 70`, + expectedError: `error in eval sum by (group) (http_requests) (line 10): expected 80 for {group="canary"} but got 70`, }, "instant query with expected histogram result": { input: ` @@ -230,7 +238,7 @@ eval instant at 0 testmetric eval instant at 5m sum by (group) (http_requests) {group="production"} 30 `, - expectedError: `error in eval sum by (group) (http_requests) (line 8): unexpected metric {group="canary"} in result, has value 70`, + expectedError: `error in eval sum by (group) (http_requests) (line 10): unexpected metric {group="canary"} in result, has value 70`, }, "instant query, but result has an unexpected series with a histogram value": { input: ` @@ -248,7 +256,7 @@ eval instant at 5m sum by (group) (http_requests) {group="canary"} 70 {group="test"} 100 `, - expectedError: `error in eval sum by (group) (http_requests) (line 8): expected metric {group="test"} with 3: [100.000000] not found`, + expectedError: `error in eval sum by (group) (http_requests) (line 10): expected metric {group="test"} with 3: [100.000000] not found`, }, "instant query expected to fail, and query fails": { input: ` @@ -334,7 +342,7 @@ eval_ordered instant at 50m sort(http_requests) http_requests{group="canary", instance="1", job="api-server"} 400 http_requests{group="canary", instance="0", job="api-server"} 300 `, - expectedError: `error in eval sort(http_requests) (line 8): expected metric {__name__="http_requests", group="canary", instance="0", job="api-server"} with [300.000000] at position 4 but was at 3`, + expectedError: `error in eval sort(http_requests) (line 10): expected metric {__name__="http_requests", group="canary", instance="0", job="api-server"} with [300.000000] at position 4 but was at 3`, }, "instant query with results expected to match provided order, but result has an unexpected series": { input: testData + ` @@ -343,7 +351,7 @@ eval_ordered instant at 50m sort(http_requests) http_requests{group="production", instance="1", job="api-server"} 200 http_requests{group="canary", instance="0", job="api-server"} 300 `, - expectedError: `error in eval sort(http_requests) (line 8): unexpected metric {__name__="http_requests", group="canary", instance="1", job="api-server"} in result, has value 400`, + expectedError: `error in eval sort(http_requests) (line 10): unexpected metric {__name__="http_requests", group="canary", instance="1", job="api-server"} in result, has value 400`, }, "instant query with invalid timestamp": { input: `eval instant at abc123 vector(0)`, @@ -362,7 +370,7 @@ eval range from 0 to 10m step 5m sum by (group) (http_requests) {group="production"} 0 30 60 {group="canary"} 0 80 140 `, - expectedError: `error in eval sum by (group) (http_requests) (line 8): expected float value at index 1 (t=300000) for {group="canary"} to be 80, but got 70 (result has 3 float points [0 @[0] 70 @[300000] 140 @[600000]] and 0 histogram points [])`, + expectedError: `error in eval sum by (group) (http_requests) (line 10): expected float value at index 1 (t=300000) for {group="canary"} to be 80, but got 70 (result has 3 float points [0 @[0] 70 @[300000] 140 @[600000]] and 0 histogram points [])`, }, "range query with expected histogram values": { input: ` @@ -389,7 +397,7 @@ eval range from 0 to 10m step 5m sum by (group) (http_requests) {group="production"} 0 30 60 90 {group="canary"} 0 70 140 `, - expectedError: `error in eval sum by (group) (http_requests) (line 8): expected 4 points for {group="production"}, but query time range cannot return this many points`, + expectedError: `error in eval sum by (group) (http_requests) (line 10): expected 4 points for {group="production"}, but query time range cannot return this many points`, }, "range query with missing point in result": { input: ` @@ -407,14 +415,14 @@ eval range from 0 to 10m step 5m sum by (group) (http_requests) {group="production"} 0 30 {group="canary"} 0 70 140 `, - expectedError: `error in eval sum by (group) (http_requests) (line 8): expected 2 float points and 0 histogram points for {group="production"}, but got 3 float points [0 @[0] 30 @[300000] 60 @[600000]] and 0 histogram points []`, + expectedError: `error in eval sum by (group) (http_requests) (line 10): expected 2 float points and 0 histogram points for {group="production"}, but got 3 float points [0 @[0] 30 @[300000] 60 @[600000]] and 0 histogram points []`, }, "range query, but result has an unexpected series": { input: testData + ` eval range from 0 to 10m step 5m sum by (group) (http_requests) {group="production"} 0 30 60 `, - expectedError: `error in eval sum by (group) (http_requests) (line 8): unexpected metric {group="canary"} in result, has 3 float points [0 @[0] 70 @[300000] 140 @[600000]] and 0 histogram points []`, + expectedError: `error in eval sum by (group) (http_requests) (line 10): unexpected metric {group="canary"} in result, has 3 float points [0 @[0] 70 @[300000] 140 @[600000]] and 0 histogram points []`, }, "range query, but result is missing a series": { input: testData + ` @@ -423,7 +431,7 @@ eval range from 0 to 10m step 5m sum by (group) (http_requests) {group="canary"} 0 70 140 {group="test"} 0 100 200 `, - expectedError: `error in eval sum by (group) (http_requests) (line 8): expected metric {group="test"} not found`, + expectedError: `error in eval sum by (group) (http_requests) (line 10): expected metric {group="test"} not found`, }, "range query expected to fail, and query fails": { input: ` diff --git a/storage/remote/codec_test.go b/storage/remote/codec_test.go index 35cd4c4cd..6fb1ae1e4 100644 --- a/storage/remote/codec_test.go +++ b/storage/remote/codec_test.go @@ -163,6 +163,11 @@ func TestWriteV2RequestFixture(t *testing.T) { } func TestValidateLabelsAndMetricName(t *testing.T) { + oldScheme := model.NameValidationScheme + model.NameValidationScheme = model.LegacyValidation + defer func() { + model.NameValidationScheme = oldScheme + }() tests := []struct { input []prompb.Label expectedErr string