UTF-8: Add support for parsing UTF8 metric and label names

This adds support for the new grammar of `{"metric_name", "l1"="val",}`. This grammar will also be valid for non-UTF-8 names.
UTF-8 names will not be considered valid unless model.NameValidationScheme is changed.

Signed-off-by: Owen Williams <owen.williams@grafana.com>

go.mod

@@ -194,6 +194,8 @@ require (
 )
 
 replace (
+	github.com/prometheus/client_golang => /home/owilliams/src/grafana/client_golang
+	github.com/prometheus/common => /home/owilliams/src/third_party/common
 	k8s.io/klog => github.com/simonpasquier/klog-gokit v0.3.0
 	k8s.io/klog/v2 => github.com/simonpasquier/klog-gokit/v3 v3.3.0
 )

model/textparse/openmetricslex.l

@@ -50,12 +50,15 @@ S     [ ]
 <sComment>TYPE{S}                     l.state = sMeta1; return tType
 <sComment>UNIT{S}                     l.state = sMeta1; return tUnit
 <sComment>"EOF"\n?                    l.state = sInit; return tEOFWord
+<sMeta1>\"(\\.|[^\\"])*\"             l.state = sMeta2; return tMName
 <sMeta1>{M}({M}|{D})*                 l.state = sMeta2; return tMName
 <sMeta2>{S}{C}*\n                     l.state = sInit; return tText
 
 {M}({M}|{D})*                         l.state = sValue; return tMName
 <sValue>\{                            l.state = sLabels; return tBraceOpen
+\{                                    l.state = sLabels; return tBraceOpen
 <sLabels>{L}({L}|{D})*                return tLName
+<sLabels>\"(\\.|[^\\"])*\"            l.state = sLabels; return tQString
 <sLabels>\}                           l.state = sValue; return tBraceClose
 <sLabels>=                            l.state = sLValue; return tEqual
 <sLabels>,                            return tComma

model/textparse/openmetricsparse.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/gogo/protobuf/types"
 
+	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/model/exemplar"
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
@@ -154,9 +155,9 @@ func (p *OpenMetricsParser) Metric(l *labels.Labels) string {
 	s := string(p.series)
 
 	p.builder.Reset()
-	p.builder.Add(labels.MetricName, s[:p.offsets[0]-p.start])
+	p.builder.Add(labels.MetricName, s[p.offsets[0]-p.start:p.offsets[1]-p.start])
 
-	for i := 1; i < len(p.offsets); i += 4 {
+	for i := 2; i < len(p.offsets); i += 4 {
 		a := p.offsets[i] - p.start
 		b := p.offsets[i+1] - p.start
 		c := p.offsets[i+2] - p.start
@@ -225,11 +226,12 @@ func (p *OpenMetricsParser) nextToken() token {
 }
 
 func (p *OpenMetricsParser) parseError(exp string, got token) error {
-	e := p.l.i + 1
+	e := p.l.i + 80
 	if len(p.l.b) < e {
 		e = len(p.l.b)
 	}
-	return fmt.Errorf("%s, got %q (%q) while parsing: %q", exp, p.l.b[p.l.start:e], got, p.l.b[p.start:e])
+	start := int(math.Max(0, float64(p.start-80)))
+	return fmt.Errorf("%s, got %q (%q) while parsing: %q", exp, p.l.b[p.l.start:e], got, p.l.b[start:e])
 }
 
 // Next advances the parser to the next sample. It returns false if no
@@ -253,11 +255,18 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
 	case tEOF:
 		return EntryInvalid, errors.New("data does not end with # EOF")
 	case tHelp, tType, tUnit:
+		tStart := p.l.start
 		switch t2 := p.nextToken(); t2 {
 		case tMName:
-			p.offsets = append(p.offsets, p.l.start, p.l.i)
+			mStart := p.l.start
+			mEnd := p.l.i
+			if p.l.b[mStart] == '"' && p.l.b[mEnd-1] == '"' {
+				mStart++
+				mEnd--
+			}
+			p.offsets = append(p.offsets, mStart, mEnd)
 		default:
-			return EntryInvalid, p.parseError("expected metric name after "+t.String(), t2)
+			return EntryInvalid, p.parseError("expected metric name after "+t.String()+" "+string(p.l.b), t2)
 		}
 		switch t2 := p.nextToken(); t2 {
 		case tText:
@@ -267,7 +276,11 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
 				p.text = []byte{}
 			}
 		default:
-			return EntryInvalid, fmt.Errorf("expected text in %s", t.String())
+			end := tStart + 40
+			if end >= len(p.l.b) {
+				end = len(p.l.b) - 1
+			}
+			return EntryInvalid, fmt.Errorf("expected text in %s: got %v (%v)", t.String(), t2.String(), string(p.l.b[tStart:end]))
 		}
 		switch t {
 		case tType:
@@ -312,8 +325,24 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
 			return EntryUnit, nil
 		}
 
+	case tBraceOpen:
+		// We found a brace, so make room for the eventual metric name. If these
+		// values aren't updated, then the metric name was not set inside the
+		// braces and we can return an error.
+		if len(p.offsets) == 0 {
+			p.offsets = []int{-1, -1}
+		}
+		if p.offsets, err = p.parseLVals(p.offsets); err != nil {
+			return EntryInvalid, err
+		}
+
+		p.series = p.l.b[p.start:p.l.i]
+		return p.parseMetricSuffix(p.nextToken())
 	case tMName:
-		p.offsets = append(p.offsets, p.l.i)
+		p.offsets = append(p.offsets, p.start, p.l.i)
+		if err := p.verifyMetricName(p.offsets[0], p.offsets[1]); err != nil {
+			return EntryInvalid, err
+		}
 		p.series = p.l.b[p.start:p.l.i]
 
 		t2 := p.nextToken()
@@ -325,45 +354,7 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
 			p.series = p.l.b[p.start:p.l.i]
 			t2 = p.nextToken()
 		}
-		p.val, err = p.getFloatValue(t2, "metric")
+		return p.parseMetricSuffix(t2)
-		if err != nil {
-			return EntryInvalid, err
-		}
-
-		p.hasTS = false
-		switch t2 := p.nextToken(); t2 {
-		case tEOF:
-			return EntryInvalid, errors.New("data does not end with # EOF")
-		case tLinebreak:
-			break
-		case tComment:
-			if err := p.parseComment(); err != nil {
-				return EntryInvalid, err
-			}
-		case tTimestamp:
-			p.hasTS = true
-			var ts float64
-			// A float is enough to hold what we need for millisecond resolution.
-			if ts, err = parseFloat(yoloString(p.l.buf()[1:])); err != nil {
-				return EntryInvalid, fmt.Errorf("%w while parsing: %q", err, p.l.b[p.start:p.l.i])
-			}
-			if math.IsNaN(ts) || math.IsInf(ts, 0) {
-				return EntryInvalid, fmt.Errorf("invalid timestamp %f", ts)
-			}
-			p.ts = int64(ts * 1000)
-			switch t3 := p.nextToken(); t3 {
-			case tLinebreak:
-			case tComment:
-				if err := p.parseComment(); err != nil {
-					return EntryInvalid, err
-				}
-			default:
-				return EntryInvalid, p.parseError("expected next entry after timestamp", t3)
-			}
-		default:
-			return EntryInvalid, p.parseError("expected timestamp or # symbol", t2)
-		}
-		return EntrySeries, nil
 
 	default:
 		err = p.parseError("expected a valid start token", t)
@@ -416,37 +407,41 @@ func (p *OpenMetricsParser) parseComment() error {
 }
 
 func (p *OpenMetricsParser) parseLVals(offsets []int) ([]int, error) {
-	first := true
+	t := p.nextToken()
 	for {
-		t := p.nextToken()
+		curTStart := p.l.start
+		curTI := p.l.i
 		switch t {
 		case tBraceClose:
 			return offsets, nil
-		case tComma:
+		case tLName:
-			if first {
+		case tQString:
-				return nil, p.parseError("expected label name or left brace", t)
+		default:
+			return nil, p.parseError("expected label name", t)
+		}
+
+		t = p.nextToken()
+		// A quoted string followed by a comma or brace is a metric name. Set the
+		// offsets and continue processing.
+		if t == tComma || t == tBraceClose {
+			if offsets[0] != -1 || offsets[1] != -1 {
+				return nil, fmt.Errorf("metric name already set while parsing: %q", p.l.b[p.start:p.l.i])
+			}
+			offsets[0] = curTStart + 1
+			offsets[1] = curTI - 1
+			if err := p.verifyMetricName(offsets[0], offsets[1]); err != nil {
+				return nil, err
+			}
+			if t == tBraceClose {
+				return offsets, nil
 			}
 			t = p.nextToken()
-			if t != tLName {
+			continue
-				return nil, p.parseError("expected label name", t)
-			}
-		case tLName:
-			if !first {
-				return nil, p.parseError("expected comma", t)
-			}
-		default:
-			if first {
-				return nil, p.parseError("expected label name or left brace", t)
-			}
-			return nil, p.parseError("expected comma or left brace", t)
-
 		}
-		first = false
+		// We have a label name.
-		// t is now a label name.
+		offsets = append(offsets, curTStart, curTI)
 
-		offsets = append(offsets, p.l.start, p.l.i)
+		if t != tEqual {
-
-		if t := p.nextToken(); t != tEqual {
 			return nil, p.parseError("expected equal", t)
 		}
 		if t := p.nextToken(); t != tLValue {
@@ -459,9 +454,60 @@ func (p *OpenMetricsParser) parseLVals(offsets []int) ([]int, error) {
 		// The openMetricsLexer ensures the value string is quoted. Strip first
 		// and last character.
 		offsets = append(offsets, p.l.start+1, p.l.i-1)
+
+		// Free trailing commas are allowed.
+		t = p.nextToken()
+		if t == tComma {
+			t = p.nextToken()
+		} else if t != tBraceClose {
+			return nil, p.parseError("expected comma or brace close", t)
+		}
 	}
 }
 
+// parseMetricSuffix parses the end of the line after the metric name and
+// labels. It starts parsing with the provided token.
+func (p *OpenMetricsParser) parseMetricSuffix(t token) (Entry, error) {
+	var err error
+	p.val, err = p.getFloatValue(t, "metric")
+	if err != nil {
+		return EntryInvalid, err
+	}
+
+	p.hasTS = false
+	switch t2 := p.nextToken(); t2 {
+	case tEOF:
+		return EntryInvalid, errors.New("data does not end with # EOF")
+	case tLinebreak:
+		break
+	case tComment:
+		if err := p.parseComment(); err != nil {
+			return EntryInvalid, err
+		}
+	case tTimestamp:
+		p.hasTS = true
+		var ts float64
+		// A float is enough to hold what we need for millisecond resolution.
+		if ts, err = parseFloat(yoloString(p.l.buf()[1:])); err != nil {
+			return EntryInvalid, fmt.Errorf("%v while parsing: %q", err, p.l.b[p.start:p.l.i])
+		}
+		if math.IsNaN(ts) || math.IsInf(ts, 0) {
+			return EntryInvalid, fmt.Errorf("invalid timestamp %f", ts)
+		}
+		p.ts = int64(ts * 1000)
+		switch t3 := p.nextToken(); t3 {
+		case tLinebreak:
+		case tComment:
+			if err := p.parseComment(); err != nil {
+				return EntryInvalid, err
+			}
+		default:
+			return EntryInvalid, p.parseError("expected next entry after timestamp", t3)
+		}
+	}
+	return EntrySeries, nil
+}
+
 func (p *OpenMetricsParser) getFloatValue(t token, after string) (float64, error) {
 	if t != tValue {
 		return 0, p.parseError(fmt.Sprintf("expected value after %v", after), t)
@@ -476,3 +522,11 @@ func (p *OpenMetricsParser) getFloatValue(t token, after string) (float64, error
 	}
 	return val, nil
 }
+
+func (p *OpenMetricsParser) verifyMetricName(start, end int) error {
+	m := yoloString(p.l.b[start:end])
+	if !model.IsValidMetricName(model.LabelValue(m)) {
+		return fmt.Errorf("metric name %q is not valid", m)
+	}
+	return nil
+}

model/textparse/openmetricsparse_test.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/model/exemplar"
 	"github.com/prometheus/prometheus/model/labels"
 )
@@ -299,6 +300,130 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5`
 	require.Len(t, exp, i)
 }
 
+func TestUTF8OpenMetricsParse(t *testing.T) {
+	model.NameValidationScheme = model.UTF8Validation
+	defer func(){
+		model.NameValidationScheme = model.LegacyValidation
+	}()
+
+	input := `# HELP "go.gc_duration_seconds" A summary of the GC invocation durations.
+# TYPE "go.gc_duration_seconds" summary
+# UNIT "go.gc_duration_seconds" seconds
+{"go.gc_duration_seconds",quantile="0"} 4.9351e-05
+{"go.gc_duration_seconds",quantile="0.25"} 7.424100000000001e-05
+{"go.gc_duration_seconds",quantile="0.5",a="b"} 8.3835e-05
+{"http.status",q="0.9",a="b"} 8.3835e-05
+{"http.status",q="0.9",a="b"} 8.3835e-05
+{q="0.9","http.status",a="b"} 8.3835e-05
+{"go.gc_duration_seconds_sum"} 0.004304266`
+
+	input += "\n# EOF\n"
+
+	exp := []struct {
+		lset    labels.Labels
+		m       string
+		t       *int64
+		v       float64
+		typ     MetricType
+		help    string
+		unit    string
+		comment string
+		e       *exemplar.Exemplar
+	}{
+		{
+			m:    "go.gc_duration_seconds",
+			help: "A summary of the GC invocation durations.",
+		}, {
+			m:   "go.gc_duration_seconds",
+			typ: MetricTypeSummary,
+		}, {
+			m:    "go.gc_duration_seconds",
+			unit: "seconds",
+		}, {
+			m:    `{"go.gc_duration_seconds",quantile="0"}`,
+			v:    4.9351e-05,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0"),
+		}, {
+			m:    `{"go.gc_duration_seconds",quantile="0.25"}`,
+			v:    7.424100000000001e-05,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0.25"),
+		}, {
+			m:    `{"go.gc_duration_seconds",quantile="0.5",a="b"}`,
+			v:    8.3835e-05,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0.5", "a", "b"),
+		}, {
+			m:    `{"http.status",q="0.9",a="b"}`,
+			v:    8.3835e-05,
+			lset: labels.FromStrings("__name__", "http.status", "q", "0.9", "a", "b"),
+		}, {
+			m:    `{"http.status",q="0.9",a="b"}`,
+			v:    8.3835e-05,
+			lset: labels.FromStrings("__name__", "http.status", "q", "0.9", "a", "b"),
+		}, {
+			m:    `{q="0.9","http.status",a="b"}`,
+			v:    8.3835e-05,
+			lset: labels.FromStrings("__name__", "http.status", "q", "0.9", "a", "b"),
+		}, {
+			m:    `{"go.gc_duration_seconds_sum"}`,
+			v:    0.004304266,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds_sum"),
+		},
+	}
+
+	p := NewOpenMetricsParser([]byte(input))
+	i := 0
+
+	var res labels.Labels
+
+	for {
+		et, err := p.Next()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		require.NoError(t, err)
+
+		switch et {
+		case EntrySeries:
+			m, ts, v := p.Series()
+
+			var e exemplar.Exemplar
+			p.Metric(&res)
+			found := p.Exemplar(&e)
+			require.Equal(t, exp[i].m, string(m))
+			require.Equal(t, exp[i].t, ts)
+			require.Equal(t, exp[i].v, v)
+			require.Equal(t, exp[i].lset, res)
+			if exp[i].e == nil {
+				require.False(t, found)
+			} else {
+				require.True(t, found)
+				require.Equal(t, *exp[i].e, e)
+			}
+
+		case EntryType:
+			m, typ := p.Type()
+			require.Equal(t, exp[i].m, string(m))
+			require.Equal(t, exp[i].typ, typ)
+
+		case EntryHelp:
+			m, h := p.Help()
+			require.Equal(t, exp[i].m, string(m))
+			require.Equal(t, exp[i].help, string(h))
+
+		case EntryUnit:
+			m, u := p.Unit()
+			require.Equal(t, exp[i].m, string(m))
+			require.Equal(t, exp[i].unit, string(u))
+
+		case EntryComment:
+			require.Equal(t, exp[i].comment, string(p.Comment()))
+		}
+
+		i++
+	}
+	require.Len(t, exp, i)
+}
+
 func TestOpenMetricsParseErrors(t *testing.T) {
 	cases := []struct {
 		input string
@@ -341,19 +466,19 @@ func TestOpenMetricsParseErrors(t *testing.T) {
 		},
 		{
 			input: "a\n#EOF\n",
-			err:   "expected value after metric, got \"\\n\" (\"INVALID\") while parsing: \"a\\n\"",
+			err:   "expected value after metric, got \"\\n#EOF\\n\" (\"INVALID\") while parsing: \"a\\n#EOF\\n\"",
 		},
 		{
 			input: "\n\n#EOF\n",
-			err:   "expected a valid start token, got \"\\n\" (\"INVALID\") while parsing: \"\\n\"",
+			err:   "expected a valid start token, got \"\\n\\n#EOF\\n\" (\"INVALID\") while parsing: \"\\n\\n#EOF\\n\"",
 		},
 		{
 			input: " a 1\n#EOF\n",
-			err:   "expected a valid start token, got \" \" (\"INVALID\") while parsing: \" \"",
+			err:   "expected a valid start token, got \" a 1\\n#EOF\\n\" (\"INVALID\") while parsing: \" a 1\\n#EOF\\n\"",
 		},
 		{
 			input: "9\n#EOF\n",
-			err:   "expected a valid start token, got \"9\" (\"INVALID\") while parsing: \"9\"",
+			err:   "expected a valid start token, got \"9\\n#EOF\\n\" (\"INVALID\") while parsing: \"9\\n#EOF\\n\"",
 		},
 		{
 			input: "# TYPE u untyped\n#EOF\n",
@@ -365,15 +490,15 @@ func TestOpenMetricsParseErrors(t *testing.T) {
 		},
 		{
 			input: "#  TYPE c counter\n#EOF\n",
-			err:   "expected a valid start token, got \"#  \" (\"INVALID\") while parsing: \"#  \"",
+			err:   "expected a valid start token, got \"#  TYPE c counter\\n#EOF\\n\" (\"INVALID\") while parsing: \"#  TYPE c counter\\n#EOF\\n\"",
 		},
 		{
 			input: "# TYPE \n#EOF\n",
-			err:   "expected metric name after TYPE, got \"\\n\" (\"INVALID\") while parsing: \"# TYPE \\n\"",
+			err:   "expected metric name after TYPE # TYPE \n#EOF\n, got \"\\n#EOF\\n\" (\"INVALID\") while parsing: \"# TYPE \\n#EOF\\n\"",
 		},
 		{
 			input: "# TYPE m\n#EOF\n",
-			err:   "expected text in TYPE",
+			err:   "expected text in TYPE: got INVALID (# TYPE m\n#EOF)",
 		},
 		{
 			input: "# UNIT metric suffix\n#EOF\n",
@@ -389,23 +514,23 @@ func TestOpenMetricsParseErrors(t *testing.T) {
 		},
 		{
 			input: "# UNIT \n#EOF\n",
-			err:   "expected metric name after UNIT, got \"\\n\" (\"INVALID\") while parsing: \"# UNIT \\n\"",
+			err:   "expected metric name after UNIT # UNIT \n#EOF\n, got \"\\n#EOF\\n\" (\"INVALID\") while parsing: \"# UNIT \\n#EOF\\n\"",
 		},
 		{
 			input: "# UNIT m\n#EOF\n",
-			err:   "expected text in UNIT",
+			err:   "expected text in UNIT: got INVALID (# UNIT m\n#EOF)",
 		},
 		{
 			input: "# HELP \n#EOF\n",
-			err:   "expected metric name after HELP, got \"\\n\" (\"INVALID\") while parsing: \"# HELP \\n\"",
+			err:   "expected metric name after HELP # HELP \n#EOF\n, got \"\\n#EOF\\n\" (\"INVALID\") while parsing: \"# HELP \\n#EOF\\n\"",
 		},
 		{
 			input: "# HELP m\n#EOF\n",
-			err:   "expected text in HELP",
+			err:   "expected text in HELP: got INVALID (# HELP m\n#EOF)",
 		},
 		{
 			input: "a\t1\n#EOF\n",
-			err:   "expected value after metric, got \"\\t\" (\"INVALID\") while parsing: \"a\\t\"",
+			err:   "expected value after metric, got \"\\t1\\n#EOF\\n\" (\"INVALID\") while parsing: \"a\\t1\\n#EOF\\n\"",
 		},
 		{
 			input: "a 1\t2\n#EOF\n",
@@ -413,11 +538,11 @@ func TestOpenMetricsParseErrors(t *testing.T) {
 		},
 		{
 			input: "a 1 2 \n#EOF\n",
-			err:   "expected next entry after timestamp, got \" \\n\" (\"INVALID\") while parsing: \"a 1 2 \\n\"",
+			err:   "expected next entry after timestamp, got \" \\n#EOF\\n\" (\"INVALID\") while parsing: \"a 1 2 \\n#EOF\\n\"",
 		},
 		{
 			input: "a 1 2 #\n#EOF\n",
-			err:   "expected next entry after timestamp, got \" #\\n\" (\"TIMESTAMP\") while parsing: \"a 1 2 #\\n\"",
+			err:   "expected next entry after timestamp, got \" #\\n#EOF\\n\" (\"TIMESTAMP\") while parsing: \"a 1 2 #\\n#EOF\\n\"",
 		},
 		{
 			input: "a 1 1z\n#EOF\n",
@@ -425,7 +550,7 @@ func TestOpenMetricsParseErrors(t *testing.T) {
 		},
 		{
 			input: " # EOF\n",
-			err:   "expected a valid start token, got \" \" (\"INVALID\") while parsing: \" \"",
+			err:   "expected a valid start token, got \" # EOF\\n\" (\"INVALID\") while parsing: \" # EOF\\n\"",
 		},
 		{
 			input: "# EOF\na 1",
@@ -441,7 +566,7 @@ func TestOpenMetricsParseErrors(t *testing.T) {
 		},
 		{
 			input: "#\tTYPE c counter\n",
-			err:   "expected a valid start token, got \"#\\t\" (\"INVALID\") while parsing: \"#\\t\"",
+			err:   "expected a valid start token, got \"#\\tTYPE c counter\\n\" (\"INVALID\") while parsing: \"#\\tTYPE c counter\\n\"",
 		},
 		{
 			input: "# TYPE c  counter\n",
@@ -449,35 +574,31 @@ func TestOpenMetricsParseErrors(t *testing.T) {
 		},
 		{
 			input: "a 1 1 1\n# EOF\n",
-			err:   "expected next entry after timestamp, got \" 1\\n\" (\"TIMESTAMP\") while parsing: \"a 1 1 1\\n\"",
+			err:   "expected next entry after timestamp, got \" 1\\n# EOF\\n\" (\"TIMESTAMP\") while parsing: \"a 1 1 1\\n# EOF\\n\"",
 		},
 		{
 			input: "a{b='c'} 1\n# EOF\n",
-			err:   "expected label value, got \"'\" (\"INVALID\") while parsing: \"a{b='\"",
+			err:   "expected label value, got \"'c'} 1\\n# EOF\\n\" (\"INVALID\") while parsing: \"a{b='c'} 1\\n# EOF\\n\"",
-		},
-		{
-			input: "a{b=\"c\",} 1\n# EOF\n",
-			err:   "expected label name, got \"} \" (\"BCLOSE\") while parsing: \"a{b=\\\"c\\\",} \"",
 		},
 		{
 			input: "a{,b=\"c\"} 1\n# EOF\n",
-			err:   "expected label name or left brace, got \",b\" (\"COMMA\") while parsing: \"a{,b\"",
+			err:   "expected label name, got \",b=\\\"c\\\"} 1\\n# EOF\\n\" (\"COMMA\") while parsing: \"a{,b=\\\"c\\\"} 1\\n# EOF\\n\"",
 		},
 		{
 			input: "a{b=\"c\"d=\"e\"} 1\n# EOF\n",
-			err:   "expected comma, got \"d=\" (\"LNAME\") while parsing: \"a{b=\\\"c\\\"d=\"",
+			err:   "expected comma or brace close, got \"d=\\\"e\\\"} 1\\n# EOF\\n\" (\"LNAME\") while parsing: \"a{b=\\\"c\\\"d=\\\"e\\\"} 1\\n# EOF\\n\"",
 		},
 		{
 			input: "a{b=\"c\",,d=\"e\"} 1\n# EOF\n",
-			err:   "expected label name, got \",d\" (\"COMMA\") while parsing: \"a{b=\\\"c\\\",,d\"",
+			err:   "expected label name, got \",d=\\\"e\\\"} 1\\n# EOF\\n\" (\"COMMA\") while parsing: \"a{b=\\\"c\\\",,d=\\\"e\\\"} 1\\n# EOF\\n\"",
 		},
 		{
 			input: "a{b=\n# EOF\n",
-			err:   "expected label value, got \"\\n\" (\"INVALID\") while parsing: \"a{b=\\n\"",
+			err:   "expected label value, got \"\\n# EOF\\n\" (\"INVALID\") while parsing: \"a{b=\\n# EOF\\n\"",
 		},
 		{
 			input: "a{\xff=\"foo\"} 1\n# EOF\n",
-			err:   "expected label name or left brace, got \"\\xff\" (\"INVALID\") while parsing: \"a{\\xff\"",
+			err:   "expected label name, got \"\\xff=\\\"foo\\\"} 1\\n# EOF\\n\" (\"INVALID\") while parsing: \"a{\\xff=\\\"foo\\\"} 1\\n# EOF\\n\"",
 		},
 		{
 			input: "a{b=\"\xff\"} 1\n# EOF\n",
@@ -489,11 +610,11 @@ func TestOpenMetricsParseErrors(t *testing.T) {
 		},
 		{
 			input: "something_weird{problem=\"\n# EOF\n",
-			err:   "expected label value, got \"\\\"\\n\" (\"INVALID\") while parsing: \"something_weird{problem=\\\"\\n\"",
+			err:   "expected label value, got \"\\\"\\n# EOF\\n\" (\"INVALID\") while parsing: \"something_weird{problem=\\\"\\n# EOF\\n\"",
 		},
 		{
 			input: "empty_label_name{=\"\"} 0\n# EOF\n",
-			err:   "expected label name or left brace, got \"=\\\"\" (\"EQUAL\") while parsing: \"empty_label_name{=\\\"\"",
+			err:   "expected label name, got \"=\\\"\\\"} 0\\n# EOF\\n\" (\"EQUAL\") while parsing: \"empty_label_name{=\\\"\\\"} 0\\n# EOF\\n\"",
 		},
 		{
 			input: "foo 1_2\n\n# EOF\n",
@@ -513,11 +634,11 @@ func TestOpenMetricsParseErrors(t *testing.T) {
 		},
 		{
 			input: "custom_metric_total 1 # {aa=bb}\n# EOF\n",
-			err:   "expected label value, got \"b\" (\"INVALID\") while parsing: \"custom_metric_total 1 # {aa=b\"",
+			err:   "expected label value, got \"bb}\\n# EOF\\n\" (\"INVALID\") while parsing: \"custom_metric_total 1 # {aa=bb}\\n# EOF\\n\"",
 		},
 		{
 			input: "custom_metric_total 1 # {aa=\"bb\"}\n# EOF\n",
-			err:   "expected value after exemplar labels, got \"\\n\" (\"INVALID\") while parsing: \"custom_metric_total 1 # {aa=\\\"bb\\\"}\\n\"",
+			err:   "expected value after exemplar labels, got \"\\n# EOF\\n\" (\"INVALID\") while parsing: \"custom_metric_total 1 # {aa=\\\"bb\\\"}\\n# EOF\\n\"",
 		},
 		{
 			input: `custom_metric_total 1 # {aa="bb"}`,
@@ -525,7 +646,7 @@ func TestOpenMetricsParseErrors(t *testing.T) {
 		},
 		{
 			input: `custom_metric_total 1 # {aa="bb",,cc="dd"} 1`,
-			err:   "expected label name, got \",c\" (\"COMMA\") while parsing: \"custom_metric_total 1 # {aa=\\\"bb\\\",,c\"",
+			err:   "expected label name, got \",cc=\\\"dd\\\"} 1\" (\"COMMA\") while parsing: \"custom_metric_total 1 # {aa=\\\"bb\\\",,cc=\\\"dd\\\"} 1\"",
 		},
 		{
 			input: `custom_metric_total 1 # {aa="bb"} 1_2`,
@@ -545,11 +666,11 @@ func TestOpenMetricsParseErrors(t *testing.T) {
 		},
 		{
 			input: `custom_metric_total 1 # {aa=\"\xff\"} 9.0`,
-			err:   "expected label value, got \"\\\\\" (\"INVALID\") while parsing: \"custom_metric_total 1 # {aa=\\\\\"",
+			err:   "expected label value, got \"\\\\\\\"\\\\xff\\\\\\\"} 9.0\" (\"INVALID\") while parsing: \"custom_metric_total 1 # {aa=\\\\\\\"\\\\xff\\\\\\\"} 9.0\"",
 		},
 		{
 			input: `{b="c",} 1`,
-			err:   "expected a valid start token, got \"{\" (\"INVALID\") while parsing: \"{\"",
+			err:   "data does not end with # EOF",
 		},
 		{
 			input: `a 1 NaN`,
@@ -610,7 +731,7 @@ func TestOMNullByteHandling(t *testing.T) {
 		},
 		{
 			input: "a{b=\x00\"ssss\"} 1\n# EOF\n",
-			err:   "expected label value, got \"\\x00\" (\"INVALID\") while parsing: \"a{b=\\x00\"",
+			err:   "expected label value, got \"\\x00\\\"ssss\\\"} 1\\n# EOF\\n\" (\"INVALID\") while parsing: \"a{b=\\x00\\\"ssss\\\"} 1\\n# EOF\\n\"",
 		},
 		{
 			input: "a{b=\"\x00",
@@ -618,11 +739,11 @@ func TestOMNullByteHandling(t *testing.T) {
 		},
 		{
 			input: "a{b\x00=\"hiih\"}	1",
-			err:   "expected equal, got \"\\x00\" (\"INVALID\") while parsing: \"a{b\\x00\"",
+			err:   "expected equal, got \"\\x00=\\\"hiih\\\"}\\t1\" (\"INVALID\") while parsing: \"a{b\\x00=\\\"hiih\\\"}\\t1\"",
 		},
 		{
 			input: "a\x00{b=\"ddd\"} 1",
-			err:   "expected value after metric, got \"\\x00\" (\"INVALID\") while parsing: \"a\\x00\"",
+			err:   "expected value after metric, got \"\\x00{b=\\\"ddd\\\"} 1\" (\"INVALID\") while parsing: \"a\\x00{b=\\\"ddd\\\"} 1\"",
 		},
 		{
 			input: "#",
@@ -634,11 +755,11 @@ func TestOMNullByteHandling(t *testing.T) {
 		},
 		{
 			input: "custom_metric_total 1 # {b=\x00\"ssss\"} 1\n",
-			err:   "expected label value, got \"\\x00\" (\"INVALID\") while parsing: \"custom_metric_total 1 # {b=\\x00\"",
+			err:   "expected label value, got \"\\x00\\\"ssss\\\"} 1\\n\" (\"INVALID\") while parsing: \"custom_metric_total 1 # {b=\\x00\\\"ssss\\\"} 1\\n\"",
 		},
 		{
 			input: "custom_metric_total 1 # {b=\"\x00ss\"} 1\n",
-			err:   "expected label value, got \"\\\"\\x00\" (\"INVALID\") while parsing: \"custom_metric_total 1 # {b=\\\"\\x00\"",
+			err:   "expected label value, got \"\\\"\\x00ss\\\"} 1\\n\" (\"INVALID\") while parsing: \"custom_metric_total 1 # {b=\\\"\\x00ss\\\"} 1\\n\"",
 		},
 	}
 

model/textparse/promlex.l

@@ -66,12 +66,15 @@ C     [^\n]
 #                                     return l.consumeComment()
 <sComment>HELP[\t ]+                  l.state = sMeta1; return tHelp
 <sComment>TYPE[\t ]+                  l.state = sMeta1; return tType
+<sMeta1>\"(\\.|[^\\"])*\"             l.state = sMeta2; return tMName
 <sMeta1>{M}({M}|{D})*                 l.state = sMeta2; return tMName
 <sMeta2>{C}*                          l.state = sInit; return tText
 
 {M}({M}|{D})*                         l.state = sValue; return tMName
 <sValue>\{                            l.state = sLabels; return tBraceOpen
+\{                                    l.state = sLabels; return tBraceOpen
 <sLabels>{L}({L}|{D})*                return tLName
+<sLabels>\"(\\.|[^\\"])*\"            l.state = sLabels; return tQString
 <sLabels>\}                           l.state = sValue; return tBraceClose
 <sLabels>=                            l.state = sLValue; return tEqual
 <sLabels>,                            return tComma

model/textparse/promlex.l.go

@@ -51,19 +51,19 @@ yystate0:
 	case 0: // start condition: INITIAL
 		goto yystart1
 	case 1: // start condition: sComment
-		goto yystart8
+		goto yystart9
 	case 2: // start condition: sMeta1
-		goto yystart19
+		goto yystart20
 	case 3: // start condition: sMeta2
-		goto yystart21
+		goto yystart25
 	case 4: // start condition: sLabels
-		goto yystart24
+		goto yystart28
 	case 5: // start condition: sLValue
-		goto yystart29
-	case 6: // start condition: sValue
-		goto yystart33
-	case 7: // start condition: sTimestamp
 		goto yystart36
+	case 6: // start condition: sValue
+		goto yystart40
+	case 7: // start condition: sTimestamp
+		goto yystart43
 	}
 
 yystate1:
@@ -82,6 +82,8 @@ yystart1:
 		goto yystate3
 	case c == '\x00':
 		goto yystate2
+	case c == '{':
+		goto yystate8
 	}
 
 yystate2:
@@ -123,40 +125,35 @@ yystate7:
 	c = l.next()
 	switch {
 	default:
-		goto yyrule10
+		goto yyrule11
 	case c >= '0' && c <= ':' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
 		goto yystate7
 	}
 
 yystate8:
 	c = l.next()
-yystart8:
+	goto yyrule13
+
+yystate9:
+	c = l.next()
+yystart9:
 	switch {
 	default:
 		goto yyabort
 	case c == 'H':
-		goto yystate9
+		goto yystate10
 	case c == 'T':
-		goto yystate14
+		goto yystate15
 	case c == '\t' || c == ' ':
 		goto yystate3
 	}
 
-yystate9:
-	c = l.next()
-	switch {
-	default:
-		goto yyabort
-	case c == 'E':
-		goto yystate10
-	}
-
 yystate10:
 	c = l.next()
 	switch {
 	default:
 		goto yyabort
-	case c == 'L':
+	case c == 'E':
 		goto yystate11
 	}
 
@@ -165,7 +162,7 @@ yystate11:
 	switch {
 	default:
 		goto yyabort
-	case c == 'P':
+	case c == 'L':
 		goto yystate12
 	}
 
@@ -174,7 +171,7 @@ yystate12:
 	switch {
 	default:
 		goto yyabort
-	case c == '\t' || c == ' ':
+	case c == 'P':
 		goto yystate13
 	}
 
@@ -182,18 +179,18 @@ yystate13:
 	c = l.next()
 	switch {
 	default:
-		goto yyrule6
+		goto yyabort
 	case c == '\t' || c == ' ':
-		goto yystate13
+		goto yystate14
 	}
 
 yystate14:
 	c = l.next()
 	switch {
 	default:
-		goto yyabort
+		goto yyrule6
-	case c == 'Y':
+	case c == '\t' || c == ' ':
-		goto yystate15
+		goto yystate14
 	}
 
 yystate15:
@@ -201,7 +198,7 @@ yystate15:
 	switch {
 	default:
 		goto yyabort
-	case c == 'P':
+	case c == 'Y':
 		goto yystate16
 	}
 
@@ -210,7 +207,7 @@ yystate16:
 	switch {
 	default:
 		goto yyabort
-	case c == 'E':
+	case c == 'P':
 		goto yystate17
 	}
 
@@ -219,7 +216,7 @@ yystate17:
 	switch {
 	default:
 		goto yyabort
-	case c == '\t' || c == ' ':
+	case c == 'E':
 		goto yystate18
 	}
 
@@ -227,167 +224,167 @@ yystate18:
 	c = l.next()
 	switch {
 	default:
-		goto yyrule7
+		goto yyabort
 	case c == '\t' || c == ' ':
-		goto yystate18
+		goto yystate19
 	}
 
 yystate19:
 	c = l.next()
-yystart19:
 	switch {
 	default:
-		goto yyabort
+		goto yyrule7
-	case c == ':' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
-		goto yystate20
 	case c == '\t' || c == ' ':
-		goto yystate3
+		goto yystate19
 	}
 
 yystate20:
 	c = l.next()
+yystart20:
 	switch {
 	default:
-		goto yyrule8
+		goto yyabort
-	case c >= '0' && c <= ':' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
+	case c == '"':
-		goto yystate20
+		goto yystate21
+	case c == ':' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
+		goto yystate24
+	case c == '\t' || c == ' ':
+		goto yystate3
 	}
 
 yystate21:
 	c = l.next()
-yystart21:
 	switch {
 	default:
-		goto yyrule9
+		goto yyabort
-	case c == '\t' || c == ' ':
+	case c == '"':
-		goto yystate23
-	case c >= '\x01' && c <= '\b' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'ÿ':
 		goto yystate22
+	case c == '\\':
+		goto yystate23
+	case c >= '\x01' && c <= '!' || c >= '#' && c <= '[' || c >= ']' && c <= 'ÿ':
+		goto yystate21
 	}
 
 yystate22:
 	c = l.next()
-	switch {
+	goto yyrule8
-	default:
-		goto yyrule9
-	case c >= '\x01' && c <= '\t' || c >= '\v' && c <= 'ÿ':
-		goto yystate22
-	}
 
 yystate23:
 	c = l.next()
 	switch {
 	default:
-		goto yyrule3
+		goto yyabort
-	case c == '\t' || c == ' ':
+	case c >= '\x01' && c <= '\t' || c >= '\v' && c <= 'ÿ':
-		goto yystate23
+		goto yystate21
-	case c >= '\x01' && c <= '\b' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'ÿ':
-		goto yystate22
 	}
 
 yystate24:
 	c = l.next()
-yystart24:
 	switch {
 	default:
-		goto yyabort
+		goto yyrule9
-	case c == ',':
+	case c >= '0' && c <= ':' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
-		goto yystate25
+		goto yystate24
-	case c == '=':
-		goto yystate26
-	case c == '\t' || c == ' ':
-		goto yystate3
-	case c == '}':
-		goto yystate28
-	case c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
-		goto yystate27
 	}
 
 yystate25:
 	c = l.next()
-	goto yyrule15
+yystart25:
+	switch {
+	default:
+		goto yyrule10
+	case c == '\t' || c == ' ':
+		goto yystate27
+	case c >= '\x01' && c <= '\b' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'ÿ':
+		goto yystate26
+	}
 
 yystate26:
 	c = l.next()
-	goto yyrule14
+	switch {
+	default:
+		goto yyrule10
+	case c >= '\x01' && c <= '\t' || c >= '\v' && c <= 'ÿ':
+		goto yystate26
+	}
 
 yystate27:
 	c = l.next()
 	switch {
 	default:
-		goto yyrule12
+		goto yyrule3
-	case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
+	case c == '\t' || c == ' ':
 		goto yystate27
+	case c >= '\x01' && c <= '\b' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'ÿ':
+		goto yystate26
 	}
 
 yystate28:
 	c = l.next()
-	goto yyrule13
+yystart28:
+	switch {
+	default:
+		goto yyabort
+	case c == '"':
+		goto yystate29
+	case c == ',':
+		goto yystate32
+	case c == '=':
+		goto yystate33
+	case c == '\t' || c == ' ':
+		goto yystate3
+	case c == '}':
+		goto yystate35
+	case c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
+		goto yystate34
+	}
 
 yystate29:
 	c = l.next()
-yystart29:
 	switch {
 	default:
 		goto yyabort
 	case c == '"':
 		goto yystate30
-	case c == '\t' || c == ' ':
+	case c == '\\':
-		goto yystate3
+		goto yystate31
+	case c >= '\x01' && c <= '!' || c >= '#' && c <= '[' || c >= ']' && c <= 'ÿ':
+		goto yystate29
 	}
 
 yystate30:
 	c = l.next()
-	switch {
+	goto yyrule15
-	default:
-		goto yyabort
-	case c == '"':
-		goto yystate31
-	case c == '\\':
-		goto yystate32
-	case c >= '\x01' && c <= '!' || c >= '#' && c <= '[' || c >= ']' && c <= 'ÿ':
-		goto yystate30
-	}
 
 yystate31:
-	c = l.next()
-	goto yyrule16
-
-yystate32:
 	c = l.next()
 	switch {
 	default:
 		goto yyabort
 	case c >= '\x01' && c <= '\t' || c >= '\v' && c <= 'ÿ':
-		goto yystate30
+		goto yystate29
 	}
 
+yystate32:
+	c = l.next()
+	goto yyrule18
+
 yystate33:
 	c = l.next()
-yystart33:
+	goto yyrule17
-	switch {
-	default:
-		goto yyabort
-	case c == '\t' || c == ' ':
-		goto yystate3
-	case c == '{':
-		goto yystate35
-	case c >= '\x01' && c <= '\b' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'z' || c >= '|' && c <= 'ÿ':
-		goto yystate34
-	}
 
 yystate34:
 	c = l.next()
 	switch {
 	default:
-		goto yyrule17
+		goto yyrule14
-	case c >= '\x01' && c <= '\b' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'z' || c >= '|' && c <= 'ÿ':
+	case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
 		goto yystate34
 	}
 
 yystate35:
 	c = l.next()
-	goto yyrule11
+	goto yyrule16
 
 yystate36:
 	c = l.next()
@@ -395,25 +392,90 @@ yystart36:
 	switch {
 	default:
 		goto yyabort
-	case c == '\n':
+	case c == '"':
 		goto yystate37
 	case c == '\t' || c == ' ':
 		goto yystate3
-	case c >= '0' && c <= '9':
-		goto yystate38
 	}
 
 yystate37:
 	c = l.next()
-	goto yyrule19
+	switch {
+	default:
+		goto yyabort
+	case c == '"':
+		goto yystate38
+	case c == '\\':
+		goto yystate39
+	case c >= '\x01' && c <= '!' || c >= '#' && c <= '[' || c >= ']' && c <= 'ÿ':
+		goto yystate37
+	}
 
 yystate38:
+	c = l.next()
+	goto yyrule19
+
+yystate39:
 	c = l.next()
 	switch {
 	default:
-		goto yyrule18
+		goto yyabort
+	case c >= '\x01' && c <= '\t' || c >= '\v' && c <= 'ÿ':
+		goto yystate37
+	}
+
+yystate40:
+	c = l.next()
+yystart40:
+	switch {
+	default:
+		goto yyabort
+	case c == '\t' || c == ' ':
+		goto yystate3
+	case c == '{':
+		goto yystate42
+	case c >= '\x01' && c <= '\b' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'z' || c >= '|' && c <= 'ÿ':
+		goto yystate41
+	}
+
+yystate41:
+	c = l.next()
+	switch {
+	default:
+		goto yyrule20
+	case c >= '\x01' && c <= '\b' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'z' || c >= '|' && c <= 'ÿ':
+		goto yystate41
+	}
+
+yystate42:
+	c = l.next()
+	goto yyrule12
+
+yystate43:
+	c = l.next()
+yystart43:
+	switch {
+	default:
+		goto yyabort
+	case c == '\n':
+		goto yystate44
+	case c == '\t' || c == ' ':
+		goto yystate3
 	case c >= '0' && c <= '9':
-		goto yystate38
+		goto yystate45
+	}
+
+yystate44:
+	c = l.next()
+	goto yyrule22
+
+yystate45:
+	c = l.next()
+	switch {
+	default:
+		goto yyrule21
+	case c >= '0' && c <= '9':
+		goto yystate45
 	}
 
 yyrule1: // \0
@@ -451,67 +513,85 @@ yyrule7: // TYPE[\t ]+
 		return tType
 		goto yystate0
 	}
-yyrule8: // {M}({M}|{D})*
+yyrule8: // \"(\\.|[^\\"])*\"
 	{
 		l.state = sMeta2
 		return tMName
 		goto yystate0
 	}
-yyrule9: // {C}*
+yyrule9: // {M}({M}|{D})*
+	{
+		l.state = sMeta2
+		return tMName
+		goto yystate0
+	}
+yyrule10: // {C}*
 	{
 		l.state = sInit
 		return tText
 		goto yystate0
 	}
-yyrule10: // {M}({M}|{D})*
+yyrule11: // {M}({M}|{D})*
 	{
 		l.state = sValue
 		return tMName
 		goto yystate0
 	}
-yyrule11: // \{
+yyrule12: // \{
 	{
 		l.state = sLabels
 		return tBraceOpen
 		goto yystate0
 	}
-yyrule12: // {L}({L}|{D})*
+yyrule13: // \{
+	{
+		l.state = sLabels
+		return tBraceOpen
+		goto yystate0
+	}
+yyrule14: // {L}({L}|{D})*
 	{
 		return tLName
 	}
-yyrule13: // \}
+yyrule15: // \"(\\.|[^\\"])*\"
+	{
+		l.state = sLabels
+		return tQString
+		goto yystate0
+	}
+yyrule16: // \}
 	{
 		l.state = sValue
 		return tBraceClose
 		goto yystate0
 	}
-yyrule14: // =
+yyrule17: // =
 	{
 		l.state = sLValue
 		return tEqual
 		goto yystate0
 	}
-yyrule15: // ,
+yyrule18: // ,
 	{
 		return tComma
 	}
-yyrule16: // \"(\\.|[^\\"])*\"
+yyrule19: // \"(\\.|[^\\"])*\"
 	{
 		l.state = sLabels
 		return tLValue
 		goto yystate0
 	}
-yyrule17: // [^{ \t\n]+
+yyrule20: // [^{ \t\n]+
 	{
 		l.state = sTimestamp
 		return tValue
 		goto yystate0
 	}
-yyrule18: // {D}+
+yyrule21: // {D}+
 	{
 		return tTimestamp
 	}
-yyrule19: // \n
+yyrule22: // \n
 	if true { // avoid go vet determining the below panic will not be reached
 		l.state = sInit
 		return tLinebreak
@@ -520,9 +600,7 @@ yyrule19: // \n
 	panic("unreachable")
 
 yyabort: // no lexem recognized
-	//
 	// silence unused label errors for build and satisfy go vet reachability analysis
-	//
 	{
 		if false {
 			goto yyabort
@@ -534,26 +612,26 @@ yyabort: // no lexem recognized
 			goto yystate1
 		}
 		if false {
-			goto yystate8
+			goto yystate9
 		}
 		if false {
-			goto yystate19
+			goto yystate20
 		}
 		if false {
-			goto yystate21
+			goto yystate25
 		}
 		if false {
-			goto yystate24
+			goto yystate28
-		}
-		if false {
-			goto yystate29
-		}
-		if false {
-			goto yystate33
 		}
 		if false {
 			goto yystate36
 		}
+		if false {
+			goto yystate40
+		}
+		if false {
+			goto yystate43
+		}
 	}
 
 	// Workaround to gobble up comments that started with a HELP or TYPE

model/textparse/promparse.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/gogo/protobuf/types"
 
+	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/model/exemplar"
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
@@ -57,6 +58,7 @@ const (
 	tComment
 	tBlank
 	tMName
+	tQString
 	tBraceOpen
 	tBraceClose
 	tLName
@@ -93,6 +95,8 @@ func (t token) String() string {
 		return "BLANK"
 	case tMName:
 		return "MNAME"
+	case tQString:
+		return "QSTRING"
 	case tBraceOpen:
 		return "BOPEN"
 	case tBraceClose:
@@ -153,6 +157,12 @@ type PromParser struct {
 	ts      int64
 	hasTS   bool
 	start   int
+	// offsets is a list of offsets into series that describe the positions
+	// of the metric name and label names and values for this series.
+	// p.offsets[0] is the start character of the metric name
+	// p.offsets[1] is the end of the metric name
+	// Subsequently, p.offsets is a pair of pair of offsets for the positions
+	// of the label name and value start and end characters.
 	offsets []int
 }
 
@@ -218,9 +228,9 @@ func (p *PromParser) Metric(l *labels.Labels) string {
 	s := string(p.series)
 
 	p.builder.Reset()
-	p.builder.Add(labels.MetricName, s[:p.offsets[0]-p.start])
+	p.builder.Add(labels.MetricName, s[p.offsets[0]-p.start:p.offsets[1]-p.start])
 
-	for i := 1; i < len(p.offsets); i += 4 {
+	for i := 2; i < len(p.offsets); i += 4 {
 		a := p.offsets[i] - p.start
 		b := p.offsets[i+1] - p.start
 		c := p.offsets[i+2] - p.start
@@ -288,7 +298,16 @@ func (p *PromParser) Next() (Entry, error) {
 	case tHelp, tType:
 		switch t2 := p.nextToken(); t2 {
 		case tMName:
-			p.offsets = append(p.offsets, p.l.start, p.l.i)
+			mStart := p.l.start
+			mEnd := p.l.i
+			if p.l.b[mStart] == '"' && p.l.b[mEnd-1] == '"' {
+				mStart++
+				mEnd--
+			}
+			p.offsets = append(p.offsets, mStart, mEnd)
+			if err := p.verifyMetricName(); err != nil {
+				return EntryInvalid, err
+			}
 		default:
 			return EntryInvalid, p.parseError("expected metric name after "+t.String(), t2)
 		}
@@ -300,7 +319,7 @@ func (p *PromParser) Next() (Entry, error) {
 				p.text = []byte{}
 			}
 		default:
-			return EntryInvalid, fmt.Errorf("expected text in %s", t.String())
+			return EntryInvalid, fmt.Errorf("expected text in %s, got %v", t.String(), t2.String())
 		}
 		switch t {
 		case tType:
@@ -338,12 +357,27 @@ func (p *PromParser) Next() (Entry, error) {
 			return EntryInvalid, p.parseError("linebreak expected after comment", t)
 		}
 		return EntryComment, nil
+	case tBraceOpen:
+		// We found a brace, so make room for the eventual metric name. If these
+		// values aren't updated, then the metric name was not set inside the
+		// braces and we can return an error.
+		if len(p.offsets) == 0 {
+			p.offsets = []int{-1, -1}
+		}
+		if err := p.parseLVals(); err != nil {
+			return EntryInvalid, err
+		}
 
-	case tMName:
-		p.offsets = append(p.offsets, p.l.i)
 		p.series = p.l.b[p.start:p.l.i]
-
+		return p.parseMetricSuffix(p.nextToken())
+	case tMName:
+		p.offsets = append(p.offsets, p.start, p.l.i)
+		if err := p.verifyMetricName(); err != nil {
+			return EntryInvalid, err
+		}
+		p.series = p.l.b[p.start:p.l.i]
 		t2 := p.nextToken()
+		// If there's a brace, consume and parse the label values
 		if t2 == tBraceOpen {
 			if err := p.parseLVals(); err != nil {
 				return EntryInvalid, err
@@ -351,32 +385,7 @@ func (p *PromParser) Next() (Entry, error) {
 			p.series = p.l.b[p.start:p.l.i]
 			t2 = p.nextToken()
 		}
-		if t2 != tValue {
+		return p.parseMetricSuffix(t2)
-			return EntryInvalid, p.parseError("expected value after metric", t2)
-		}
-		if p.val, err = parseFloat(yoloString(p.l.buf())); err != nil {
-			return EntryInvalid, fmt.Errorf("%w while parsing: %q", err, p.l.b[p.start:p.l.i])
-		}
-		// Ensure canonical NaN value.
-		if math.IsNaN(p.val) {
-			p.val = math.Float64frombits(value.NormalNaN)
-		}
-		p.hasTS = false
-		switch t := p.nextToken(); t {
-		case tLinebreak:
-			break
-		case tTimestamp:
-			p.hasTS = true
-			if p.ts, err = strconv.ParseInt(yoloString(p.l.buf()), 10, 64); err != nil {
-				return EntryInvalid, fmt.Errorf("%w while parsing: %q", err, p.l.b[p.start:p.l.i])
-			}
-			if t2 := p.nextToken(); t2 != tLinebreak {
-				return EntryInvalid, p.parseError("expected next entry after timestamp", t2)
-			}
-		default:
-			return EntryInvalid, p.parseError("expected timestamp or new record", t)
-		}
-		return EntrySeries, nil
 
 	default:
 		err = p.parseError("expected a valid start token", t)
@@ -384,19 +393,42 @@ func (p *PromParser) Next() (Entry, error) {
 	return EntryInvalid, err
 }
 
+// parseLVals parses the contents inside the braces.
 func (p *PromParser) parseLVals() error {
 	t := p.nextToken()
 	for {
+		curTStart := p.l.start
+		curTI := p.l.i
 		switch t {
 		case tBraceClose:
 			return nil
 		case tLName:
+		case tQString:
 		default:
 			return p.parseError("expected label name", t)
 		}
-		p.offsets = append(p.offsets, p.l.start, p.l.i)
 
-		if t := p.nextToken(); t != tEqual {
+		t = p.nextToken()
+		// A quoted string followed by a comma or brace is a metric name. Set the
+		// offsets and continue processing.
+		if t == tComma || t == tBraceClose {
+			if p.offsets[0] != -1 || p.offsets[1] != -1 {
+				return fmt.Errorf("metric name already set while parsing: %q", p.l.b[p.start:p.l.i])
+			}
+			p.offsets[0] = curTStart + 1
+			p.offsets[1] = curTI - 1
+			if err := p.verifyMetricName(); err != nil {
+				return err
+			}
+			if t == tBraceClose {
+				return nil
+			}
+			t = p.nextToken()
+			continue
+		}
+		// We have a label name.
+		p.offsets = append(p.offsets, curTStart, curTI)
+		if t != tEqual {
 			return p.parseError("expected equal", t)
 		}
 		if t := p.nextToken(); t != tLValue {
@@ -411,12 +443,51 @@ func (p *PromParser) parseLVals() error {
 		p.offsets = append(p.offsets, p.l.start+1, p.l.i-1)
 
 		// Free trailing commas are allowed.
-		if t = p.nextToken(); t == tComma {
+		t = p.nextToken()
+		if t == tComma {
 			t = p.nextToken()
+		} else if t != tBraceClose {
+			return p.parseError("expected comma or brace close", t)
 		}
 	}
 }
 
+// parseMetricSuffix parses the end of the line after the metric name and
+// labels. It starts parsing with the provided token.
+func (p *PromParser) parseMetricSuffix(t token) (Entry, error) {
+	if t != tValue {
+		return EntryInvalid, p.parseError("expected value after metric", t)
+	}
+	var err error
+	if p.val, err = parseFloat(yoloString(p.l.buf())); err != nil {
+		return EntryInvalid, fmt.Errorf("%v while parsing: %q", err, p.l.b[p.start:p.l.i])
+	}
+	// Ensure canonical NaN value.
+	if math.IsNaN(p.val) {
+		p.val = math.Float64frombits(value.NormalNaN)
+	}
+	p.hasTS = false
+	switch t := p.nextToken(); t {
+	case tLinebreak:
+		break
+	case tTimestamp:
+		p.hasTS = true
+		if p.ts, err = strconv.ParseInt(yoloString(p.l.buf()), 10, 64); err != nil {
+			return EntryInvalid, fmt.Errorf("%v while parsing: %q", err, p.l.b[p.start:p.l.i])
+		}
+		if t2 := p.nextToken(); t2 != tLinebreak {
+			return EntryInvalid, p.parseError("expected next entry after timestamp", t2)
+		}
+	default:
+		return EntryInvalid, p.parseError("expected timestamp or new record", t)
+	}
+
+	if p.offsets[0] == -1 {
+		return EntryInvalid, fmt.Errorf("metric name not set while parsing: %q", p.l.b[p.start:p.l.i])
+	}
+	return EntrySeries, nil
+}
+
 var lvalReplacer = strings.NewReplacer(
 	`\"`, "\"",
 	`\\`, "\\",
@@ -432,6 +503,14 @@ func yoloString(b []byte) string {
 	return *((*string)(unsafe.Pointer(&b)))
 }
 
+func (p *PromParser) verifyMetricName() error {
+	m := yoloString(p.l.b[p.offsets[0]:p.offsets[1]])
+	if !model.IsValidMetricName(model.LabelValue(m)) {
+		return fmt.Errorf("metric name %q is not valid", m)
+	}
+	return nil
+}
+
 func parseFloat(s string) (float64, error) {
 	// Keep to pre-Go 1.13 float formats.
 	if strings.ContainsAny(s, "pP_") {

model/textparse/promparse_test.go

@@ -212,6 +212,125 @@ testmetric{label="\"bar\""} 1`
 	require.Len(t, exp, i)
 }
 
+func TestUTF8PromParse(t *testing.T) {
+	model.NameValidationScheme = model.UTF8Validation
+	defer func() {
+		model.NameValidationScheme = model.LegacyValidation
+	}()
+
+	input := `# HELP "go.gc_duration_seconds" A summary of the GC invocation durations.
+# 	TYPE "go.gc_duration_seconds" summary
+{"go.gc_duration_seconds",quantile="0"} 4.9351e-05
+{"go.gc_duration_seconds",quantile="0.25",} 7.424100000000001e-05
+{"go.gc_duration_seconds",quantile="0.5",a="b"} 8.3835e-05
+{"go.gc_duration_seconds",quantile="0.8", a="b"} 8.3835e-05
+{"go.gc_duration_seconds", quantile="0.9", a="b"} 8.3835e-05
+{"go.gc_duration_seconds", quantile="1.0", a="b" } 8.3835e-05
+{ "go.gc_duration_seconds", quantile="1.0", a="b" } 8.3835e-05
+{ "go.gc_duration_seconds", quantile= "1.0", a= "b", } 8.3835e-05
+{ "go.gc_duration_seconds", quantile = "1.0", a = "b" } 8.3835e-05
+{"go.gc_duration_seconds_count"} 99`
+
+	exp := []struct {
+		lset    labels.Labels
+		m       string
+		t       *int64
+		v       float64
+		typ     MetricType
+		help    string
+		comment string
+	}{
+		{
+			m:    "go.gc_duration_seconds",
+			help: "A summary of the GC invocation durations.",
+		}, {
+			m:   "go.gc_duration_seconds",
+			typ: MetricTypeSummary,
+		}, {
+			m:    `{"go.gc_duration_seconds",quantile="0"}`,
+			v:    4.9351e-05,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0"),
+		}, {
+			m:    `{"go.gc_duration_seconds",quantile="0.25",}`,
+			v:    7.424100000000001e-05,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0.25"),
+		}, {
+			m:    `{"go.gc_duration_seconds",quantile="0.5",a="b"}`,
+			v:    8.3835e-05,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0.5", "a", "b"),
+		}, {
+			m:    `{"go.gc_duration_seconds",quantile="0.8", a="b"}`,
+			v:    8.3835e-05,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0.8", "a", "b"),
+		}, {
+			m:    `{"go.gc_duration_seconds", quantile="0.9", a="b"}`,
+			v:    8.3835e-05,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "0.9", "a", "b"),
+		}, {
+			m:    `{"go.gc_duration_seconds", quantile="1.0", a="b" }`,
+			v:    8.3835e-05,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "1.0", "a", "b"),
+		}, {
+			m:    `{ "go.gc_duration_seconds", quantile="1.0", a="b" }`,
+			v:    8.3835e-05,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "1.0", "a", "b"),
+		}, {
+			m:    `{ "go.gc_duration_seconds", quantile= "1.0", a= "b", }`,
+			v:    8.3835e-05,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "1.0", "a", "b"),
+		}, {
+			m:    `{ "go.gc_duration_seconds", quantile = "1.0", a = "b" }`,
+			v:    8.3835e-05,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds", "quantile", "1.0", "a", "b"),
+		}, {
+			m:    `{"go.gc_duration_seconds_count"}`,
+			v:    99,
+			lset: labels.FromStrings("__name__", "go.gc_duration_seconds_count"),
+		},
+	}
+
+	p := NewPromParser([]byte(input))
+	i := 0
+
+	var res labels.Labels
+
+	for {
+		et, err := p.Next()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		require.NoError(t, err)
+
+		switch et {
+		case EntrySeries:
+			m, ts, v := p.Series()
+
+			p.Metric(&res)
+
+			require.Equal(t, exp[i].m, string(m))
+			require.Equal(t, exp[i].t, ts)
+			require.Equal(t, exp[i].v, v)
+			require.Equal(t, exp[i].lset, res)
+
+		case EntryType:
+			m, typ := p.Type()
+			require.Equal(t, exp[i].m, string(m))
+			require.Equal(t, exp[i].typ, typ)
+
+		case EntryHelp:
+			m, h := p.Help()
+			require.Equal(t, exp[i].m, string(m))
+			require.Equal(t, exp[i].help, string(h))
+
+		case EntryComment:
+			require.Equal(t, exp[i].comment, string(p.Comment()))
+		}
+
+		i++
+	}
+	require.Len(t, exp, i)
+}
+
 func TestPromParseErrors(t *testing.T) {
 	cases := []struct {
 		input string
@@ -267,7 +386,7 @@ func TestPromParseErrors(t *testing.T) {
 		},
 		{
 			input: `{a="ok"} 1`,
-			err:   "expected a valid start token, got \"{\" (\"INVALID\") while parsing: \"{\"",
+			err:   "metric name not set while parsing: \"{a=\\\"ok\\\"} 1\\n\"",
 		},
 		{
 			input: "# TYPE #\n#EOF\n",

promql/parser/generated_parser.y

@@ -161,7 +161,7 @@ START_METRIC_SELECTOR
 // Type definitions for grammar rules.
 %type <matchers> label_match_list
 %type <matcher> label_matcher
-%type <item> aggregate_op grouping_label match_op maybe_label metric_identifier unary_op at_modifier_preprocessors
+%type <item> aggregate_op grouping_label match_op maybe_label metric_identifier unary_op at_modifier_preprocessors string_identifier
 %type <labels> label_set metric
 %type <lblList> label_set_list
 %type <label> label_set_item
@@ -582,7 +582,11 @@ label_match_list: label_match_list COMMA label_matcher
                 ;
 
 label_matcher   : IDENTIFIER match_op STRING
-                        { $$ = yylex.(*parser).newLabelMatcher($1, $2, $3);  }
+                        { $$ = yylex.(*parser).newLabelMatcher($1, $2, $3); }
+                | string_identifier match_op STRING
+                        { $$ = yylex.(*parser).newLabelMatcher($1, $2, $3); }
+                | string_identifier
+                        { $$ = yylex.(*parser).newMetricNameMatcher($1); }
                 | IDENTIFIER match_op error
                         { yylex.(*parser).unexpected("label matching", "string"); $$ = nil}
                 | IDENTIFIER error
@@ -901,7 +905,17 @@ string_literal  : STRING
                                 PosRange: $1.PositionRange(),
                         }
                         }
-                        ;
+                ;
+
+string_identifier  : STRING
+                        {
+                        $$ = Item{
+                                Typ: METRIC_IDENTIFIER,
+                                Pos: $1.PositionRange().Start,
+                                Val: yylex.(*parser).unquoteString($1.Val),
+                        }
+                        }
+                ;
 
 /*
  * Wrappers for optional arguments.

promql/parser/parse.go

@@ -775,6 +775,18 @@ func (p *parser) checkAST(node Node) (typ ValueType) {
 			// Skip the check for non-empty matchers because an explicit
 			// metric name is a non-empty matcher.
 			break
+		} else {
+			// We also have to make sure a metric name was not set twice inside the
+			// braces.
+			foundMetricName := ""
+			for _, m := range n.LabelMatchers {
+				if m != nil && m.Name == labels.MetricName {
+					if foundMetricName != "" {
+						p.addParseErrf(n.PositionRange(), "metric name must not be set twice: %q or %q", foundMetricName, m.Value)
+					}
+					foundMetricName = m.Value
+				}
+			}
 		}
 
 		// A Vector selector must contain at least one non-empty matcher to prevent
@@ -858,6 +870,15 @@ func (p *parser) newLabelMatcher(label, operator, value Item) *labels.Matcher {
 	return m
 }
 
+func (p *parser) newMetricNameMatcher(value Item) *labels.Matcher {
+	m, err := labels.NewMatcher(labels.MatchEqual, labels.MetricName, value.Val)
+	if err != nil {
+		p.addParseErr(value.PositionRange(), err)
+	}
+
+	return m
+}
+
 // addOffset is used to set the offset in the generated parser.
 func (p *parser) addOffset(e Node, offset time.Duration) {
 	var orgoffsetp *time.Duration

scrape/scrape_test.go

@@ -1327,7 +1327,7 @@ func TestScrapeLoopAppend(t *testing.T) {
 			// Honor Labels should ignore labels with the same name.
 			title:           "Honor Labels",
 			honorLabels:     true,
-			scrapeLabels:    `metric{n1="1" n2="2"} 0`,
+			scrapeLabels:    `metric{n1="1", n2="2"} 0`,
 			discoveryLabels: []string{"n1", "0"},
 			expLset:         labels.FromStrings("__name__", "metric", "n1", "1", "n2", "2"),
 			expValue:        0,
@@ -1402,7 +1402,7 @@ func TestScrapeLoopAppendForConflictingPrefixedLabels(t *testing.T) {
 		},
 		"One target label collides with existing label, plus existing label already with prefix 'exported": {
 			targetLabels:  []string{"foo", "3"},
-			exposedLabels: `metric{foo="1" exported_foo="2"} 0`,
+			exposedLabels: `metric{foo="1", exported_foo="2"} 0`,
 			expected:      []string{"__name__", "metric", "exported_exported_foo", "1", "exported_foo", "2", "foo", "3"},
 		},
 		"One target label collides with existing label, both already with prefix 'exported'": {
@@ -1412,7 +1412,7 @@ func TestScrapeLoopAppendForConflictingPrefixedLabels(t *testing.T) {
 		},
 		"Two target labels collide with existing labels, both with and without prefix 'exported'": {
 			targetLabels:  []string{"foo", "3", "exported_foo", "4"},
-			exposedLabels: `metric{foo="1" exported_foo="2"} 0`,
+			exposedLabels: `metric{foo="1", exported_foo="2"} 0`,
 			expected: []string{
 				"__name__", "metric", "exported_exported_foo", "1", "exported_exported_exported_foo",
 				"2", "exported_foo", "4", "foo", "3",
@@ -1420,7 +1420,7 @@ func TestScrapeLoopAppendForConflictingPrefixedLabels(t *testing.T) {
 		},
 		"Extreme example": {
 			targetLabels:  []string{"foo", "0", "exported_exported_foo", "1", "exported_exported_exported_foo", "2"},
-			exposedLabels: `metric{foo="3" exported_foo="4" exported_exported_exported_foo="5"} 0`,
+			exposedLabels: `metric{foo="3", exported_foo="4", exported_exported_exported_foo="5"} 0`,
 			expected: []string{
 				"__name__", "metric",
 				"exported_exported_exported_exported_exported_foo", "5",