diff --git a/model/textparse/openmetricsparse.go b/model/textparse/openmetricsparse.go
index 15a95a959..0fe801087 100644
--- a/model/textparse/openmetricsparse.go
+++ b/model/textparse/openmetricsparse.go
@@ -46,13 +46,6 @@ func (l *openMetricsLexer) buf() []byte {
 	return l.b[l.start:l.i]
 }
 
-func (l *openMetricsLexer) cur() byte {
-	if l.i < len(l.b) {
-		return l.b[l.i]
-	}
-	return byte(' ')
-}
-
 // next advances the openMetricsLexer to the next character.
 func (l *openMetricsLexer) next() byte {
 	l.i++
@@ -223,6 +216,14 @@ func (p *OpenMetricsParser) nextToken() token {
 	return tok
 }
 
+func (p *OpenMetricsParser) parseError(exp string, got token) error {
+	e := p.l.i + 1
+	if len(p.l.b) < e {
+		e = len(p.l.b)
+	}
+	return fmt.Errorf("%s, got %q (%q) while parsing: %q", exp, p.l.b[p.l.start:e], got, p.l.b[p.start:e])
+}
+
 // Next advances the parser to the next sample. It returns false if no
 // more samples were read or an error occurred.
 func (p *OpenMetricsParser) Next() (Entry, error) {
@@ -248,7 +249,7 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
 		case tMName:
 			p.offsets = append(p.offsets, p.l.start, p.l.i)
 		default:
-			return EntryInvalid, parseError("expected metric name after "+t.String(), t2)
+			return EntryInvalid, p.parseError("expected metric name after "+t.String(), t2)
 		}
 		switch t2 := p.nextToken(); t2 {
 		case tText:
@@ -284,7 +285,7 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
 			}
 		case tHelp:
 			if !utf8.Valid(p.text) {
-				return EntryInvalid, errors.New("help text is not a valid utf8 string")
+				return EntryInvalid, fmt.Errorf("help text %q is not a valid utf8 string", p.text)
 			}
 		}
 		switch t {
@@ -297,7 +298,7 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
 			u := yoloString(p.text)
 			if len(u) > 0 {
 				if !strings.HasSuffix(m, u) || len(m) < len(u)+1 || p.l.b[p.offsets[1]-len(u)-1] != '_' {
-					return EntryInvalid, fmt.Errorf("unit not a suffix of metric %q", m)
+					return EntryInvalid, fmt.Errorf("unit %q not a suffix of metric %q", u, m)
 				}
 			}
 			return EntryUnit, nil
@@ -336,10 +337,10 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
 			var ts float64
 			// A float is enough to hold what we need for millisecond resolution.
 			if ts, err = parseFloat(yoloString(p.l.buf()[1:])); err != nil {
-				return EntryInvalid, err
+				return EntryInvalid, fmt.Errorf("%v while parsing: %q", err, p.l.b[p.start:p.l.i])
 			}
 			if math.IsNaN(ts) || math.IsInf(ts, 0) {
-				return EntryInvalid, errors.New("invalid timestamp")
+				return EntryInvalid, fmt.Errorf("invalid timestamp %f", ts)
 			}
 			p.ts = int64(ts * 1000)
 			switch t3 := p.nextToken(); t3 {
@@ -349,15 +350,15 @@ func (p *OpenMetricsParser) Next() (Entry, error) {
 					return EntryInvalid, err
 				}
 			default:
-				return EntryInvalid, parseError("expected next entry after timestamp", t3)
+				return EntryInvalid, p.parseError("expected next entry after timestamp", t3)
 			}
 		default:
-			return EntryInvalid, parseError("expected timestamp or # symbol", t2)
+			return EntryInvalid, p.parseError("expected timestamp or # symbol", t2)
 		}
 		return EntrySeries, nil
 
 	default:
-		err = fmt.Errorf("%q %q is not a valid start token", t, string(p.l.cur()))
+		err = p.parseError("expected a valid start token", t)
 	}
 	return EntryInvalid, err
 }
@@ -395,19 +396,19 @@ func (p *OpenMetricsParser) parseComment() error {
 		var ts float64
 		// A float is enough to hold what we need for millisecond resolution.
 		if ts, err = parseFloat(yoloString(p.l.buf()[1:])); err != nil {
-			return err
+			return fmt.Errorf("%v while parsing: %q", err, p.l.b[p.start:p.l.i])
 		}
 		if math.IsNaN(ts) || math.IsInf(ts, 0) {
-			return errors.New("invalid exemplar timestamp")
+			return fmt.Errorf("invalid exemplar timestamp %f", ts)
 		}
 		p.exemplarTs = int64(ts * 1000)
 		switch t3 := p.nextToken(); t3 {
 		case tLinebreak:
 		default:
-			return parseError("expected next entry after exemplar timestamp", t3)
+			return p.parseError("expected next entry after exemplar timestamp", t3)
 		}
 	default:
-		return parseError("expected timestamp or comment", t2)
+		return p.parseError("expected timestamp or comment", t2)
 	}
 	return nil
 }
@@ -421,21 +422,21 @@ func (p *OpenMetricsParser) parseLVals(offsets []int) ([]int, error) {
 			return offsets, nil
 		case tComma:
 			if first {
-				return nil, parseError("expected label name or left brace", t)
+				return nil, p.parseError("expected label name or left brace", t)
 			}
 			t = p.nextToken()
 			if t != tLName {
-				return nil, parseError("expected label name", t)
+				return nil, p.parseError("expected label name", t)
 			}
 		case tLName:
 			if !first {
-				return nil, parseError("expected comma", t)
+				return nil, p.parseError("expected comma", t)
 			}
 		default:
 			if first {
-				return nil, parseError("expected label name or left brace", t)
+				return nil, p.parseError("expected label name or left brace", t)
 			}
-			return nil, parseError("expected comma or left brace", t)
+			return nil, p.parseError("expected comma or left brace", t)
 
 		}
 		first = false
@@ -444,13 +445,13 @@ func (p *OpenMetricsParser) parseLVals(offsets []int) ([]int, error) {
 		offsets = append(offsets, p.l.start, p.l.i)
 
 		if t := p.nextToken(); t != tEqual {
-			return nil, parseError("expected equal", t)
+			return nil, p.parseError("expected equal", t)
 		}
 		if t := p.nextToken(); t != tLValue {
-			return nil, parseError("expected label value", t)
+			return nil, p.parseError("expected label value", t)
 		}
 		if !utf8.Valid(p.l.buf()) {
-			return nil, errors.New("invalid UTF-8 label value")
+			return nil, fmt.Errorf("invalid UTF-8 label value: %q", p.l.buf())
 		}
 
 		// The openMetricsLexer ensures the value string is quoted. Strip first
@@ -461,11 +462,11 @@ func (p *OpenMetricsParser) parseLVals(offsets []int) ([]int, error) {
 
 func (p *OpenMetricsParser) getFloatValue(t token, after string) (float64, error) {
 	if t != tValue {
-		return 0, parseError(fmt.Sprintf("expected value after %v", after), t)
+		return 0, p.parseError(fmt.Sprintf("expected value after %v", after), t)
 	}
 	val, err := parseFloat(yoloString(p.l.buf()[1:]))
 	if err != nil {
-		return 0, err
+		return 0, fmt.Errorf("%v while parsing: %q", err, p.l.b[p.start:p.l.i])
 	}
 	// Ensure canonical NaN value.
 	if math.IsNaN(p.exemplarVal) {
diff --git a/model/textparse/openmetricsparse_test.go b/model/textparse/openmetricsparse_test.go
index 68b7fea8a..12fb03f01 100644
--- a/model/textparse/openmetricsparse_test.go
+++ b/model/textparse/openmetricsparse_test.go
@@ -293,11 +293,11 @@ func TestOpenMetricsParseErrors(t *testing.T) {
 		},
 		{
 			input: "\n",
-			err:   "\"INVALID\" \"\\n\" is not a valid start token",
+			err:   "expected a valid start token, got \"\\n\" (\"INVALID\") while parsing: \"\\n\"",
 		},
 		{
 			input: "metric",
-			err:   "expected value after metric, got \"EOF\"",
+			err:   "expected value after metric, got \"metric\" (\"EOF\") while parsing: \"metric\"",
 		},
 		{
 			input: "metric 1",
@@ -313,19 +313,19 @@ func TestOpenMetricsParseErrors(t *testing.T) {
 		},
 		{
 			input: "a\n#EOF\n",
-			err:   "expected value after metric, got \"INVALID\"",
+			err:   "expected value after metric, got \"\\n\" (\"INVALID\") while parsing: \"a\\n\"",
 		},
 		{
 			input: "\n\n#EOF\n",
-			err:   "\"INVALID\" \"\\n\" is not a valid start token",
+			err:   "expected a valid start token, got \"\\n\" (\"INVALID\") while parsing: \"\\n\"",
 		},
 		{
 			input: " a 1\n#EOF\n",
-			err:   "\"INVALID\" \" \" is not a valid start token",
+			err:   "expected a valid start token, got \" \" (\"INVALID\") while parsing: \" \"",
 		},
 		{
 			input: "9\n#EOF\n",
-			err:   "\"INVALID\" \"9\" is not a valid start token",
+			err:   "expected a valid start token, got \"9\" (\"INVALID\") while parsing: \"9\"",
 		},
 		{
 			input: "# TYPE u untyped\n#EOF\n",
@@ -337,11 +337,11 @@ func TestOpenMetricsParseErrors(t *testing.T) {
 		},
 		{
 			input: "#  TYPE c counter\n#EOF\n",
-			err:   "\"INVALID\" \" \" is not a valid start token",
+			err:   "expected a valid start token, got \"#  \" (\"INVALID\") while parsing: \"#  \"",
 		},
 		{
 			input: "# TYPE \n#EOF\n",
-			err:   "expected metric name after TYPE, got \"INVALID\"",
+			err:   "expected metric name after TYPE, got \"\\n\" (\"INVALID\") while parsing: \"# TYPE \\n\"",
 		},
 		{
 			input: "# TYPE m\n#EOF\n",
@@ -349,19 +349,19 @@ func TestOpenMetricsParseErrors(t *testing.T) {
 		},
 		{
 			input: "# UNIT metric suffix\n#EOF\n",
-			err:   "unit not a suffix of metric \"metric\"",
+			err:   "unit \"suffix\" not a suffix of metric \"metric\"",
 		},
 		{
 			input: "# UNIT metricsuffix suffix\n#EOF\n",
-			err:   "unit not a suffix of metric \"metricsuffix\"",
+			err:   "unit \"suffix\" not a suffix of metric \"metricsuffix\"",
 		},
 		{
 			input: "# UNIT m suffix\n#EOF\n",
-			err:   "unit not a suffix of metric \"m\"",
+			err:   "unit \"suffix\" not a suffix of metric \"m\"",
 		},
 		{
 			input: "# UNIT \n#EOF\n",
-			err:   "expected metric name after UNIT, got \"INVALID\"",
+			err:   "expected metric name after UNIT, got \"\\n\" (\"INVALID\") while parsing: \"# UNIT \\n\"",
 		},
 		{
 			input: "# UNIT m\n#EOF\n",
@@ -369,7 +369,7 @@ func TestOpenMetricsParseErrors(t *testing.T) {
 		},
 		{
 			input: "# HELP \n#EOF\n",
-			err:   "expected metric name after HELP, got \"INVALID\"",
+			err:   "expected metric name after HELP, got \"\\n\" (\"INVALID\") while parsing: \"# HELP \\n\"",
 		},
 		{
 			input: "# HELP m\n#EOF\n",
@@ -377,27 +377,27 @@ func TestOpenMetricsParseErrors(t *testing.T) {
 		},
 		{
 			input: "a\t1\n#EOF\n",
-			err:   "expected value after metric, got \"INVALID\"",
+			err:   "expected value after metric, got \"\\t\" (\"INVALID\") while parsing: \"a\\t\"",
 		},
 		{
 			input: "a 1\t2\n#EOF\n",
-			err:   "strconv.ParseFloat: parsing \"1\\t2\": invalid syntax",
+			err:   "strconv.ParseFloat: parsing \"1\\t2\": invalid syntax while parsing: \"a 1\\t2\"",
 		},
 		{
 			input: "a 1 2 \n#EOF\n",
-			err:   "expected next entry after timestamp, got \"INVALID\"",
+			err:   "expected next entry after timestamp, got \" \\n\" (\"INVALID\") while parsing: \"a 1 2 \\n\"",
 		},
 		{
 			input: "a 1 2 #\n#EOF\n",
-			err:   "expected next entry after timestamp, got \"TIMESTAMP\"",
+			err:   "expected next entry after timestamp, got \" #\\n\" (\"TIMESTAMP\") while parsing: \"a 1 2 #\\n\"",
 		},
 		{
 			input: "a 1 1z\n#EOF\n",
-			err:   "strconv.ParseFloat: parsing \"1z\": invalid syntax",
+			err:   "strconv.ParseFloat: parsing \"1z\": invalid syntax while parsing: \"a 1 1z\"",
 		},
 		{
 			input: " # EOF\n",
-			err:   "\"INVALID\" \" \" is not a valid start token",
+			err:   "expected a valid start token, got \" \" (\"INVALID\") while parsing: \" \"",
 		},
 		{
 			input: "# EOF\na 1",
@@ -413,7 +413,7 @@ func TestOpenMetricsParseErrors(t *testing.T) {
 		},
 		{
 			input: "#\tTYPE c counter\n",
-			err:   "\"INVALID\" \"\\t\" is not a valid start token",
+			err:   "expected a valid start token, got \"#\\t\" (\"INVALID\") while parsing: \"#\\t\"",
 		},
 		{
 			input: "# TYPE c  counter\n",
@@ -421,79 +421,79 @@ func TestOpenMetricsParseErrors(t *testing.T) {
 		},
 		{
 			input: "a 1 1 1\n# EOF\n",
-			err:   "expected next entry after timestamp, got \"TIMESTAMP\"",
+			err:   "expected next entry after timestamp, got \" 1\\n\" (\"TIMESTAMP\") while parsing: \"a 1 1 1\\n\"",
 		},
 		{
 			input: "a{b='c'} 1\n# EOF\n",
-			err:   "expected label value, got \"INVALID\"",
+			err:   "expected label value, got \"'\" (\"INVALID\") while parsing: \"a{b='\"",
 		},
 		{
 			input: "a{b=\"c\",} 1\n# EOF\n",
-			err:   "expected label name, got \"BCLOSE\"",
+			err:   "expected label name, got \"} \" (\"BCLOSE\") while parsing: \"a{b=\\\"c\\\",} \"",
 		},
 		{
 			input: "a{,b=\"c\"} 1\n# EOF\n",
-			err:   "expected label name or left brace, got \"COMMA\"",
+			err:   "expected label name or left brace, got \",b\" (\"COMMA\") while parsing: \"a{,b\"",
 		},
 		{
 			input: "a{b=\"c\"d=\"e\"} 1\n# EOF\n",
-			err:   "expected comma, got \"LNAME\"",
+			err:   "expected comma, got \"d=\" (\"LNAME\") while parsing: \"a{b=\\\"c\\\"d=\"",
 		},
 		{
 			input: "a{b=\"c\",,d=\"e\"} 1\n# EOF\n",
-			err:   "expected label name, got \"COMMA\"",
+			err:   "expected label name, got \",d\" (\"COMMA\") while parsing: \"a{b=\\\"c\\\",,d\"",
 		},
 		{
 			input: "a{b=\n# EOF\n",
-			err:   "expected label value, got \"INVALID\"",
+			err:   "expected label value, got \"\\n\" (\"INVALID\") while parsing: \"a{b=\\n\"",
 		},
 		{
 			input: "a{\xff=\"foo\"} 1\n# EOF\n",
-			err:   "expected label name or left brace, got \"INVALID\"",
+			err:   "expected label name or left brace, got \"\\xff\" (\"INVALID\") while parsing: \"a{\\xff\"",
 		},
 		{
 			input: "a{b=\"\xff\"} 1\n# EOF\n",
-			err:   "invalid UTF-8 label value",
+			err:   "invalid UTF-8 label value: \"\\\"\\xff\\\"\"",
 		},
 		{
 			input: "a true\n",
-			err:   "strconv.ParseFloat: parsing \"true\": invalid syntax",
+			err:   "strconv.ParseFloat: parsing \"true\": invalid syntax while parsing: \"a true\"",
 		},
 		{
 			input: "something_weird{problem=\"\n# EOF\n",
-			err:   "expected label value, got \"INVALID\"",
+			err:   "expected label value, got \"\\\"\\n\" (\"INVALID\") while parsing: \"something_weird{problem=\\\"\\n\"",
 		},
 		{
 			input: "empty_label_name{=\"\"} 0\n# EOF\n",
-			err:   "expected label name or left brace, got \"EQUAL\"",
+			err:   "expected label name or left brace, got \"=\\\"\" (\"EQUAL\") while parsing: \"empty_label_name{=\\\"\"",
 		},
 		{
 			input: "foo 1_2\n\n# EOF\n",
-			err:   "unsupported character in float",
+			err:   "unsupported character in float while parsing: \"foo 1_2\"",
 		},
 		{
 			input: "foo 0x1p-3\n\n# EOF\n",
-			err:   "unsupported character in float",
+			err:   "unsupported character in float while parsing: \"foo 0x1p-3\"",
 		},
 		{
 			input: "foo 0x1P-3\n\n# EOF\n",
-			err:   "unsupported character in float",
+			err:   "unsupported character in float while parsing: \"foo 0x1P-3\"",
 		},
 		{
 			input: "foo 0 1_2\n\n# EOF\n",
-			err:   "unsupported character in float",
+			err:   "unsupported character in float while parsing: \"foo 0 1_2\"",
 		},
 		{
 			input: "custom_metric_total 1 # {aa=bb}\n# EOF\n",
-			err:   "expected label value, got \"INVALID\"",
+			err:   "expected label value, got \"b\" (\"INVALID\") while parsing: \"custom_metric_total 1 # {aa=b\"",
 		},
 		{
 			input: "custom_metric_total 1 # {aa=\"bb\"}\n# EOF\n",
-			err:   "expected value after exemplar labels, got \"INVALID\"",
+			err:   "expected value after exemplar labels, got \"\\n\" (\"INVALID\") while parsing: \"custom_metric_total 1 # {aa=\\\"bb\\\"}\\n\"",
 		},
 		{
 			input: `custom_metric_total 1 # {aa="bb"}`,
-			err:   "expected value after exemplar labels, got \"EOF\"",
+			err:   "expected value after exemplar labels, got \"}\" (\"EOF\") while parsing: \"custom_metric_total 1 # {aa=\\\"bb\\\"}\"",
 		},
 		{
 			input: `custom_metric 1 # {aa="bb"}`,
@@ -501,55 +501,55 @@ func TestOpenMetricsParseErrors(t *testing.T) {
 		},
 		{
 			input: `custom_metric_total 1 # {aa="bb",,cc="dd"} 1`,
-			err:   "expected label name, got \"COMMA\"",
+			err:   "expected label name, got \",c\" (\"COMMA\") while parsing: \"custom_metric_total 1 # {aa=\\\"bb\\\",,c\"",
 		},
 		{
 			input: `custom_metric_total 1 # {aa="bb"} 1_2`,
-			err:   "unsupported character in float",
+			err:   "unsupported character in float while parsing: \"custom_metric_total 1 # {aa=\\\"bb\\\"} 1_2\"",
 		},
 		{
 			input: `custom_metric_total 1 # {aa="bb"} 0x1p-3`,
-			err:   "unsupported character in float",
+			err:   "unsupported character in float while parsing: \"custom_metric_total 1 # {aa=\\\"bb\\\"} 0x1p-3\"",
 		},
 		{
 			input: `custom_metric_total 1 # {aa="bb"} true`,
-			err:   "strconv.ParseFloat: parsing \"true\": invalid syntax",
+			err:   "strconv.ParseFloat: parsing \"true\": invalid syntax while parsing: \"custom_metric_total 1 # {aa=\\\"bb\\\"} true\"",
 		},
 		{
 			input: `custom_metric_total 1 # {aa="bb",cc=}`,
-			err:   "expected label value, got \"INVALID\"",
+			err:   "expected label value, got \"}\" (\"INVALID\") while parsing: \"custom_metric_total 1 # {aa=\\\"bb\\\",cc=}\"",
 		},
 		{
 			input: `custom_metric_total 1 # {aa=\"\xff\"} 9.0`,
-			err:   "expected label value, got \"INVALID\"",
+			err:   "expected label value, got \"\\\\\" (\"INVALID\") while parsing: \"custom_metric_total 1 # {aa=\\\\\"",
 		},
 		{
 			input: `{b="c",} 1`,
-			err:   `"INVALID" "{" is not a valid start token`,
+			err:   "expected a valid start token, got \"{\" (\"INVALID\") while parsing: \"{\"",
 		},
 		{
 			input: `a 1 NaN`,
-			err:   `invalid timestamp`,
+			err:   `invalid timestamp NaN`,
 		},
 		{
 			input: `a 1 -Inf`,
-			err:   `invalid timestamp`,
+			err:   `invalid timestamp -Inf`,
 		},
 		{
 			input: `a 1 Inf`,
-			err:   `invalid timestamp`,
+			err:   `invalid timestamp +Inf`,
 		},
 		{
 			input: "# TYPE hhh histogram\nhhh_bucket{le=\"+Inf\"} 1 # {aa=\"bb\"} 4 NaN",
-			err:   `invalid exemplar timestamp`,
+			err:   `invalid exemplar timestamp NaN`,
 		},
 		{
 			input: "# TYPE hhh histogram\nhhh_bucket{le=\"+Inf\"} 1 # {aa=\"bb\"} 4 -Inf",
-			err:   `invalid exemplar timestamp`,
+			err:   `invalid exemplar timestamp -Inf`,
 		},
 		{
 			input: "# TYPE hhh histogram\nhhh_bucket{le=\"+Inf\"} 1 # {aa=\"bb\"} 4 Inf",
-			err:   `invalid exemplar timestamp`,
+			err:   `invalid exemplar timestamp +Inf`,
 		},
 	}
 
@@ -586,35 +586,35 @@ func TestOMNullByteHandling(t *testing.T) {
 		},
 		{
 			input: "a{b=\x00\"ssss\"} 1\n# EOF\n",
-			err:   "expected label value, got \"INVALID\"",
+			err:   "expected label value, got \"\\x00\" (\"INVALID\") while parsing: \"a{b=\\x00\"",
 		},
 		{
 			input: "a{b=\"\x00",
-			err:   "expected label value, got \"INVALID\"",
+			err:   "expected label value, got \"\\\"\\x00\" (\"INVALID\") while parsing: \"a{b=\\\"\\x00\"",
 		},
 		{
 			input: "a{b\x00=\"hiih\"}	1",
-			err:   "expected equal, got \"INVALID\"",
+			err:   "expected equal, got \"\\x00\" (\"INVALID\") while parsing: \"a{b\\x00\"",
 		},
 		{
 			input: "a\x00{b=\"ddd\"} 1",
-			err:   "expected value after metric, got \"INVALID\"",
+			err:   "expected value after metric, got \"\\x00\" (\"INVALID\") while parsing: \"a\\x00\"",
 		},
 		{
 			input: "#",
-			err:   "\"INVALID\" \" \" is not a valid start token",
+			err:   "expected a valid start token, got \"#\" (\"INVALID\") while parsing: \"#\"",
 		},
 		{
 			input: "# H",
-			err:   "\"INVALID\" \" \" is not a valid start token",
+			err:   "expected a valid start token, got \"# H\" (\"INVALID\") while parsing: \"# H\"",
 		},
 		{
 			input: "custom_metric_total 1 # {b=\x00\"ssss\"} 1\n",
-			err:   "expected label value, got \"INVALID\"",
+			err:   "expected label value, got \"\\x00\" (\"INVALID\") while parsing: \"custom_metric_total 1 # {b=\\x00\"",
 		},
 		{
 			input: "custom_metric_total 1 # {b=\"\x00ss\"} 1\n",
-			err:   "expected label value, got \"INVALID\"",
+			err:   "expected label value, got \"\\\"\\x00\" (\"INVALID\") while parsing: \"custom_metric_total 1 # {b=\\\"\\x00\"",
 		},
 	}
 
diff --git a/model/textparse/promparse.go b/model/textparse/promparse.go
index b0c963392..2c981f050 100644
--- a/model/textparse/promparse.go
+++ b/model/textparse/promparse.go
@@ -254,8 +254,12 @@ func (p *PromParser) nextToken() token {
 	}
 }
 
-func parseError(exp string, got token) error {
-	return fmt.Errorf("%s, got %q", exp, got)
+func (p *PromParser) parseError(exp string, got token) error {
+	e := p.l.i + 1
+	if len(p.l.b) < e {
+		e = len(p.l.b)
+	}
+	return fmt.Errorf("%s, got %q (%q) while parsing: %q", exp, p.l.b[p.l.start:e], got, p.l.b[p.start:e])
 }
 
 // Next advances the parser to the next sample. It returns false if no
@@ -278,7 +282,7 @@ func (p *PromParser) Next() (Entry, error) {
 		case tMName:
 			p.offsets = append(p.offsets, p.l.start, p.l.i)
 		default:
-			return EntryInvalid, parseError("expected metric name after "+t.String(), t2)
+			return EntryInvalid, p.parseError("expected metric name after "+t.String(), t2)
 		}
 		switch t2 := p.nextToken(); t2 {
 		case tText:
@@ -308,11 +312,11 @@ func (p *PromParser) Next() (Entry, error) {
 			}
 		case tHelp:
 			if !utf8.Valid(p.text) {
-				return EntryInvalid, fmt.Errorf("help text is not a valid utf8 string")
+				return EntryInvalid, fmt.Errorf("help text %q is not a valid utf8 string", p.text)
 			}
 		}
 		if t := p.nextToken(); t != tLinebreak {
-			return EntryInvalid, parseError("linebreak expected after metadata", t)
+			return EntryInvalid, p.parseError("linebreak expected after metadata", t)
 		}
 		switch t {
 		case tHelp:
@@ -323,7 +327,7 @@ func (p *PromParser) Next() (Entry, error) {
 	case tComment:
 		p.text = p.l.buf()
 		if t := p.nextToken(); t != tLinebreak {
-			return EntryInvalid, parseError("linebreak expected after comment", t)
+			return EntryInvalid, p.parseError("linebreak expected after comment", t)
 		}
 		return EntryComment, nil
 
@@ -340,10 +344,10 @@ func (p *PromParser) Next() (Entry, error) {
 			t2 = p.nextToken()
 		}
 		if t2 != tValue {
-			return EntryInvalid, parseError("expected value after metric", t2)
+			return EntryInvalid, p.parseError("expected value after metric", t2)
 		}
 		if p.val, err = parseFloat(yoloString(p.l.buf())); err != nil {
-			return EntryInvalid, err
+			return EntryInvalid, fmt.Errorf("%v while parsing: %q", err, p.l.b[p.start:p.l.i])
 		}
 		// Ensure canonical NaN value.
 		if math.IsNaN(p.val) {
@@ -356,18 +360,18 @@ func (p *PromParser) Next() (Entry, error) {
 		case tTimestamp:
 			p.hasTS = true
 			if p.ts, err = strconv.ParseInt(yoloString(p.l.buf()), 10, 64); err != nil {
-				return EntryInvalid, err
+				return EntryInvalid, fmt.Errorf("%v while parsing: %q", err, p.l.b[p.start:p.l.i])
 			}
 			if t2 := p.nextToken(); t2 != tLinebreak {
-				return EntryInvalid, parseError("expected next entry after timestamp", t2)
+				return EntryInvalid, p.parseError("expected next entry after timestamp", t2)
 			}
 		default:
-			return EntryInvalid, parseError("expected timestamp or new record", t)
+			return EntryInvalid, p.parseError("expected timestamp or new record", t)
 		}
 		return EntrySeries, nil
 
 	default:
-		err = fmt.Errorf("%q is not a valid start token", t)
+		err = p.parseError("expected a valid start token", t)
 	}
 	return EntryInvalid, err
 }
@@ -380,18 +384,18 @@ func (p *PromParser) parseLVals() error {
 			return nil
 		case tLName:
 		default:
-			return parseError("expected label name", t)
+			return p.parseError("expected label name", t)
 		}
 		p.offsets = append(p.offsets, p.l.start, p.l.i)
 
 		if t := p.nextToken(); t != tEqual {
-			return parseError("expected equal", t)
+			return p.parseError("expected equal", t)
 		}
 		if t := p.nextToken(); t != tLValue {
-			return parseError("expected label value", t)
+			return p.parseError("expected label value", t)
 		}
 		if !utf8.Valid(p.l.buf()) {
-			return fmt.Errorf("invalid UTF-8 label value")
+			return fmt.Errorf("invalid UTF-8 label value: %q", p.l.buf())
 		}
 
 		// The promlexer ensures the value string is quoted. Strip first
diff --git a/model/textparse/promparse_test.go b/model/textparse/promparse_test.go
index 6788b3c2e..e0ecf62f5 100644
--- a/model/textparse/promparse_test.go
+++ b/model/textparse/promparse_test.go
@@ -219,63 +219,63 @@ func TestPromParseErrors(t *testing.T) {
 	}{
 		{
 			input: "a",
-			err:   "expected value after metric, got \"INVALID\"",
+			err:   "expected value after metric, got \"\\n\" (\"INVALID\") while parsing: \"a\\n\"",
 		},
 		{
 			input: "a{b='c'} 1\n",
-			err:   "expected label value, got \"INVALID\"",
+			err:   "expected label value, got \"'\" (\"INVALID\") while parsing: \"a{b='\"",
 		},
 		{
 			input: "a{b=\n",
-			err:   "expected label value, got \"INVALID\"",
+			err:   "expected label value, got \"\\n\" (\"INVALID\") while parsing: \"a{b=\\n\"",
 		},
 		{
 			input: "a{\xff=\"foo\"} 1\n",
-			err:   "expected label name, got \"INVALID\"",
+			err:   "expected label name, got \"\\xff\" (\"INVALID\") while parsing: \"a{\\xff\"",
 		},
 		{
 			input: "a{b=\"\xff\"} 1\n",
-			err:   "invalid UTF-8 label value",
+			err:   "invalid UTF-8 label value: \"\\\"\\xff\\\"\"",
 		},
 		{
 			input: "a true\n",
-			err:   "strconv.ParseFloat: parsing \"true\": invalid syntax",
+			err:   "strconv.ParseFloat: parsing \"true\": invalid syntax while parsing: \"a true\"",
 		},
 		{
 			input: "something_weird{problem=\"",
-			err:   "expected label value, got \"INVALID\"",
+			err:   "expected label value, got \"\\\"\\n\" (\"INVALID\") while parsing: \"something_weird{problem=\\\"\\n\"",
 		},
 		{
 			input: "empty_label_name{=\"\"} 0",
-			err:   "expected label name, got \"EQUAL\"",
+			err:   "expected label name, got \"=\\\"\" (\"EQUAL\") while parsing: \"empty_label_name{=\\\"\"",
 		},
 		{
 			input: "foo 1_2\n",
-			err:   "unsupported character in float",
+			err:   "unsupported character in float while parsing: \"foo 1_2\"",
 		},
 		{
 			input: "foo 0x1p-3\n",
-			err:   "unsupported character in float",
+			err:   "unsupported character in float while parsing: \"foo 0x1p-3\"",
 		},
 		{
 			input: "foo 0x1P-3\n",
-			err:   "unsupported character in float",
+			err:   "unsupported character in float while parsing: \"foo 0x1P-3\"",
 		},
 		{
 			input: "foo 0 1_2\n",
-			err:   "expected next entry after timestamp, got \"INVALID\"",
+			err:   "expected next entry after timestamp, got \"_\" (\"INVALID\") while parsing: \"foo 0 1_\"",
 		},
 		{
 			input: `{a="ok"} 1`,
-			err:   `"INVALID" is not a valid start token`,
+			err:   "expected a valid start token, got \"{\" (\"INVALID\") while parsing: \"{\"",
 		},
 		{
 			input: "# TYPE #\n#EOF\n",
-			err:   "expected metric name after TYPE, got \"INVALID\"",
+			err:   "expected metric name after TYPE, got \"#\" (\"INVALID\") while parsing: \"# TYPE #\"",
 		},
 		{
 			input: "# HELP #\n#EOF\n",
-			err:   "expected metric name after HELP, got \"INVALID\"",
+			err:   "expected metric name after HELP, got \"#\" (\"INVALID\") while parsing: \"# HELP #\"",
 		},
 	}
 
@@ -313,23 +313,23 @@ func TestPromNullByteHandling(t *testing.T) {
 		},
 		{
 			input: "a{b=\x00\"ssss\"} 1\n",
-			err:   "expected label value, got \"INVALID\"",
+			err:   "expected label value, got \"\\x00\" (\"INVALID\") while parsing: \"a{b=\\x00\"",
 		},
 		{
 			input: "a{b=\"\x00",
-			err:   "expected label value, got \"INVALID\"",
+			err:   "expected label value, got \"\\\"\\x00\\n\" (\"INVALID\") while parsing: \"a{b=\\\"\\x00\\n\"",
 		},
 		{
 			input: "a{b\x00=\"hiih\"}	1",
-			err:   "expected equal, got \"INVALID\"",
+			err:   "expected equal, got \"\\x00\" (\"INVALID\") while parsing: \"a{b\\x00\"",
 		},
 		{
 			input: "a\x00{b=\"ddd\"} 1",
-			err:   "expected value after metric, got \"INVALID\"",
+			err:   "expected value after metric, got \"\\x00\" (\"INVALID\") while parsing: \"a\\x00\"",
 		},
 		{
 			input: "a 0 1\x00",
-			err:   "expected next entry after timestamp, got \"INVALID\"",
+			err:   "expected next entry after timestamp, got \"\\x00\" (\"INVALID\") while parsing: \"a 0 1\\x00\"",
 		},
 	}