Enhance lexer testing output

promql/lex.go

@@ -104,6 +104,8 @@ const (
 	itemString
 	itemNumber
 	itemDuration
+	itemBlank
+	itemTimes
 
 	operatorsStart
 	// Operators.
@@ -193,6 +195,8 @@ var itemTypeStr = map[itemType]string{
 	itemComma:        ",",
 	itemAssign:       "=",
 	itemSemicolon:    ";",
+	itemBlank:        "_",
+	itemTimes:        "x",
 
 	itemSUB:      "-",
 	itemADD:      "+",
@@ -214,6 +218,9 @@ func init() {
 	for s, ty := range key {
 		itemTypeStr[ty] = s
 	}
+	// Special numbers.
+	key["inf"] = itemNumber
+	key["nan"] = itemNumber
 }
 
 func (t itemType) String() string {
@@ -450,21 +457,6 @@ func lexStatements(l *lexer) stateFn {
 	case r == '"' || r == '\'':
 		l.stringOpen = r
 		return lexString
-	case r == 'N' || r == 'n' || r == 'I' || r == 'i':
-		n2 := strings.ToLower(l.input[l.pos:])
-		if len(n2) < 3 || !isAlphaNumeric(rune(n2[2])) {
-			if (r == 'N' || r == 'n') && strings.HasPrefix(n2, "an") {
-				l.pos += 2
-				l.emit(itemNumber)
-				break
-			}
-			if (r == 'I' || r == 'i') && strings.HasPrefix(n2, "nf") {
-				l.pos += 2
-				l.emit(itemNumber)
-				break
-			}
-		}
-		fallthrough
 	case isAlpha(r) || r == ':':
 		l.backup()
 		return lexKeywordOrIdentifier
@@ -551,6 +543,34 @@ func lexInsideBraces(l *lexer) stateFn {
 	return lexInsideBraces
 }
 
+// lexValueSequence scans a value sequence of a series description.
+func lexValueSequence(l *lexer) stateFn {
+	switch r := l.next(); {
+	case r == eof:
+		return lexStatements
+	case isSpace(r):
+		lexSpace(l)
+	case r == '+':
+		l.emit(itemADD)
+	case r == '-':
+		l.emit(itemSUB)
+	case r == 'x':
+		l.emit(itemTimes)
+	case r == '_':
+		l.emit(itemBlank)
+	case unicode.IsDigit(r) || (r == '.' && unicode.IsDigit(l.peek())):
+		l.backup()
+		lexNumber(l)
+	case isAlpha(r):
+		l.backup()
+		// We might lex invalid items here but this will be caught by the parser.
+		return lexKeywordOrIdentifier
+	default:
+		return l.errorf("unexpected character in series sequence: %q", r)
+	}
+	return lexValueSequence
+}
+
 // lexString scans a quoted string. The initial quote has already been seen.
 func lexString(l *lexer) stateFn {
 Loop:

promql/lex_test.go

@@ -14,6 +14,7 @@
 package promql
 
 import (
+	"fmt"
 	"reflect"
 	"testing"
 )
@@ -354,6 +355,42 @@ var tests = []struct {
 	}, {
 		input: `]`, fail: true,
 	},
+	// Test series description.
+	{
+		input: `{} _ 1 x .3`,
+		expected: []item{
+			{itemLeftBrace, 0, `{`},
+			{itemRightBrace, 1, `}`},
+			{itemBlank, 3, `_`},
+			{itemNumber, 5, `1`},
+			{itemTimes, 7, `x`},
+			{itemNumber, 9, `.3`},
+		},
+		seriesDesc: true,
+	},
+	{
+		input: `metric +Inf Inf NaN`,
+		expected: []item{
+			{itemIdentifier, 0, `metric`},
+			{itemADD, 7, `+`},
+			{itemNumber, 8, `Inf`},
+			{itemNumber, 12, `Inf`},
+			{itemNumber, 16, `NaN`},
+		},
+		seriesDesc: true,
+	},
+	{
+		input: `metric 1+1x4`,
+		expected: []item{
+			{itemIdentifier, 0, `metric`},
+			{itemNumber, 7, `1`},
+			{itemADD, 8, `+`},
+			{itemNumber, 9, `1`},
+			{itemTimes, 10, `x`},
+			{itemNumber, 11, `4`},
+		},
+		seriesDesc: true,
+	},
 }
 
 // TestLexer tests basic functionality of the lexer. More elaborate tests are implemented
@@ -370,20 +407,32 @@ func TestLexer(t *testing.T) {
 		lastItem := out[len(out)-1]
 		if test.fail {
 			if lastItem.typ != itemError {
-				t.Fatalf("%d: expected lexing error but did not fail", i)
+				t.Logf("%d: input %q", i, test.input)
+				t.Fatalf("expected lexing error but did not fail")
 			}
 			continue
 		}
 		if lastItem.typ == itemError {
-			t.Fatalf("%d: unexpected lexing error: %s", i, lastItem)
+			t.Logf("%d: input %q", i, test.input)
+			t.Fatalf("unexpected lexing error at position %d: %s", lastItem.pos, lastItem)
 		}
 
 		if !reflect.DeepEqual(lastItem, item{itemEOF, Pos(len(test.input)), ""}) {
-			t.Fatalf("%d: lexing error: expected output to end with EOF item", i)
+			t.Logf("%d: input %q", i, test.input)
+			t.Fatalf("lexing error: expected output to end with EOF item.\ngot:\n%s", expectedList(out))
 		}
 		out = out[:len(out)-1]
 		if !reflect.DeepEqual(out, test.expected) {
-			t.Errorf("%d: lexing mismatch:\nexpected: %#v\n-----\ngot: %#v", i, test.expected, out)
+			t.Logf("%d: input %q", i, test.input)
+			t.Fatalf("lexing mismatch:\nexpected:\n%s\ngot:\n%s", expectedList(test.expected), expectedList(out))
 		}
 	}
 }
+
+func expectedList(exp []item) string {
+	s := ""
+	for _, it := range exp {
+		s += fmt.Sprintf("\t%#v\n", it)
+	}
+	return s
+}