Add parsing and execution of new test format.

This commit adds a new test structure that parses and executes
the new testing language.

promql/engine.go

@@ -279,20 +279,25 @@ func (ng *Engine) NewRangeQuery(qs string, start, end clientmodel.Timestamp, int
 	if err != nil {
 		return nil, err
 	}
+	qry := ng.newQuery(expr, start, end, interval)
+	qry.q = qs
+
+	return qry, nil
+}
+
+func (ng *Engine) newQuery(expr Expr, start, end clientmodel.Timestamp, interval time.Duration) *query {
 	es := &EvalStmt{
 		Expr:     expr,
 		Start:    start,
 		End:      end,
 		Interval: interval,
 	}
-
 	qry := &query{
-		q:     qs,
 		stmts: Statements{es},
 		ng:    ng,
 		stats: stats.NewTimerGroup(),
 	}
-	return qry, nil
+	return qry
 }
 
 // testStmt is an internal helper statement that allows execution

promql/parse_test.go

@@ -1272,7 +1272,7 @@ var testSeries = []struct {
 	},
 }
 
-// For these tests only, we use the samallest float64 to signal an omitted value.
+// For these tests only, we use the smallest float64 to signal an omitted value.
 const none = math.SmallestNonzeroFloat64
 
 func newSeq(vals ...float64) (res []sequenceValue) {

promql/promql_test.go

@@ -34,12 +34,12 @@ var (
 	fixturesPath = "fixtures"
 
 	reSample = regexp.MustCompile(`^(.*)(?: \=\>|:) (\-?\d+\.?\d*(?:e-?\d+)?|[+-]Inf|NaN) \@\[(\d+)\]$`)
-	minNormal = math.Float64frombits(0x0010000000000000) // The smallest positive normal value of type float64.
+	// minNormal = math.Float64frombits(0x0010000000000000) // The smallest positive normal value of type float64.
 )
 
-const (
-	epsilon = 0.000001 // Relative error allowed for sample values.
-)
+// const (
+// 	epsilon = 0.000001 // Relative error allowed for sample values.
+// )
 
 func annotateWithTime(lines []string, timestamp clientmodel.Timestamp) []string {
 	annotatedLines := []string{}

promql/setup_test.go

@@ -23,7 +23,8 @@ import (
 )
 
 var testSampleInterval = time.Duration(5) * time.Minute
-var testStartTime = clientmodel.Timestamp(0)
+
+// var testStartTime = clientmodel.Timestamp(0)
 
 func getTestValueStream(startVal, endVal, stepVal clientmodel.SampleValue, startTime clientmodel.Timestamp) (resultValues metric.Values) {
 	currentTime := startTime

promql/test.go

@@ -0,0 +1,507 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package promql
+
+import (
+	"fmt"
+	"io/ioutil"
+	"math"
+	"regexp"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	clientmodel "github.com/prometheus/client_golang/model"
+
+	"github.com/prometheus/prometheus/storage"
+	"github.com/prometheus/prometheus/storage/local"
+	"github.com/prometheus/prometheus/storage/metric"
+	"github.com/prometheus/prometheus/utility"
+
+	testutil "github.com/prometheus/prometheus/utility/test"
+)
+
+var (
+	minNormal = math.Float64frombits(0x0010000000000000) // The smallest positive normal value of type float64.
+
+	patSpace       = regexp.MustCompile("[\t ]+")
+	patLoad        = regexp.MustCompile(`^load\s+(.+?)$`)
+	patEvalInstant = regexp.MustCompile(`^eval(?:_(fail|ordered))?\s+instant\s+(?:at\s+(.+?))?\s+(.+)$`)
+)
+
+const (
+	testStartTime = clientmodel.Timestamp(0)
+	epsilon       = 0.000001 // Relative error allowed for sample values.
+	maxErrorCount = 10
+)
+
+// Test is a sequence of read and write commands that are run
+// against a test storage.
+type Test struct {
+	*testing.T
+
+	cmds []testCommand
+
+	storage      local.Storage
+	closeStorage func()
+	queryEngine  *Engine
+}
+
+// NewTest returns an initialized empty Test.
+func NewTest(t *testing.T, input string) (*Test, error) {
+	test := &Test{
+		T:    t,
+		cmds: []testCommand{},
+	}
+	err := test.parse(input)
+	test.clear()
+
+	return test, err
+}
+
+func NewTestFromFile(t *testing.T, filename string) (*Test, error) {
+	content, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	return NewTest(t, string(content))
+}
+
+func raise(line int, format string, v ...interface{}) error {
+	return &ParseErr{
+		Line: line + 1,
+		Err:  fmt.Errorf(format, v...),
+	}
+}
+
+func (t *Test) parseLoad(lines []string, i int) (int, *loadCmd, error) {
+	if !patLoad.MatchString(lines[i]) {
+		return i, nil, raise(i, "invalid load command. (load <step:duration>)")
+	}
+	parts := patLoad.FindStringSubmatch(lines[i])
+
+	gap, err := utility.StringToDuration(parts[1])
+	if err != nil {
+		return i, nil, raise(i, "invalid step definition %q: %s", parts[1], err)
+	}
+	cmd := newLoadCmd(gap)
+	for i+1 < len(lines) {
+		i++
+		defLine := lines[i]
+		if len(defLine) == 0 {
+			i--
+			break
+		}
+		metric, vals, err := parseSeriesDesc(defLine)
+		if err != nil {
+			perr := err.(*ParseErr)
+			perr.Line = i + 1
+			return i, nil, err
+		}
+		cmd.set(metric, vals...)
+	}
+	return i, cmd, nil
+}
+
+func (t *Test) parseEval(lines []string, i int) (int, *evalCmd, error) {
+	if !patEvalInstant.MatchString(lines[i]) {
+		return i, nil, raise(i, "invalid evaluation command. (eval[_fail|_ordered] instant [at <offset:duration>] <query>")
+	}
+	parts := patEvalInstant.FindStringSubmatch(lines[i])
+	var (
+		mod = parts[1]
+		at  = parts[2]
+		qry = parts[3]
+	)
+	expr, err := ParseExpr(qry)
+	if err != nil {
+		perr := err.(*ParseErr)
+		perr.Line = i + 1
+		perr.Pos += strings.Index(lines[i], qry)
+		return i, nil, perr
+	}
+
+	offset, err := utility.StringToDuration(at)
+	if err != nil {
+		return i, nil, raise(i, "invalid step definition %q: %s", parts[1], err)
+	}
+	ts := testStartTime.Add(offset)
+
+	cmd := newEvalCmd(expr, ts, ts, 0)
+	switch mod {
+	case "ordered":
+		cmd.ordered = true
+	case "fail":
+		cmd.fail = true
+	}
+
+	for j := 1; i+1 < len(lines); j++ {
+		i++
+		defLine := lines[i]
+		if len(defLine) == 0 {
+			i--
+			break
+		}
+		if f, err := parseNumber(defLine); err == nil {
+			cmd.expect(0, nil, sequenceValue{value: clientmodel.SampleValue(f)})
+			break
+		}
+		metric, vals, err := parseSeriesDesc(defLine)
+		if err != nil {
+			perr := err.(*ParseErr)
+			perr.Line = i + 1
+			return i, nil, err
+		}
+
+		// Currently, we are not expecting any matrices.
+		if len(vals) > 1 {
+			return i, nil, raise(i, "expecting multiple values in instant evaluation not allowed")
+		}
+		cmd.expect(j, metric, vals...)
+	}
+	return i, cmd, nil
+}
+
+// parse the given command sequence and appends it to the test.
+func (t *Test) parse(input string) error {
+	// Trim lines and remove comments.
+	lines := strings.Split(input, "\n")
+	for i, l := range lines {
+		l = strings.TrimSpace(l)
+		if strings.HasPrefix(l, "#") {
+			l = ""
+		}
+		lines[i] = l
+	}
+	var err error
+
+	// Scan for steps line by line.
+	for i := 0; i < len(lines); i++ {
+		l := lines[i]
+		if len(l) == 0 {
+			continue
+		}
+		var cmd testCommand
+
+		switch c := strings.ToLower(patSpace.Split(l, 2)[0]); {
+		case c == "clear":
+			cmd = &clearCmd{}
+		case c == "load":
+			i, cmd, err = t.parseLoad(lines, i)
+		case strings.HasPrefix(c, "eval"):
+			i, cmd, err = t.parseEval(lines, i)
+		default:
+			return raise(i, "invalid command %q", l)
+		}
+		if err != nil {
+			return err
+		}
+		t.cmds = append(t.cmds, cmd)
+	}
+	return nil
+}
+
+// testCommand is an interface that ensures that only the package internal
+// types can be a valid command for a test.
+type testCommand interface {
+	testCmd()
+}
+
+func (*clearCmd) testCmd() {}
+func (*loadCmd) testCmd()  {}
+func (*evalCmd) testCmd()  {}
+
+// loadCmd is a command that loads sequences of sample values for specific
+// metrics into the storage.
+type loadCmd struct {
+	gap     time.Duration
+	metrics map[clientmodel.Fingerprint]clientmodel.Metric
+	defs    map[clientmodel.Fingerprint]metric.Values
+}
+
+func newLoadCmd(gap time.Duration) *loadCmd {
+	return &loadCmd{
+		gap:     gap,
+		metrics: map[clientmodel.Fingerprint]clientmodel.Metric{},
+		defs:    map[clientmodel.Fingerprint]metric.Values{},
+	}
+}
+
+func (cmd loadCmd) String() string {
+	return "load"
+}
+
+// set a sequence of sample values for the given metric.
+func (cmd *loadCmd) set(m clientmodel.Metric, vals ...sequenceValue) {
+	fp := m.Fingerprint()
+
+	samples := make(metric.Values, 0, len(vals))
+	ts := testStartTime
+	for _, v := range vals {
+		if !v.omitted {
+			samples = append(samples, metric.SamplePair{
+				Timestamp: ts,
+				Value:     v.value,
+			})
+		}
+		ts = ts.Add(cmd.gap)
+	}
+	cmd.defs[fp] = samples
+	cmd.metrics[fp] = m
+}
+
+// append the defined time series to the storage.
+func (cmd *loadCmd) append(a storage.SampleAppender) {
+	for fp, samples := range cmd.defs {
+		met := cmd.metrics[fp]
+		for _, smpl := range samples {
+			s := &clientmodel.Sample{
+				Metric:    met,
+				Value:     smpl.Value,
+				Timestamp: smpl.Timestamp,
+			}
+			a.Append(s)
+		}
+	}
+}
+
+// evalCmd is a command that evaluates an expression for the given time (range)
+// and expects a specific result.
+type evalCmd struct {
+	expr       Expr
+	start, end clientmodel.Timestamp
+	interval   time.Duration
+
+	instant       bool
+	fail, ordered bool
+
+	metrics  map[clientmodel.Fingerprint]clientmodel.Metric
+	expected map[clientmodel.Fingerprint]entry
+}
+
+type entry struct {
+	pos  int
+	vals []sequenceValue
+}
+
+func (e entry) String() string {
+	return fmt.Sprintf("%d: %s", e.pos, e.vals)
+}
+
+func newEvalCmd(expr Expr, start, end clientmodel.Timestamp, interval time.Duration) *evalCmd {
+	return &evalCmd{
+		expr:     expr,
+		start:    start,
+		end:      end,
+		interval: interval,
+		instant:  start == end && interval == 0,
+
+		metrics:  map[clientmodel.Fingerprint]clientmodel.Metric{},
+		expected: map[clientmodel.Fingerprint]entry{},
+	}
+}
+
+func (ev *evalCmd) String() string {
+	return "eval"
+}
+
+// expect adds a new metric with a sequence of values to the set of expected
+// results for the query.
+func (ev *evalCmd) expect(pos int, m clientmodel.Metric, vals ...sequenceValue) {
+	if m == nil {
+		ev.expected[0] = entry{pos: pos, vals: vals}
+		return
+	}
+	fp := m.Fingerprint()
+	ev.metrics[fp] = m
+	ev.expected[fp] = entry{pos: pos, vals: vals}
+}
+
+// compareResult compares the result value with the defined expectation.
+func (ev *evalCmd) compareResult(result Value) error {
+	switch val := result.(type) {
+	case Matrix:
+		if ev.instant {
+			return fmt.Errorf("received range result on instant evaluation")
+		}
+		seen := map[clientmodel.Fingerprint]bool{}
+		for pos, v := range val {
+			fp := v.Metric.Metric.Fingerprint()
+			if _, ok := ev.metrics[fp]; !ok {
+				return fmt.Errorf("unexpected metric %s in result", v.Metric.Metric)
+			}
+			exp := ev.expected[fp]
+			if ev.ordered && exp.pos != pos+1 {
+				return fmt.Errorf("expected metric %s with %v at position %d but was at %d", v.Metric.Metric, exp.vals, exp.pos, pos+1)
+			}
+			for i, expVal := range exp.vals {
+				if !almostEqual(float64(expVal.value), float64(v.Values[i].Value)) {
+					return fmt.Errorf("expected %v for %s but got %v", expVal, v.Metric.Metric, v.Values)
+				}
+			}
+			seen[fp] = true
+		}
+		for fp, expVals := range ev.expected {
+			if !seen[fp] {
+				return fmt.Errorf("expected metric %s with %v not found", ev.metrics[fp], expVals)
+			}
+		}
+
+	case Vector:
+		if !ev.instant {
+			fmt.Errorf("received instant result on range evaluation")
+		}
+		seen := map[clientmodel.Fingerprint]bool{}
+		for pos, v := range val {
+			fp := v.Metric.Metric.Fingerprint()
+			if _, ok := ev.metrics[fp]; !ok {
+				return fmt.Errorf("unexpected metric %s in result", v.Metric.Metric)
+			}
+			exp := ev.expected[fp]
+			if ev.ordered && exp.pos != pos+1 {
+				return fmt.Errorf("expected metric %s with %v at position %d but was at %d", v.Metric.Metric, exp.vals, exp.pos, pos+1)
+			}
+			if !almostEqual(float64(exp.vals[0].value), float64(v.Value)) {
+				return fmt.Errorf("expected %v for %s but got %v", exp.vals[0].value, v.Metric.Metric, v.Value)
+			}
+
+			seen[fp] = true
+		}
+		for fp, expVals := range ev.expected {
+			if !seen[fp] {
+				return fmt.Errorf("expected metric %s with %v not found", ev.metrics[fp], expVals)
+			}
+		}
+
+	case *Scalar:
+		if !almostEqual(float64(ev.expected[0].vals[0].value), float64(val.Value)) {
+			return fmt.Errorf("expected scalar %v but got %v", val.Value, ev.expected[0].vals[0].value)
+		}
+
+	default:
+		panic(fmt.Errorf("promql.Test.compareResult: unexpected result type %T", result))
+	}
+	return nil
+}
+
+// clearCmd is a command that wipes the test's storage state.
+type clearCmd struct{}
+
+func (cmd clearCmd) String() string {
+	return "clear"
+}
+
+// Run executes the command sequence of the test. Until the maximum error number
+// is reached, evaluation errors do not terminate execution.
+func (t *Test) Run() error {
+	for _, cmd := range t.cmds {
+		err := t.exec(cmd)
+		// TODO(fabxc): aggregate command errors, yield diffs for result
+		// comparison errors.
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// exec processes a single step of the test
+func (t *Test) exec(tc testCommand) error {
+	switch cmd := tc.(type) {
+	case *clearCmd:
+		t.clear()
+
+	case *loadCmd:
+		cmd.append(t.storage)
+		t.storage.WaitForIndexing()
+
+	case *evalCmd:
+		q := t.queryEngine.newQuery(cmd.expr, cmd.start, cmd.end, cmd.interval)
+		res := q.Exec()
+		if res.Err != nil {
+			if cmd.fail {
+				return nil
+			}
+			return fmt.Errorf("error evaluating query: %s", res.Err)
+		}
+		if res.Err == nil && cmd.fail {
+			return fmt.Errorf("expected error evaluating query but got none")
+		}
+
+		err := cmd.compareResult(res.Value)
+		if err != nil {
+			return fmt.Errorf("error in %s %s: %s", cmd, cmd.expr, err)
+		}
+
+	default:
+		panic("promql.Test.exec: unknown test command type")
+	}
+	return nil
+}
+
+// clear the current test storage of all inserted samples.
+func (t *Test) clear() {
+	if t.closeStorage != nil {
+		t.closeStorage()
+	}
+	if t.queryEngine != nil {
+		t.queryEngine.Stop()
+	}
+
+	var closer testutil.Closer
+	t.storage, closer = local.NewTestStorage(t, 1)
+
+	t.closeStorage = closer.Close
+	t.queryEngine = NewEngine(t.storage)
+}
+
+func (t *Test) Close() {
+	t.queryEngine.Stop()
+	t.closeStorage()
+}
+
+// samplesAlmostEqual returns true if the two sample lines only differ by a
+// small relative error in their sample value.
+func almostEqual(a, b float64) bool {
+	// NaN has no equality but for testing we still want to know whether both values
+	// are NaN.
+	if math.IsNaN(a) && math.IsNaN(b) {
+		return true
+	}
+
+	// Cf. http://floating-point-gui.de/errors/comparison/
+	if a == b {
+		return true
+	}
+
+	diff := math.Abs(a - b)
+
+	if a == 0 || b == 0 || diff < minNormal {
+		return diff < epsilon*minNormal
+	}
+	return diff/(math.Abs(a)+math.Abs(b)) < epsilon
+}
+
+func parseNumber(s string) (float64, error) {
+	n, err := strconv.ParseInt(s, 0, 64)
+	f := float64(n)
+	if err != nil {
+		f, err = strconv.ParseFloat(s, 64)
+	}
+	if err != nil {
+		return 0, fmt.Errorf("error parsing number: %s", err)
+	}
+	return f, nil
+}