Improve testing.

In particular, create a fuzz test for time series.

Change-Id: I523a17912405a0b6b46bd395c781d201dfe55036

storage/local/persistence_test.go

@@ -122,7 +122,7 @@ func TestIndexPersistence(t *testing.T) {
 	for fp, actualSeries := range actual.FingerprintToSeries {
 		expectedSeries := indexes.FingerprintToSeries[fp]
 		if !expectedSeries.metric.Equal(actualSeries.metric) {
-			t.Fatalf("%s: Got %s; want %s", fp, actualSeries.metric, expectedSeries.metric)
+			t.Fatalf("%v: Got %s; want %s", fp, actualSeries.metric, expectedSeries.metric)
 		}
 	}
 

storage/local/storage_test.go

@@ -2,10 +2,14 @@ package storage_ng
 
 import (
 	"fmt"
+	"math"
+	"math/rand"
 	"testing"
+	"testing/quick"
 	"time"
 
 	clientmodel "github.com/prometheus/client_golang/model"
+	"github.com/prometheus/prometheus/storage/metric"
 )
 
 func TestChunk(t *testing.T) {
@@ -37,7 +41,7 @@ func TestGetValueAtTime(t *testing.T) {
 	samples := make(clientmodel.Samples, 50000)
 	for i := range samples {
 		samples[i] = &clientmodel.Sample{
-			Timestamp: clientmodel.Timestamp(time.Duration(i) * time.Second),
+			Timestamp: clientmodel.Timestamp(2 * i),
 			Value:     clientmodel.SampleValue(float64(i) * 0.2),
 		}
 	}
@@ -50,23 +54,76 @@ func TestGetValueAtTime(t *testing.T) {
 
 	it := s.NewIterator(fp)
 
+	// #1 Exactly on a sample.
 	for i, expected := range samples {
-		actual := it.GetValueAtTime(samples[i].Timestamp)
+		actual := it.GetValueAtTime(expected.Timestamp)
 
+		if len(actual) != 1 {
+			t.Fatalf("1.%d. Expected exactly one result, got %d.", i, len(actual))
+		}
 		if expected.Timestamp != actual[0].Timestamp {
-			t.Fatalf("%d. Got %v; want %v", i, actual[0].Timestamp, expected.Timestamp)
+			t.Errorf("1.%d. Got %v; want %v", i, actual[0].Timestamp, expected.Timestamp)
 		}
 		if expected.Value != actual[0].Value {
-			t.Fatalf("%d. Got %v; want %v", i, actual[0].Value, expected.Value)
+			t.Errorf("1.%d. Got %v; want %v", i, actual[0].Value, expected.Value)
 		}
 	}
+
+	// #2 Between samples.
+	for i, expected1 := range samples {
+		if i == len(samples)-1 {
+			continue
+		}
+		expected2 := samples[i+1]
+		actual := it.GetValueAtTime(expected1.Timestamp + 1)
+
+		if len(actual) != 2 {
+			t.Fatalf("2.%d. Expected exactly 2 results, got %d.", i, len(actual))
+		}
+		if expected1.Timestamp != actual[0].Timestamp {
+			t.Errorf("2.%d. Got %v; want %v", i, actual[0].Timestamp, expected1.Timestamp)
+		}
+		if expected1.Value != actual[0].Value {
+			t.Errorf("2.%d. Got %v; want %v", i, actual[0].Value, expected1.Value)
+		}
+		if expected2.Timestamp != actual[1].Timestamp {
+			t.Errorf("2.%d. Got %v; want %v", i, actual[1].Timestamp, expected1.Timestamp)
+		}
+		if expected2.Value != actual[1].Value {
+			t.Errorf("2.%d. Got %v; want %v", i, actual[1].Value, expected1.Value)
+		}
+	}
+
+	// #3 Corner cases: Just before the first sample, just after the last.
+	expected := samples[0]
+	actual := it.GetValueAtTime(expected.Timestamp - 1)
+	if len(actual) != 1 {
+		t.Fatalf("3.1. Expected exactly one result, got %d.", len(actual))
+	}
+	if expected.Timestamp != actual[0].Timestamp {
+		t.Errorf("3.1. Got %v; want %v", actual[0].Timestamp, expected.Timestamp)
+	}
+	if expected.Value != actual[0].Value {
+		t.Errorf("3.1. Got %v; want %v", actual[0].Value, expected.Value)
+	}
+	expected = samples[len(samples)-1]
+	actual = it.GetValueAtTime(expected.Timestamp + 1)
+	if len(actual) != 1 {
+		t.Fatalf("3.2. Expected exactly one result, got %d.", len(actual))
+	}
+	if expected.Timestamp != actual[0].Timestamp {
+		t.Errorf("3.2. Got %v; want %v", actual[0].Timestamp, expected.Timestamp)
+	}
+	if expected.Value != actual[0].Value {
+		t.Errorf("3.2. Got %v; want %v", actual[0].Value, expected.Value)
+	}
 }
 
 func TestGetRangeValues(t *testing.T) {
 	samples := make(clientmodel.Samples, 50000)
 	for i := range samples {
 		samples[i] = &clientmodel.Sample{
-			Timestamp: clientmodel.Timestamp(time.Duration(i) * time.Second),
+			Timestamp: clientmodel.Timestamp(2 * i),
 			Value:     clientmodel.SampleValue(float64(i) * 0.2),
 		}
 	}
@@ -79,16 +136,128 @@ func TestGetRangeValues(t *testing.T) {
 
 	it := s.NewIterator(fp)
 
+	// #1 Zero length interval at sample.
 	for i, expected := range samples {
-		actual := it.GetValueAtTime(samples[i].Timestamp)
+		actual := it.GetRangeValues(metric.Interval{
+			OldestInclusive: expected.Timestamp,
+			NewestInclusive: expected.Timestamp,
+		})
 
+		if len(actual) != 1 {
+			t.Fatalf("1.%d. Expected exactly one result, got %d.", i, len(actual))
+		}
 		if expected.Timestamp != actual[0].Timestamp {
-			t.Fatalf("%d. Got %v; want %v", i, actual[0].Timestamp, expected.Timestamp)
+			t.Errorf("1.%d. Got %v; want %v.", i, actual[0].Timestamp, expected.Timestamp)
 		}
 		if expected.Value != actual[0].Value {
-			t.Fatalf("%d. Got %v; want %v", i, actual[0].Value, expected.Value)
+			t.Errorf("1.%d. Got %v; want %v.", i, actual[0].Value, expected.Value)
 		}
 	}
+
+	// #2 Zero length interval off sample.
+	for i, expected := range samples {
+		actual := it.GetRangeValues(metric.Interval{
+			OldestInclusive: expected.Timestamp + 1,
+			NewestInclusive: expected.Timestamp + 1,
+		})
+
+		if len(actual) != 0 {
+			t.Fatalf("2.%d. Expected no result, got %d.", i, len(actual))
+		}
+	}
+
+	// #3 2sec interval around sample.
+	for i, expected := range samples {
+		actual := it.GetRangeValues(metric.Interval{
+			OldestInclusive: expected.Timestamp - 1,
+			NewestInclusive: expected.Timestamp + 1,
+		})
+
+		if len(actual) != 1 {
+			t.Fatalf("3.%d. Expected exactly one result, got %d.", i, len(actual))
+		}
+		if expected.Timestamp != actual[0].Timestamp {
+			t.Errorf("3.%d. Got %v; want %v.", i, actual[0].Timestamp, expected.Timestamp)
+		}
+		if expected.Value != actual[0].Value {
+			t.Errorf("3.%d. Got %v; want %v.", i, actual[0].Value, expected.Value)
+		}
+	}
+
+	// #4 2sec interval sample to sample.
+	for i, expected1 := range samples {
+		if i == len(samples)-1 {
+			continue
+		}
+		expected2 := samples[i+1]
+		actual := it.GetRangeValues(metric.Interval{
+			OldestInclusive: expected1.Timestamp,
+			NewestInclusive: expected1.Timestamp + 2,
+		})
+
+		if len(actual) != 2 {
+			t.Fatalf("4.%d. Expected exactly 2 results, got %d.", i, len(actual))
+		}
+		if expected1.Timestamp != actual[0].Timestamp {
+			t.Errorf("4.%d. Got %v for 1st result; want %v.", i, actual[0].Timestamp, expected1.Timestamp)
+		}
+		if expected1.Value != actual[0].Value {
+			t.Errorf("4.%d. Got %v for 1st result; want %v.", i, actual[0].Value, expected1.Value)
+		}
+		if expected2.Timestamp != actual[1].Timestamp {
+			t.Errorf("4.%d. Got %v for 2nd result; want %v.", i, actual[1].Timestamp, expected2.Timestamp)
+		}
+		if expected2.Value != actual[1].Value {
+			t.Errorf("4.%d. Got %v for 2nd result; want %v.", i, actual[1].Value, expected2.Value)
+		}
+	}
+
+	// #5 corner cases: Interval ends at first sample, interval starts
+	// at last sample, interval entirely before/after samples.
+	expected := samples[0]
+	actual := it.GetRangeValues(metric.Interval{
+		OldestInclusive: expected.Timestamp - 2,
+		NewestInclusive: expected.Timestamp,
+	})
+	if len(actual) != 1 {
+		t.Fatalf("5.1. Expected exactly one result, got %d.", len(actual))
+	}
+	if expected.Timestamp != actual[0].Timestamp {
+		t.Errorf("5.1. Got %v; want %v.", actual[0].Timestamp, expected.Timestamp)
+	}
+	if expected.Value != actual[0].Value {
+		t.Errorf("5.1. Got %v; want %v.", actual[0].Value, expected.Value)
+	}
+	expected = samples[len(samples)-1]
+	actual = it.GetRangeValues(metric.Interval{
+		OldestInclusive: expected.Timestamp,
+		NewestInclusive: expected.Timestamp + 2,
+	})
+	if len(actual) != 1 {
+		t.Fatalf("5.2. Expected exactly one result, got %d.", len(actual))
+	}
+	if expected.Timestamp != actual[0].Timestamp {
+		t.Errorf("5.2. Got %v; want %v.", actual[0].Timestamp, expected.Timestamp)
+	}
+	if expected.Value != actual[0].Value {
+		t.Errorf("5.2. Got %v; want %v.", actual[0].Value, expected.Value)
+	}
+	firstSample := samples[0]
+	actual = it.GetRangeValues(metric.Interval{
+		OldestInclusive: firstSample.Timestamp - 4,
+		NewestInclusive: firstSample.Timestamp - 2,
+	})
+	if len(actual) != 0 {
+		t.Fatalf("5.3. Expected no results, got %d.", len(actual))
+	}
+	lastSample := samples[len(samples)-1]
+	actual = it.GetRangeValues(metric.Interval{
+		OldestInclusive: lastSample.Timestamp + 2,
+		NewestInclusive: lastSample.Timestamp + 4,
+	})
+	if len(actual) != 0 {
+		t.Fatalf("5.3. Expected no results, got %d.", len(actual))
+	}
 }
 
 func BenchmarkAppend(b *testing.B) {
@@ -110,3 +279,168 @@ func BenchmarkAppend(b *testing.B) {
 
 	s.AppendSamples(samples)
 }
+
+func TestFuzz(t *testing.T) {
+	r := rand.New(rand.NewSource(42))
+
+	check := func() bool {
+		s, c := NewTestStorage(t)
+		defer c.Close()
+
+		samples := createRandomSamples(r)
+		s.AppendSamples(samples)
+
+		return verifyStorage(t, s, samples, r)
+	}
+
+	if err := quick.Check(check, nil); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func createRandomSamples(r *rand.Rand) clientmodel.Samples {
+	type valueCreator func() clientmodel.SampleValue
+	type deltaApplier func(clientmodel.SampleValue) clientmodel.SampleValue
+
+	var (
+		maxMetrics         = 5
+		maxCycles          = 500
+		maxStreakLength    = 500
+		timestamp          = time.Now().Unix()
+		maxTimeDelta       = 1000
+		maxTimeDeltaFactor = 10
+		generators         = []struct {
+			createValue valueCreator
+			applyDelta  []deltaApplier
+		}{
+			{ // "Boolean".
+				createValue: func() clientmodel.SampleValue {
+					return clientmodel.SampleValue(r.Intn(2))
+				},
+				applyDelta: []deltaApplier{
+					func(_ clientmodel.SampleValue) clientmodel.SampleValue {
+						return clientmodel.SampleValue(r.Intn(2))
+					},
+				},
+			},
+			{ // Integer with int deltas of various byte length.
+				// TODO: Using larger ints yields even worse results. Improve!
+				createValue: func() clientmodel.SampleValue {
+					return clientmodel.SampleValue(r.Int31() - 1<<30)
+				},
+				applyDelta: []deltaApplier{
+					func(v clientmodel.SampleValue) clientmodel.SampleValue {
+						return clientmodel.SampleValue(r.Intn(1<<8) - 1<<7 + int(v))
+					},
+					func(v clientmodel.SampleValue) clientmodel.SampleValue {
+						return clientmodel.SampleValue(r.Intn(1<<16) - 1<<15 + int(v))
+					},
+					func(v clientmodel.SampleValue) clientmodel.SampleValue {
+						return clientmodel.SampleValue(r.Intn(1<<32) - 1<<31 + int(v))
+					},
+				},
+			},
+			{ // Float with float32 and float64 deltas.
+				createValue: func() clientmodel.SampleValue {
+					return clientmodel.SampleValue(r.NormFloat64())
+				},
+				applyDelta: []deltaApplier{
+					func(v clientmodel.SampleValue) clientmodel.SampleValue {
+						return v + clientmodel.SampleValue(float32(r.NormFloat64()))
+					},
+					func(v clientmodel.SampleValue) clientmodel.SampleValue {
+						return v + clientmodel.SampleValue(r.NormFloat64())
+					},
+				},
+			},
+		}
+	)
+
+	result := clientmodel.Samples{}
+
+	metrics := []clientmodel.Metric{}
+	for n := r.Intn(maxMetrics); n >= 0; n-- {
+		metrics = append(metrics, clientmodel.Metric{
+			clientmodel.LabelName(fmt.Sprintf("labelname_%d", n+1)): clientmodel.LabelValue(fmt.Sprintf("labelvalue_%d", n+1)),
+		})
+	}
+
+	for n := r.Intn(maxCycles); n >= 0; n-- {
+
+		// Pick a metric for this cycle.
+		metric := metrics[r.Intn(len(metrics))]
+		timeDelta := r.Intn(maxTimeDelta) + 1
+		generator := generators[r.Intn(len(generators))]
+		createValue := generator.createValue
+		applyDelta := generator.applyDelta[r.Intn(len(generator.applyDelta))]
+		incTimestamp := func() { timestamp += int64(timeDelta * (r.Intn(maxTimeDeltaFactor) + 1)) }
+		switch r.Intn(4) {
+		case 0: // A single sample.
+			result = append(result, &clientmodel.Sample{
+				Metric:    metric,
+				Value:     createValue(),
+				Timestamp: clientmodel.Timestamp(timestamp),
+			})
+			incTimestamp()
+		case 1: // A streak of random sample values.
+			for n := r.Intn(maxStreakLength); n >= 0; n-- {
+				result = append(result, &clientmodel.Sample{
+					Metric:    metric,
+					Value:     createValue(),
+					Timestamp: clientmodel.Timestamp(timestamp),
+				})
+				incTimestamp()
+			}
+		case 2: // A streak of sample values with incremental changes.
+			value := createValue()
+			for n := r.Intn(maxStreakLength); n >= 0; n-- {
+				result = append(result, &clientmodel.Sample{
+					Metric:    metric,
+					Value:     value,
+					Timestamp: clientmodel.Timestamp(timestamp),
+				})
+				incTimestamp()
+				value = applyDelta(value)
+			}
+		case 3: // A streak of constant sample values.
+			value := createValue()
+			for n := r.Intn(maxStreakLength); n >= 0; n-- {
+				result = append(result, &clientmodel.Sample{
+					Metric:    metric,
+					Value:     value,
+					Timestamp: clientmodel.Timestamp(timestamp),
+				})
+				incTimestamp()
+			}
+		}
+	}
+
+	return result
+}
+
+func verifyStorage(t *testing.T, s Storage, samples clientmodel.Samples, r *rand.Rand) bool {
+	iters := map[clientmodel.Fingerprint]SeriesIterator{}
+	result := true
+	for _, i := range r.Perm(len(samples)) {
+		sample := samples[i]
+		fp := sample.Metric.Fingerprint()
+		iter, ok := iters[fp]
+		if !ok {
+			iter = s.NewIterator(fp)
+			iters[fp] = iter
+		}
+		found := iter.GetValueAtTime(sample.Timestamp)
+		if len(found) != 1 {
+			t.Errorf("Expected exactly one value, found %d.", len(found))
+			return false
+		}
+		want := float64(sample.Value)
+		got := float64(found[0].Value)
+		// TODO: 0.01 is a horribly large deviation. Improve!
+		if want != got && (want == 0. || math.Abs(want-got)/want > 0.01) {
+			t.Errorf("Value mismatch, want %f, got %f.", want, got)
+			result = false
+		}
+	}
+	return result
+}