From 52c9dc43a3f413bec5bb8a823698bf457e04945d Mon Sep 17 00:00:00 2001 From: Bjoern Rabenstein Date: Thu, 14 Aug 2014 18:23:49 +0200 Subject: [PATCH] Improve testing. In particular, create a fuzz test for time series. Change-Id: I523a17912405a0b6b46bd395c781d201dfe55036 --- storage/local/persistence_test.go | 2 +- storage/local/storage_test.go | 350 +++++++++++++++++++++++++++++- 2 files changed, 343 insertions(+), 9 deletions(-) diff --git a/storage/local/persistence_test.go b/storage/local/persistence_test.go index f214749dc8..6d83009ace 100644 --- a/storage/local/persistence_test.go +++ b/storage/local/persistence_test.go @@ -122,7 +122,7 @@ func TestIndexPersistence(t *testing.T) { for fp, actualSeries := range actual.FingerprintToSeries { expectedSeries := indexes.FingerprintToSeries[fp] if !expectedSeries.metric.Equal(actualSeries.metric) { - t.Fatalf("%s: Got %s; want %s", fp, actualSeries.metric, expectedSeries.metric) + t.Fatalf("%v: Got %s; want %s", fp, actualSeries.metric, expectedSeries.metric) } } diff --git a/storage/local/storage_test.go b/storage/local/storage_test.go index f91754ddf5..66dd2427ba 100644 --- a/storage/local/storage_test.go +++ b/storage/local/storage_test.go @@ -2,10 +2,14 @@ package storage_ng import ( "fmt" + "math" + "math/rand" "testing" + "testing/quick" "time" clientmodel "github.com/prometheus/client_golang/model" + "github.com/prometheus/prometheus/storage/metric" ) func TestChunk(t *testing.T) { @@ -37,7 +41,7 @@ func TestGetValueAtTime(t *testing.T) { samples := make(clientmodel.Samples, 50000) for i := range samples { samples[i] = &clientmodel.Sample{ - Timestamp: clientmodel.Timestamp(time.Duration(i) * time.Second), + Timestamp: clientmodel.Timestamp(2 * i), Value: clientmodel.SampleValue(float64(i) * 0.2), } } @@ -50,23 +54,76 @@ func TestGetValueAtTime(t *testing.T) { it := s.NewIterator(fp) + // #1 Exactly on a sample. for i, expected := range samples { - actual := it.GetValueAtTime(samples[i].Timestamp) + actual := it.GetValueAtTime(expected.Timestamp) + if len(actual) != 1 { + t.Fatalf("1.%d. Expected exactly one result, got %d.", i, len(actual)) + } if expected.Timestamp != actual[0].Timestamp { - t.Fatalf("%d. Got %v; want %v", i, actual[0].Timestamp, expected.Timestamp) + t.Errorf("1.%d. Got %v; want %v", i, actual[0].Timestamp, expected.Timestamp) } if expected.Value != actual[0].Value { - t.Fatalf("%d. Got %v; want %v", i, actual[0].Value, expected.Value) + t.Errorf("1.%d. Got %v; want %v", i, actual[0].Value, expected.Value) } } + + // #2 Between samples. + for i, expected1 := range samples { + if i == len(samples)-1 { + continue + } + expected2 := samples[i+1] + actual := it.GetValueAtTime(expected1.Timestamp + 1) + + if len(actual) != 2 { + t.Fatalf("2.%d. Expected exactly 2 results, got %d.", i, len(actual)) + } + if expected1.Timestamp != actual[0].Timestamp { + t.Errorf("2.%d. Got %v; want %v", i, actual[0].Timestamp, expected1.Timestamp) + } + if expected1.Value != actual[0].Value { + t.Errorf("2.%d. Got %v; want %v", i, actual[0].Value, expected1.Value) + } + if expected2.Timestamp != actual[1].Timestamp { + t.Errorf("2.%d. Got %v; want %v", i, actual[1].Timestamp, expected1.Timestamp) + } + if expected2.Value != actual[1].Value { + t.Errorf("2.%d. Got %v; want %v", i, actual[1].Value, expected1.Value) + } + } + + // #3 Corner cases: Just before the first sample, just after the last. + expected := samples[0] + actual := it.GetValueAtTime(expected.Timestamp - 1) + if len(actual) != 1 { + t.Fatalf("3.1. Expected exactly one result, got %d.", len(actual)) + } + if expected.Timestamp != actual[0].Timestamp { + t.Errorf("3.1. Got %v; want %v", actual[0].Timestamp, expected.Timestamp) + } + if expected.Value != actual[0].Value { + t.Errorf("3.1. Got %v; want %v", actual[0].Value, expected.Value) + } + expected = samples[len(samples)-1] + actual = it.GetValueAtTime(expected.Timestamp + 1) + if len(actual) != 1 { + t.Fatalf("3.2. Expected exactly one result, got %d.", len(actual)) + } + if expected.Timestamp != actual[0].Timestamp { + t.Errorf("3.2. Got %v; want %v", actual[0].Timestamp, expected.Timestamp) + } + if expected.Value != actual[0].Value { + t.Errorf("3.2. Got %v; want %v", actual[0].Value, expected.Value) + } } func TestGetRangeValues(t *testing.T) { samples := make(clientmodel.Samples, 50000) for i := range samples { samples[i] = &clientmodel.Sample{ - Timestamp: clientmodel.Timestamp(time.Duration(i) * time.Second), + Timestamp: clientmodel.Timestamp(2 * i), Value: clientmodel.SampleValue(float64(i) * 0.2), } } @@ -79,16 +136,128 @@ func TestGetRangeValues(t *testing.T) { it := s.NewIterator(fp) + // #1 Zero length interval at sample. for i, expected := range samples { - actual := it.GetValueAtTime(samples[i].Timestamp) + actual := it.GetRangeValues(metric.Interval{ + OldestInclusive: expected.Timestamp, + NewestInclusive: expected.Timestamp, + }) + if len(actual) != 1 { + t.Fatalf("1.%d. Expected exactly one result, got %d.", i, len(actual)) + } if expected.Timestamp != actual[0].Timestamp { - t.Fatalf("%d. Got %v; want %v", i, actual[0].Timestamp, expected.Timestamp) + t.Errorf("1.%d. Got %v; want %v.", i, actual[0].Timestamp, expected.Timestamp) } if expected.Value != actual[0].Value { - t.Fatalf("%d. Got %v; want %v", i, actual[0].Value, expected.Value) + t.Errorf("1.%d. Got %v; want %v.", i, actual[0].Value, expected.Value) } } + + // #2 Zero length interval off sample. + for i, expected := range samples { + actual := it.GetRangeValues(metric.Interval{ + OldestInclusive: expected.Timestamp + 1, + NewestInclusive: expected.Timestamp + 1, + }) + + if len(actual) != 0 { + t.Fatalf("2.%d. Expected no result, got %d.", i, len(actual)) + } + } + + // #3 2sec interval around sample. + for i, expected := range samples { + actual := it.GetRangeValues(metric.Interval{ + OldestInclusive: expected.Timestamp - 1, + NewestInclusive: expected.Timestamp + 1, + }) + + if len(actual) != 1 { + t.Fatalf("3.%d. Expected exactly one result, got %d.", i, len(actual)) + } + if expected.Timestamp != actual[0].Timestamp { + t.Errorf("3.%d. Got %v; want %v.", i, actual[0].Timestamp, expected.Timestamp) + } + if expected.Value != actual[0].Value { + t.Errorf("3.%d. Got %v; want %v.", i, actual[0].Value, expected.Value) + } + } + + // #4 2sec interval sample to sample. + for i, expected1 := range samples { + if i == len(samples)-1 { + continue + } + expected2 := samples[i+1] + actual := it.GetRangeValues(metric.Interval{ + OldestInclusive: expected1.Timestamp, + NewestInclusive: expected1.Timestamp + 2, + }) + + if len(actual) != 2 { + t.Fatalf("4.%d. Expected exactly 2 results, got %d.", i, len(actual)) + } + if expected1.Timestamp != actual[0].Timestamp { + t.Errorf("4.%d. Got %v for 1st result; want %v.", i, actual[0].Timestamp, expected1.Timestamp) + } + if expected1.Value != actual[0].Value { + t.Errorf("4.%d. Got %v for 1st result; want %v.", i, actual[0].Value, expected1.Value) + } + if expected2.Timestamp != actual[1].Timestamp { + t.Errorf("4.%d. Got %v for 2nd result; want %v.", i, actual[1].Timestamp, expected2.Timestamp) + } + if expected2.Value != actual[1].Value { + t.Errorf("4.%d. Got %v for 2nd result; want %v.", i, actual[1].Value, expected2.Value) + } + } + + // #5 corner cases: Interval ends at first sample, interval starts + // at last sample, interval entirely before/after samples. + expected := samples[0] + actual := it.GetRangeValues(metric.Interval{ + OldestInclusive: expected.Timestamp - 2, + NewestInclusive: expected.Timestamp, + }) + if len(actual) != 1 { + t.Fatalf("5.1. Expected exactly one result, got %d.", len(actual)) + } + if expected.Timestamp != actual[0].Timestamp { + t.Errorf("5.1. Got %v; want %v.", actual[0].Timestamp, expected.Timestamp) + } + if expected.Value != actual[0].Value { + t.Errorf("5.1. Got %v; want %v.", actual[0].Value, expected.Value) + } + expected = samples[len(samples)-1] + actual = it.GetRangeValues(metric.Interval{ + OldestInclusive: expected.Timestamp, + NewestInclusive: expected.Timestamp + 2, + }) + if len(actual) != 1 { + t.Fatalf("5.2. Expected exactly one result, got %d.", len(actual)) + } + if expected.Timestamp != actual[0].Timestamp { + t.Errorf("5.2. Got %v; want %v.", actual[0].Timestamp, expected.Timestamp) + } + if expected.Value != actual[0].Value { + t.Errorf("5.2. Got %v; want %v.", actual[0].Value, expected.Value) + } + firstSample := samples[0] + actual = it.GetRangeValues(metric.Interval{ + OldestInclusive: firstSample.Timestamp - 4, + NewestInclusive: firstSample.Timestamp - 2, + }) + if len(actual) != 0 { + t.Fatalf("5.3. Expected no results, got %d.", len(actual)) + } + lastSample := samples[len(samples)-1] + actual = it.GetRangeValues(metric.Interval{ + OldestInclusive: lastSample.Timestamp + 2, + NewestInclusive: lastSample.Timestamp + 4, + }) + if len(actual) != 0 { + t.Fatalf("5.3. Expected no results, got %d.", len(actual)) + } } func BenchmarkAppend(b *testing.B) { @@ -110,3 +279,168 @@ func BenchmarkAppend(b *testing.B) { s.AppendSamples(samples) } + +func TestFuzz(t *testing.T) { + r := rand.New(rand.NewSource(42)) + + check := func() bool { + s, c := NewTestStorage(t) + defer c.Close() + + samples := createRandomSamples(r) + s.AppendSamples(samples) + + return verifyStorage(t, s, samples, r) + } + + if err := quick.Check(check, nil); err != nil { + t.Fatal(err) + } +} + +func createRandomSamples(r *rand.Rand) clientmodel.Samples { + type valueCreator func() clientmodel.SampleValue + type deltaApplier func(clientmodel.SampleValue) clientmodel.SampleValue + + var ( + maxMetrics = 5 + maxCycles = 500 + maxStreakLength = 500 + timestamp = time.Now().Unix() + maxTimeDelta = 1000 + maxTimeDeltaFactor = 10 + generators = []struct { + createValue valueCreator + applyDelta []deltaApplier + }{ + { // "Boolean". + createValue: func() clientmodel.SampleValue { + return clientmodel.SampleValue(r.Intn(2)) + }, + applyDelta: []deltaApplier{ + func(_ clientmodel.SampleValue) clientmodel.SampleValue { + return clientmodel.SampleValue(r.Intn(2)) + }, + }, + }, + { // Integer with int deltas of various byte length. + // TODO: Using larger ints yields even worse results. Improve! + createValue: func() clientmodel.SampleValue { + return clientmodel.SampleValue(r.Int31() - 1<<30) + }, + applyDelta: []deltaApplier{ + func(v clientmodel.SampleValue) clientmodel.SampleValue { + return clientmodel.SampleValue(r.Intn(1<<8) - 1<<7 + int(v)) + }, + func(v clientmodel.SampleValue) clientmodel.SampleValue { + return clientmodel.SampleValue(r.Intn(1<<16) - 1<<15 + int(v)) + }, + func(v clientmodel.SampleValue) clientmodel.SampleValue { + return clientmodel.SampleValue(r.Intn(1<<32) - 1<<31 + int(v)) + }, + }, + }, + { // Float with float32 and float64 deltas. + createValue: func() clientmodel.SampleValue { + return clientmodel.SampleValue(r.NormFloat64()) + }, + applyDelta: []deltaApplier{ + func(v clientmodel.SampleValue) clientmodel.SampleValue { + return v + clientmodel.SampleValue(float32(r.NormFloat64())) + }, + func(v clientmodel.SampleValue) clientmodel.SampleValue { + return v + clientmodel.SampleValue(r.NormFloat64()) + }, + }, + }, + } + ) + + result := clientmodel.Samples{} + + metrics := []clientmodel.Metric{} + for n := r.Intn(maxMetrics); n >= 0; n-- { + metrics = append(metrics, clientmodel.Metric{ + clientmodel.LabelName(fmt.Sprintf("labelname_%d", n+1)): clientmodel.LabelValue(fmt.Sprintf("labelvalue_%d", n+1)), + }) + } + + for n := r.Intn(maxCycles); n >= 0; n-- { + + // Pick a metric for this cycle. + metric := metrics[r.Intn(len(metrics))] + timeDelta := r.Intn(maxTimeDelta) + 1 + generator := generators[r.Intn(len(generators))] + createValue := generator.createValue + applyDelta := generator.applyDelta[r.Intn(len(generator.applyDelta))] + incTimestamp := func() { timestamp += int64(timeDelta * (r.Intn(maxTimeDeltaFactor) + 1)) } + switch r.Intn(4) { + case 0: // A single sample. + result = append(result, &clientmodel.Sample{ + Metric: metric, + Value: createValue(), + Timestamp: clientmodel.Timestamp(timestamp), + }) + incTimestamp() + case 1: // A streak of random sample values. + for n := r.Intn(maxStreakLength); n >= 0; n-- { + result = append(result, &clientmodel.Sample{ + Metric: metric, + Value: createValue(), + Timestamp: clientmodel.Timestamp(timestamp), + }) + incTimestamp() + } + case 2: // A streak of sample values with incremental changes. + value := createValue() + for n := r.Intn(maxStreakLength); n >= 0; n-- { + result = append(result, &clientmodel.Sample{ + Metric: metric, + Value: value, + Timestamp: clientmodel.Timestamp(timestamp), + }) + incTimestamp() + value = applyDelta(value) + } + case 3: // A streak of constant sample values. + value := createValue() + for n := r.Intn(maxStreakLength); n >= 0; n-- { + result = append(result, &clientmodel.Sample{ + Metric: metric, + Value: value, + Timestamp: clientmodel.Timestamp(timestamp), + }) + incTimestamp() + } + } + } + + return result +} + +func verifyStorage(t *testing.T, s Storage, samples clientmodel.Samples, r *rand.Rand) bool { + iters := map[clientmodel.Fingerprint]SeriesIterator{} + result := true + for _, i := range r.Perm(len(samples)) { + sample := samples[i] + fp := sample.Metric.Fingerprint() + iter, ok := iters[fp] + if !ok { + iter = s.NewIterator(fp) + iters[fp] = iter + } + found := iter.GetValueAtTime(sample.Timestamp) + if len(found) != 1 { + t.Errorf("Expected exactly one value, found %d.", len(found)) + return false + } + want := float64(sample.Value) + got := float64(found[0].Value) + // TODO: 0.01 is a horribly large deviation. Improve! + if want != got && (want == 0. || math.Abs(want-got)/want > 0.01) { + t.Errorf("Value mismatch, want %f, got %f.", want, got) + result = false + } + } + return result +}