From 93f9b63b0b3de31bd1b04058ada6800811984994 Mon Sep 17 00:00:00 2001 From: Callum Styan Date: Wed, 8 Nov 2023 15:38:42 -0800 Subject: [PATCH] update tests Signed-off-by: Callum Styan --- storage/remote/codec_test.go | 43 ++++++++ storage/remote/queue_manager_test.go | 21 ++-- storage/remote/write_handler_test.go | 147 ++++++++++++++++++--------- 3 files changed, 155 insertions(+), 56 deletions(-) diff --git a/storage/remote/codec_test.go b/storage/remote/codec_test.go index 6937adaa2..45a2f92d7 100644 --- a/storage/remote/codec_test.go +++ b/storage/remote/codec_test.go @@ -127,6 +127,39 @@ var writeRequestWithRefsFixture = &prompb.WriteRequestWithRefs{ }, } +var st = newRwSymbolTable() + +// writeRequestMinimizedFixture represents the same request as writeRequestFixture, but using the minimized representation. +var writeRequestMinimizedFixture = &prompb.MinimizedWriteRequest{ + Timeseries: []prompb.MinimizedTimeSeries{ + { + LabelSymbols: []uint32{ + st.Ref("__name__"), st.Ref("test_metric1"), + st.Ref("b"), st.Ref("c"), + st.Ref("baz"), st.Ref("qux"), + st.Ref("d"), st.Ref("e"), + st.Ref("foo"), st.Ref("bar"), + }, + Samples: []prompb.Sample{{Value: 1, Timestamp: 0}}, + Exemplars: []prompb.Exemplar{{Labels: []prompb.Label{{Name: "f", Value: "g"}}, Value: 1, Timestamp: 0}}, + Histograms: []prompb.Histogram{HistogramToHistogramProto(0, &testHistogram), FloatHistogramToHistogramProto(1, testHistogram.ToFloat())}, + }, + { + LabelSymbols: []uint32{ + st.Ref("__name__"), st.Ref("test_metric1"), + st.Ref("b"), st.Ref("c"), + st.Ref("baz"), st.Ref("qux"), + st.Ref("d"), st.Ref("e"), + st.Ref("foo"), st.Ref("bar"), + }, + Samples: []prompb.Sample{{Value: 2, Timestamp: 1}}, + Exemplars: []prompb.Exemplar{{Labels: []prompb.Label{{Name: "h", Value: "i"}}, Value: 2, Timestamp: 1}}, + Histograms: []prompb.Histogram{HistogramToHistogramProto(2, &testHistogram), FloatHistogramToHistogramProto(3, testHistogram.ToFloat())}, + }, + }, + Symbols: st.LabelsString(), +} + func TestValidateLabelsAndMetricName(t *testing.T) { tests := []struct { input []prompb.Label @@ -588,6 +621,16 @@ func TestDecodeReducedWriteRequest(t *testing.T) { require.Equal(t, writeRequestWithRefsFixture, actual) } +func TestDecodeMinWriteRequest(t *testing.T) { + buf, _, err := buildMinimizedWriteRequest(writeRequestMinimizedFixture.Timeseries, writeRequestMinimizedFixture.Symbols, nil, nil) + + require.NoError(t, err) + + actual, err := DecodeMinimizedWriteRequest(bytes.NewReader(buf)) + require.NoError(t, err) + require.Equal(t, writeRequestMinimizedFixture, actual) +} + func TestReducedWriteRequestToWriteRequest(t *testing.T) { actual, err := ReducedWriteRequestToWriteRequest(writeRequestWithRefsFixture) require.NoError(t, err) diff --git a/storage/remote/queue_manager_test.go b/storage/remote/queue_manager_test.go index b393017cb..621663f92 100644 --- a/storage/remote/queue_manager_test.go +++ b/storage/remote/queue_manager_test.go @@ -62,12 +62,13 @@ func newHighestTimestampMetric() *maxTimestamp { func TestSampleDelivery(t *testing.T) { testcases := []struct { - name string - samples bool - exemplars bool - histograms bool - floatHistograms bool - remoteWrite11 bool + name string + samples bool + exemplars bool + histograms bool + floatHistograms bool + remoteWrite11 bool + remoteWrite11Min bool }{ {samples: true, exemplars: false, histograms: false, floatHistograms: false, name: "samples only"}, {samples: true, exemplars: true, histograms: true, floatHistograms: true, name: "samples, exemplars, and histograms"}, @@ -80,6 +81,12 @@ func TestSampleDelivery(t *testing.T) { {remoteWrite11: true, samples: false, exemplars: true, histograms: false, name: "interned exemplars only"}, {remoteWrite11: true, samples: false, exemplars: false, histograms: true, name: "interned histograms only"}, {remoteWrite11: true, samples: false, exemplars: false, histograms: false, floatHistograms: true, name: "interned float histograms only"}, + + {remoteWrite11Min: true, samples: true, exemplars: false, histograms: false, name: "interned samples only"}, + {remoteWrite11Min: true, samples: true, exemplars: true, histograms: true, floatHistograms: true, name: "interned samples, exemplars, and histograms"}, + {remoteWrite11Min: true, samples: false, exemplars: true, histograms: false, name: "interned exemplars only"}, + {remoteWrite11Min: true, samples: false, exemplars: false, histograms: true, name: "interned histograms only"}, + {remoteWrite11Min: true, samples: false, exemplars: false, histograms: false, floatHistograms: true, name: "interned float histograms only"}, } // Let's create an even number of send batches so we don't run into the @@ -106,7 +113,7 @@ func TestSampleDelivery(t *testing.T) { for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { dir := t.TempDir() - s := NewStorage(nil, nil, nil, dir, defaultFlushDeadline, nil, tc.remoteWrite11, false) + s := NewStorage(nil, nil, nil, dir, defaultFlushDeadline, nil, tc.remoteWrite11, tc.remoteWrite11Min) defer s.Close() var ( diff --git a/storage/remote/write_handler_test.go b/storage/remote/write_handler_test.go index 97980b692..b525eac03 100644 --- a/storage/remote/write_handler_test.go +++ b/storage/remote/write_handler_test.go @@ -83,6 +83,104 @@ func TestRemoteWriteHandler(t *testing.T) { } } +func TestRemoteWriteHandlerReducedFormat(t *testing.T) { + buf, _, err := buildReducedWriteRequest(writeRequestWithRefsFixture.Timeseries, writeRequestWithRefsFixture.StringSymbolTable, nil, nil) + require.NoError(t, err) + + req, err := http.NewRequest("", "", bytes.NewReader(buf)) + req.Header.Set(RemoteWriteVersionHeader, RemoteWriteVersion11HeaderValue) + require.NoError(t, err) + + appendable := &mockAppendable{} + handler := NewWriteHandler(nil, nil, appendable, true, false) + + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, req) + + resp := recorder.Result() + require.Equal(t, http.StatusNoContent, resp.StatusCode) + + i := 0 + j := 0 + k := 0 + // the reduced write request is equivalent to the write request fixture. + // we can use it for + for _, ts := range writeRequestFixture.Timeseries { + labels := labelProtosToLabels(ts.Labels) + for _, s := range ts.Samples { + require.Equal(t, mockSample{labels, s.Timestamp, s.Value}, appendable.samples[i]) + i++ + } + + for _, e := range ts.Exemplars { + exemplarLabels := labelProtosToLabels(e.Labels) + require.Equal(t, mockExemplar{labels, exemplarLabels, e.Timestamp, e.Value}, appendable.exemplars[j]) + j++ + } + + for _, hp := range ts.Histograms { + if hp.IsFloatHistogram() { + fh := FloatHistogramProtoToFloatHistogram(hp) + require.Equal(t, mockHistogram{labels, hp.Timestamp, nil, fh}, appendable.histograms[k]) + } else { + h := HistogramProtoToHistogram(hp) + require.Equal(t, mockHistogram{labels, hp.Timestamp, h, nil}, appendable.histograms[k]) + } + + k++ + } + } +} + +func TestRemoteWriteHandlerMinimizedFormat(t *testing.T) { + buf, _, err := buildMinimizedWriteRequest(writeRequestMinimizedFixture.Timeseries, writeRequestMinimizedFixture.Symbols, nil, nil) + require.NoError(t, err) + + req, err := http.NewRequest("", "", bytes.NewReader(buf)) + req.Header.Set(RemoteWriteVersionHeader, RemoteWriteVersion11HeaderValue) + require.NoError(t, err) + + appendable := &mockAppendable{} + handler := NewWriteHandler(nil, nil, appendable, false, true) + + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, req) + + resp := recorder.Result() + require.Equal(t, http.StatusNoContent, resp.StatusCode) + + i := 0 + j := 0 + k := 0 + // the reduced write request is equivalent to the write request fixture. + // we can use it for + for _, ts := range writeRequestFixture.Timeseries { + ls := labelProtosToLabels(ts.Labels) + for _, s := range ts.Samples { + require.Equal(t, mockSample{ls, s.Timestamp, s.Value}, appendable.samples[i]) + i++ + } + + for _, e := range ts.Exemplars { + exemplarLabels := labelProtosToLabels(e.Labels) + require.Equal(t, mockExemplar{ls, exemplarLabels, e.Timestamp, e.Value}, appendable.exemplars[j]) + j++ + } + + for _, hp := range ts.Histograms { + if hp.IsFloatHistogram() { + fh := FloatHistogramProtoToFloatHistogram(hp) + require.Equal(t, mockHistogram{ls, hp.Timestamp, nil, fh}, appendable.histograms[k]) + } else { + h := HistogramProtoToHistogram(hp) + require.Equal(t, mockHistogram{ls, hp.Timestamp, h, nil}, appendable.histograms[k]) + } + + k++ + } + } +} + func TestOutOfOrderSample(t *testing.T) { buf, _, err := buildWriteRequest([]prompb.TimeSeries{{ Labels: []prompb.Label{{Name: "__name__", Value: "test_metric"}}, @@ -294,55 +392,6 @@ func genSeriesWithSample(numSeries int, ts int64) []prompb.TimeSeries { return series } -func TestRemoteWriteHandlerReducedProtocol(t *testing.T) { - buf, _, err := buildReducedWriteRequest(writeRequestWithRefsFixture.Timeseries, writeRequestWithRefsFixture.StringSymbolTable, nil, nil) - require.NoError(t, err) - - req, err := http.NewRequest("", "", bytes.NewReader(buf)) - req.Header.Set(RemoteWriteVersionHeader, RemoteWriteVersion11HeaderValue) - require.NoError(t, err) - - appendable := &mockAppendable{} - handler := NewWriteHandler(nil, nil, appendable, true, false) - - recorder := httptest.NewRecorder() - handler.ServeHTTP(recorder, req) - - resp := recorder.Result() - require.Equal(t, http.StatusNoContent, resp.StatusCode) - - i := 0 - j := 0 - k := 0 - // the reduced write request is equivalent to the write request fixture. - // we can use it for - for _, ts := range writeRequestFixture.Timeseries { - labels := labelProtosToLabels(ts.Labels) - for _, s := range ts.Samples { - require.Equal(t, mockSample{labels, s.Timestamp, s.Value}, appendable.samples[i]) - i++ - } - - for _, e := range ts.Exemplars { - exemplarLabels := labelProtosToLabels(e.Labels) - require.Equal(t, mockExemplar{labels, exemplarLabels, e.Timestamp, e.Value}, appendable.exemplars[j]) - j++ - } - - for _, hp := range ts.Histograms { - if hp.IsFloatHistogram() { - fh := FloatHistogramProtoToFloatHistogram(hp) - require.Equal(t, mockHistogram{labels, hp.Timestamp, nil, fh}, appendable.histograms[k]) - } else { - h := HistogramProtoToHistogram(hp) - require.Equal(t, mockHistogram{labels, hp.Timestamp, h, nil}, appendable.histograms[k]) - } - - k++ - } - } -} - type mockAppendable struct { latestSample int64 samples []mockSample