mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2024-12-25 05:34:05 -08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Reduce memory usage of remote read by reducing pointer usage. (#4655)

Browse source 
Signed-off-by: Tom Wilkie <tom.wilkie@gmail.com>
This commit is contained in:
Tom Wilkie 2018-09-25 19:14:00 +01:00 committed by GitHub
parent 5790d23fd8
commit d3a1ff1abf
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 53 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
documentation/examples/remote_storage/remote_storage_adapter/influxdb/client.go
View file

@ -248,8 +248,8 @@ func tagsToLabelPairs(name string, tags map[string]string) []*prompb.Label {
return pairs return pairs
} }
func valuesToSamples(values [][]interface{}) ([]*prompb.Sample, error) { func valuesToSamples(values [][]interface{}) ([]prompb.Sample, error) {
samples := make([]*prompb.Sample, 0, len(values)) samples := make([]prompb.Sample, 0, len(values))
for _, v := range values { for _, v := range values {
if len(v) != 2 { if len(v) != 2 {
return nil, fmt.Errorf("bad sample tuple length, expected [<timestamp>, <value>], got %v", v) return nil, fmt.Errorf("bad sample tuple length, expected [<timestamp>, <value>], got %v", v)
@ -275,7 +275,7 @@ func valuesToSamples(values [][]interface{}) ([]*prompb.Sample, error) {
return nil, fmt.Errorf("unable to convert sample value to float64: %v", err) return nil, fmt.Errorf("unable to convert sample value to float64: %v", err)
} }
samples = append(samples, &prompb.Sample{ samples = append(samples, prompb.Sample{
Timestamp: timestamp, Timestamp: timestamp,
Value: value, Value: value,
}) })
@ -285,8 +285,8 @@ func valuesToSamples(values [][]interface{}) ([]*prompb.Sample, error) {
// mergeSamples merges two lists of sample pairs and removes duplicate // mergeSamples merges two lists of sample pairs and removes duplicate
// timestamps. It assumes that both lists are sorted by timestamp. // timestamps. It assumes that both lists are sorted by timestamp.
func mergeSamples(a, b []*prompb.Sample) []*prompb.Sample { func mergeSamples(a, b []prompb.Sample) []prompb.Sample {
result := make([]*prompb.Sample, 0, len(a)+len(b)) result := make([]prompb.Sample, 0, len(a)+len(b))
i, j := 0, 0 i, j := 0, 0
for i < len(a) && j < len(b) { for i < len(a) && j < len(b) {
if a[i].Timestamp < b[j].Timestamp { if a[i].Timestamp < b[j].Timestamp {

58
prompb/types.pb.go
View file

@ -66,8 +66,8 @@ func (m *Sample) GetTimestamp() int64 {
} }
type TimeSeries struct { type TimeSeries struct {
Labels []*Label `protobuf:"bytes,1,rep,name=labels" json:"labels,omitempty"` Labels []*Label `protobuf:"bytes,1,rep,name=labels" json:"labels,omitempty"`
Samples []*Sample `protobuf:"bytes,2,rep,name=samples" json:"samples,omitempty"` Samples []Sample `protobuf:"bytes,2,rep,name=samples" json:"samples"`
} }
func (m *TimeSeries) Reset() { *m = TimeSeries{} } func (m *TimeSeries) Reset() { *m = TimeSeries{} }
@ -82,7 +82,7 @@ func (m *TimeSeries) GetLabels() []*Label {
return nil return nil
} }
func (m *TimeSeries) GetSamples() []*Sample { func (m *TimeSeries) GetSamples() []Sample {
if m != nil { if m != nil {
return m.Samples return m.Samples
} }
@ -720,7 +720,7 @@ func (m *TimeSeries) Unmarshal(dAtA []byte) error {
if postIndex > l { if postIndex > l {
return io.ErrUnexpectedEOF return io.ErrUnexpectedEOF
} }
m.Samples = append(m.Samples, &Sample{}) m.Samples = append(m.Samples, Sample{})
if err := m.Samples[len(m.Samples)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { if err := m.Samples[len(m.Samples)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err return err
} }
@ -1306,29 +1306,29 @@ var (
func init() { proto.RegisterFile("types.proto", fileDescriptorTypes) } func init() { proto.RegisterFile("types.proto", fileDescriptorTypes) }
var fileDescriptorTypes = []byte{ var fileDescriptorTypes = []byte{
// 380 bytes of a gzipped FileDescriptorProto // 383 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x92, 0xcd, 0x0a, 0xd3, 0x40, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x92, 0xcf, 0xca, 0xd3, 0x40,
0x14, 0x85, 0x3b, 0x49, 0x3a, 0xb1, 0xb7, 0x22, 0x71, 0xa8, 0x18, 0x45, 0x6b, 0xc9, 0x2a, 0x82, 0x14, 0xc5, 0xbf, 0x49, 0xd2, 0x89, 0xbd, 0x9f, 0x48, 0x1c, 0x2a, 0x46, 0xd1, 0x5a, 0xb2, 0x8a,
0xa4, 0xb4, 0xae, 0x04, 0x57, 0x85, 0x80, 0x8b, 0x46, 0xe8, 0xb4, 0x2b, 0x37, 0x32, 0x6d, 0xaf, 0x9b, 0x94, 0xd6, 0x95, 0xe0, 0xaa, 0x10, 0x70, 0xd1, 0x08, 0x9d, 0x76, 0xe5, 0x46, 0xa6, 0xed,
0x6d, 0x24, 0x3f, 0x43, 0x66, 0x2a, 0xf4, 0x41, 0x7c, 0xa7, 0x2e, 0x7d, 0x02, 0x91, 0x3e, 0x89, 0xb5, 0x8d, 0xe6, 0xcf, 0x90, 0x99, 0x0a, 0x7d, 0x10, 0xdf, 0xa9, 0x4b, 0x9f, 0x40, 0xa4, 0x4f,
0xcc, 0xa4, 0xb5, 0x05, 0x05, 0x77, 0xf7, 0x9c, 0x7b, 0x2e, 0xe7, 0x4b, 0x18, 0xe8, 0xeb, 0xa3, 0x22, 0x33, 0x49, 0x6d, 0x41, 0xc1, 0xdd, 0x3d, 0xe7, 0xfe, 0x6e, 0xce, 0x49, 0x08, 0xdc, 0xeb,
0x44, 0x95, 0xc8, 0xa6, 0xd6, 0x35, 0x03, 0xd9, 0xd4, 0x25, 0xea, 0x3d, 0x1e, 0xd4, 0xf3, 0xc1, 0xa3, 0x44, 0x95, 0xc8, 0xa6, 0xd6, 0x35, 0x03, 0xd9, 0xd4, 0x25, 0xea, 0x3d, 0x1e, 0xd4, 0xf3,
0xae, 0xde, 0xd5, 0xd6, 0x1e, 0x9b, 0xa9, 0x4d, 0x44, 0xef, 0x81, 0x2e, 0x45, 0x29, 0x0b, 0x64, 0xc1, 0xae, 0xde, 0xd5, 0xd6, 0x1e, 0x9b, 0xa9, 0x25, 0xa2, 0x77, 0x40, 0x97, 0xa2, 0x94, 0x05,
0x03, 0xe8, 0x7e, 0x13, 0xc5, 0x01, 0x43, 0x32, 0x22, 0x31, 0xe1, 0xad, 0x60, 0x2f, 0xa0, 0xa7, 0xb2, 0x01, 0xf4, 0xbe, 0x89, 0xe2, 0x80, 0x21, 0x19, 0x91, 0x98, 0xf0, 0x56, 0xb0, 0x17, 0xd0,
0xf3, 0x12, 0x95, 0x16, 0xa5, 0x0c, 0x9d, 0x11, 0x89, 0x5d, 0x7e, 0x33, 0x22, 0x04, 0x58, 0xe5, 0xd7, 0x79, 0x89, 0x4a, 0x8b, 0x52, 0x86, 0xce, 0x88, 0xc4, 0x2e, 0xbf, 0x1a, 0xd1, 0x57, 0x80,
0x25, 0x2e, 0xb1, 0xc9, 0x51, 0xb1, 0xd7, 0x40, 0x0b, 0xb1, 0xc6, 0x42, 0x85, 0x64, 0xe4, 0xc6, 0x55, 0x5e, 0xe2, 0x12, 0x9b, 0x1c, 0x15, 0x7b, 0x0d, 0xb4, 0x10, 0x6b, 0x2c, 0x54, 0x48, 0x46,
0xfd, 0xe9, 0xe3, 0xe4, 0x56, 0x9f, 0xcc, 0xcd, 0x86, 0x5f, 0x02, 0xec, 0x0d, 0xf8, 0xca, 0xd6, 0x6e, 0x7c, 0x3f, 0x7d, 0x9c, 0x5c, 0xe3, 0x93, 0xb9, 0xd9, 0xf0, 0x0e, 0x60, 0x53, 0xf0, 0x95,
0xaa, 0xd0, 0xb1, 0x59, 0x76, 0x9f, 0x6d, 0x89, 0xf8, 0x35, 0x12, 0x4d, 0xa0, 0x6b, 0xcf, 0x19, 0x8d, 0x55, 0xa1, 0x63, 0x59, 0x76, 0xcb, 0xb6, 0x8d, 0x66, 0xde, 0xe9, 0xe7, 0xab, 0x3b, 0x7e,
0x03, 0xaf, 0x12, 0x65, 0x8b, 0xd8, 0xe3, 0x76, 0xbe, 0x71, 0x3b, 0xd6, 0x6c, 0x45, 0xf4, 0x0e, 0x01, 0xa3, 0x09, 0xf4, 0xec, 0x43, 0x18, 0x03, 0xaf, 0x12, 0x65, 0x5b, 0xb4, 0xcf, 0xed, 0x7c,
0xe8, 0xbc, 0xad, 0x1a, 0xff, 0x97, 0x6a, 0xe6, 0x9d, 0x7e, 0xbe, 0xea, 0x5c, 0xd9, 0xa2, 0xef, 0x6d, 0xef, 0x58, 0xb3, 0x15, 0xd1, 0x5b, 0xa0, 0xf3, 0x36, 0x70, 0xfc, 0xdf, 0x6e, 0x5d, 0x5c,
0x04, 0x1e, 0x5a, 0x3f, 0x13, 0x7a, 0xb3, 0xc7, 0x86, 0x4d, 0xc0, 0x33, 0x3f, 0xd5, 0xb6, 0x3e, 0x87, 0x45, 0xdf, 0x09, 0x3c, 0xb4, 0x7e, 0x26, 0xf4, 0x66, 0x8f, 0x0d, 0x9b, 0x80, 0x67, 0x3e,
0x9a, 0xbe, 0xfc, 0xeb, 0xfe, 0x92, 0x4b, 0x56, 0x47, 0x89, 0xdc, 0x46, 0xff, 0x80, 0x3a, 0xff, 0xad, 0x4d, 0x7d, 0x34, 0x7d, 0xf9, 0xd7, 0x7d, 0xc7, 0x25, 0xab, 0xa3, 0x44, 0x6e, 0xd1, 0x3f,
0x02, 0x75, 0xef, 0x41, 0x63, 0xf0, 0xcc, 0x1d, 0xa3, 0xe0, 0xa4, 0x8b, 0xa0, 0xc3, 0x7c, 0x70, 0x45, 0x9d, 0x7f, 0x15, 0x75, 0x6f, 0x8b, 0xc6, 0xe0, 0x99, 0x3b, 0x46, 0xc1, 0x49, 0x17, 0xc1,
0x3f, 0xa6, 0x8b, 0x80, 0x18, 0x83, 0xa7, 0x81, 0x63, 0x0d, 0x9e, 0x06, 0x6e, 0xf4, 0x15, 0x7a, 0x1d, 0xf3, 0xc1, 0xfd, 0x90, 0x2e, 0x02, 0x62, 0x0c, 0x9e, 0x06, 0x8e, 0x35, 0x78, 0x1a, 0xb8,
0x1c, 0xc5, 0xf6, 0x43, 0x5e, 0x69, 0xc5, 0x9e, 0x82, 0xaf, 0x34, 0xca, 0xcf, 0xa5, 0xb2, 0x58, 0xd1, 0x17, 0xe8, 0x73, 0x14, 0xdb, 0xf7, 0x79, 0xa5, 0x15, 0x7b, 0x0a, 0xbe, 0xd2, 0x28, 0x3f,
0x2e, 0xa7, 0x46, 0x66, 0xca, 0x34, 0x7f, 0x39, 0x54, 0x9b, 0x6b, 0xb3, 0x99, 0xd9, 0x33, 0x78, 0x95, 0xca, 0xd6, 0x72, 0x39, 0x35, 0x32, 0x53, 0x26, 0xf9, 0xf3, 0xa1, 0xda, 0x5c, 0x92, 0xcd,
0xa0, 0xb4, 0x68, 0xb4, 0x49, 0xbb, 0x36, 0xed, 0x5b, 0x9d, 0x29, 0xf6, 0x04, 0x28, 0x56, 0x5b, 0xcc, 0x9e, 0xc1, 0x03, 0xa5, 0x45, 0xa3, 0x0d, 0xed, 0x5a, 0xda, 0xb7, 0x3a, 0x53, 0xec, 0x09,
0xb3, 0xf0, 0xec, 0xa2, 0x8b, 0xd5, 0x36, 0x53, 0xb3, 0xc1, 0xe9, 0x3c, 0x24, 0x3f, 0xce, 0x43, 0x50, 0xac, 0xb6, 0x66, 0xe1, 0xd9, 0x45, 0x0f, 0xab, 0x6d, 0xa6, 0x66, 0x83, 0xd3, 0x79, 0x48,
0xf2, 0xeb, 0x3c, 0x24, 0x9f, 0xa8, 0xf9, 0x62, 0xb9, 0x5e, 0x53, 0xfb, 0x66, 0xde, 0xfe, 0x0e, 0x7e, 0x9c, 0x87, 0xe4, 0xd7, 0x79, 0x48, 0x3e, 0x52, 0xf3, 0xc6, 0x72, 0xbd, 0xa6, 0xf6, 0xcf,
0x00, 0x00, 0xff, 0xff, 0x32, 0x7f, 0x5e, 0x49, 0x64, 0x02, 0x00, 0x00, 0x79, 0xf3, 0x3b, 0x00, 0x00, 0xff, 0xff, 0x44, 0x38, 0x1e, 0x5b, 0x6a, 0x02, 0x00, 0x00,
} }

2
prompb/types.proto
View file

@ -25,7 +25,7 @@ message Sample {
message TimeSeries { message TimeSeries {
repeated Label labels = 1; repeated Label labels = 1;
repeated Sample samples = 2; repeated Sample samples = 2 [(gogoproto.nullable) = false];
} }
message Label { message Label {

8
storage/remote/codec.go
View file

@ -89,7 +89,7 @@ func ToWriteRequest(samples []*model.Sample) *prompb.WriteRequest {
for _, s := range samples { for _, s := range samples {
ts := prompb.TimeSeries{ ts := prompb.TimeSeries{
Labels: MetricToLabelProtos(s.Metric), Labels: MetricToLabelProtos(s.Metric),
Samples: []*prompb.Sample{ Samples: []prompb.Sample{
{ {
Value: float64(s.Value), Value: float64(s.Value),
Timestamp: int64(s.Timestamp), Timestamp: int64(s.Timestamp),
@ -153,7 +153,7 @@ func ToQueryResult(ss storage.SeriesSet, sampleLimit int) (*prompb.QueryResult,
for ss.Next() { for ss.Next() {
series := ss.At() series := ss.At()
iter := series.Iterator() iter := series.Iterator()
samples := []*prompb.Sample{} samples := []prompb.Sample{}
for iter.Next() { for iter.Next() {
numSamples++ numSamples++
@ -164,7 +164,7 @@ func ToQueryResult(ss storage.SeriesSet, sampleLimit int) (*prompb.QueryResult,
} }
} }
ts, val := iter.At() ts, val := iter.At()
samples = append(samples, &prompb.Sample{ samples = append(samples, prompb.Sample{
Timestamp: ts, Timestamp: ts,
Value: val, Value: val,
}) })
@ -249,7 +249,7 @@ func (c *concreteSeriesSet) Err() error {
// concreteSeries implements storage.Series. // concreteSeries implements storage.Series.
type concreteSeries struct { type concreteSeries struct {
labels labels.Labels labels labels.Labels
samples []*prompb.Sample samples []prompb.Sample
} }
func (c *concreteSeries) Labels() labels.Labels { func (c *concreteSeries) Labels() labels.Labels {

4
storage/remote/codec_test.go
View file

@ -101,11 +101,11 @@ func TestValidateLabelsAndMetricName(t *testing.T) {
func TestConcreteSeriesSet(t *testing.T) { func TestConcreteSeriesSet(t *testing.T) {
series1 := &concreteSeries{ series1 := &concreteSeries{
labels: labels.FromStrings("foo", "bar"), labels: labels.FromStrings("foo", "bar"),
samples: []*prompb.Sample{&prompb.Sample{Value: 1, Timestamp: 2}}, samples: []prompb.Sample{prompb.Sample{Value: 1, Timestamp: 2}},
} }
series2 := &concreteSeries{ series2 := &concreteSeries{
labels: labels.FromStrings("foo", "baz"), labels: labels.FromStrings("foo", "baz"),
samples: []*prompb.Sample{&prompb.Sample{Value: 3, Timestamp: 4}}, samples: []prompb.Sample{prompb.Sample{Value: 3, Timestamp: 4}},
} }
c := &concreteSeriesSet{ c := &concreteSeriesSet{
series: []storage.Series{series1, series2}, series: []storage.Series{series1, series2},

14
storage/remote/queue_manager_test.go
View file

@ -30,16 +30,16 @@ import (
const defaultFlushDeadline = 1 * time.Minute const defaultFlushDeadline = 1 * time.Minute
type TestStorageClient struct { type TestStorageClient struct {
receivedSamples map[string][]*prompb.Sample receivedSamples map[string][]prompb.Sample
expectedSamples map[string][]*prompb.Sample expectedSamples map[string][]prompb.Sample
wg sync.WaitGroup wg sync.WaitGroup
mtx sync.Mutex mtx sync.Mutex
} }
func NewTestStorageClient() *TestStorageClient { func NewTestStorageClient() *TestStorageClient {
return &TestStorageClient{ return &TestStorageClient{
receivedSamples: map[string][]*prompb.Sample{}, receivedSamples: map[string][]prompb.Sample{},
expectedSamples: map[string][]*prompb.Sample{}, expectedSamples: map[string][]prompb.Sample{},
} }
} }
@ -47,12 +47,12 @@ func (c *TestStorageClient) expectSamples(ss model.Samples) {
c.mtx.Lock() c.mtx.Lock()
defer c.mtx.Unlock() defer c.mtx.Unlock()
c.expectedSamples = map[string][]*prompb.Sample{} c.expectedSamples = map[string][]prompb.Sample{}
c.receivedSamples = map[string][]*prompb.Sample{} c.receivedSamples = map[string][]prompb.Sample{}
for _, s := range ss { for _, s := range ss {
ts := labelProtosToLabels(MetricToLabelProtos(s.Metric)).String() ts := labelProtosToLabels(MetricToLabelProtos(s.Metric)).String()
c.expectedSamples[ts] = append(c.expectedSamples[ts], &prompb.Sample{ c.expectedSamples[ts] = append(c.expectedSamples[ts], prompb.Sample{
Timestamp: int64(s.Timestamp), Timestamp: int64(s.Timestamp),
Value: float64(s.Value), Value: float64(s.Value),
}) })

4
storage/remote/read_test.go
View file

@ -122,12 +122,12 @@ func TestSeriesSetFilter(t *testing.T) {
toRemove: model.LabelSet{"foo": "bar"}, toRemove: model.LabelSet{"foo": "bar"},
in: &prompb.QueryResult{ in: &prompb.QueryResult{
Timeseries: []*prompb.TimeSeries{ Timeseries: []*prompb.TimeSeries{
{Labels: labelsToLabelsProto(labels.FromStrings("foo", "bar", "a", "b")), Samples: []*prompb.Sample{}}, {Labels: labelsToLabelsProto(labels.FromStrings("foo", "bar", "a", "b")), Samples: []prompb.Sample{}},
}, },
}, },
expected: &prompb.QueryResult{ expected: &prompb.QueryResult{
Timeseries: []*prompb.TimeSeries{ Timeseries: []*prompb.TimeSeries{
{Labels: labelsToLabelsProto(labels.FromStrings("a", "b")), Samples: []*prompb.Sample{}}, {Labels: labelsToLabelsProto(labels.FromStrings("a", "b")), Samples: []prompb.Sample{}},
}, },
}, },
}, },

5
web/api/v1/api_test.go
View file

@ -19,7 +19,6 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"github.com/go-kit/kit/log"
"io/ioutil" "io/ioutil"
"math" "math"
"net/http" "net/http"
@ -30,6 +29,8 @@ import (
"testing" "testing"
"time" "time"
"github.com/go-kit/kit/log"
"github.com/gogo/protobuf/proto" "github.com/gogo/protobuf/proto"
"github.com/golang/snappy" "github.com/golang/snappy"
config_util "github.com/prometheus/common/config" config_util "github.com/prometheus/common/config"
@ -897,7 +898,7 @@ func TestReadEndpoint(t *testing.T) {
{Name: "d", Value: "e"}, {Name: "d", Value: "e"},
{Name: "foo", Value: "bar"}, {Name: "foo", Value: "bar"},
}, },
Samples: []*prompb.Sample{{Value: 1, Timestamp: 0}}, Samples: []prompb.Sample{{Value: 1, Timestamp: 0}},
}, },
}, },
} }