From 8b0cda8bb3600ebaca3a268dad99ad2bd93572bb Mon Sep 17 00:00:00 2001 From: Callum Styan Date: Mon, 13 Nov 2023 21:53:44 -0800 Subject: [PATCH] Add bytes slice (instead of slice of 32bit vars) format for testing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Nicolás Pazos Signed-off-by: Callum Styan Signed-off-by: Nicolás Pazos --- prompb/remote.pb.go | 340 ++++++++++++++--- prompb/remote.proto | 12 + prompb/types.pb.go | 541 +++++++++++++++++++++++---- prompb/types.proto | 14 + scripts/remotewrite11-bench/run.sh | 9 +- storage/remote/codec.go | 89 ++++- storage/remote/queue_manager.go | 144 ++++++- storage/remote/queue_manager_test.go | 80 ++-- storage/remote/write_handler.go | 44 +++ 9 files changed, 1072 insertions(+), 201 deletions(-) diff --git a/prompb/remote.pb.go b/prompb/remote.pb.go index 8519468b9..b5d6b6984 100644 --- a/prompb/remote.pb.go +++ b/prompb/remote.pb.go @@ -60,7 +60,7 @@ func (x ReadRequest_ResponseType) String() string { } func (ReadRequest_ResponseType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_eefc82927d57d89b, []int{5, 0} + return fileDescriptor_eefc82927d57d89b, []int{6, 0} } type WriteRequest struct { @@ -232,6 +232,63 @@ func (m *MinimizedWriteRequest) GetSymbols() string { return "" } +type MinimizedWriteRequestBytes struct { + Timeseries []MinimizedTimeSeriesBytes `protobuf:"bytes,1,rep,name=timeseries,proto3" json:"timeseries"` + // The symbols table. All symbols are concatenated strings. To read the symbols table, it's required + // to know the offset:length range of the actual symbol to read from this string. + Symbols string `protobuf:"bytes,4,opt,name=symbols,proto3" json:"symbols,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *MinimizedWriteRequestBytes) Reset() { *m = MinimizedWriteRequestBytes{} } +func (m *MinimizedWriteRequestBytes) String() string { return proto.CompactTextString(m) } +func (*MinimizedWriteRequestBytes) ProtoMessage() {} +func (*MinimizedWriteRequestBytes) Descriptor() ([]byte, []int) { + return fileDescriptor_eefc82927d57d89b, []int{3} +} +func (m *MinimizedWriteRequestBytes) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *MinimizedWriteRequestBytes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_MinimizedWriteRequestBytes.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *MinimizedWriteRequestBytes) XXX_Merge(src proto.Message) { + xxx_messageInfo_MinimizedWriteRequestBytes.Merge(m, src) +} +func (m *MinimizedWriteRequestBytes) XXX_Size() int { + return m.Size() +} +func (m *MinimizedWriteRequestBytes) XXX_DiscardUnknown() { + xxx_messageInfo_MinimizedWriteRequestBytes.DiscardUnknown(m) +} + +var xxx_messageInfo_MinimizedWriteRequestBytes proto.InternalMessageInfo + +func (m *MinimizedWriteRequestBytes) GetTimeseries() []MinimizedTimeSeriesBytes { + if m != nil { + return m.Timeseries + } + return nil +} + +func (m *MinimizedWriteRequestBytes) GetSymbols() string { + if m != nil { + return m.Symbols + } + return "" +} + type MinimizedWriteRequestFixed32 struct { Timeseries []MinimizedTimeSeriesFixed32 `protobuf:"bytes,1,rep,name=timeseries,proto3" json:"timeseries"` // The symbols table. All symbols are concatenated strings. To read the symbols table, it's required @@ -246,7 +303,7 @@ func (m *MinimizedWriteRequestFixed32) Reset() { *m = MinimizedWriteRequ func (m *MinimizedWriteRequestFixed32) String() string { return proto.CompactTextString(m) } func (*MinimizedWriteRequestFixed32) ProtoMessage() {} func (*MinimizedWriteRequestFixed32) Descriptor() ([]byte, []int) { - return fileDescriptor_eefc82927d57d89b, []int{3} + return fileDescriptor_eefc82927d57d89b, []int{4} } func (m *MinimizedWriteRequestFixed32) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -303,7 +360,7 @@ func (m *MinimizedWriteRequestFixed64) Reset() { *m = MinimizedWriteRequ func (m *MinimizedWriteRequestFixed64) String() string { return proto.CompactTextString(m) } func (*MinimizedWriteRequestFixed64) ProtoMessage() {} func (*MinimizedWriteRequestFixed64) Descriptor() ([]byte, []int) { - return fileDescriptor_eefc82927d57d89b, []int{4} + return fileDescriptor_eefc82927d57d89b, []int{5} } func (m *MinimizedWriteRequestFixed64) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -364,7 +421,7 @@ func (m *ReadRequest) Reset() { *m = ReadRequest{} } func (m *ReadRequest) String() string { return proto.CompactTextString(m) } func (*ReadRequest) ProtoMessage() {} func (*ReadRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_eefc82927d57d89b, []int{5} + return fileDescriptor_eefc82927d57d89b, []int{6} } func (m *ReadRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -420,7 +477,7 @@ func (m *ReadResponse) Reset() { *m = ReadResponse{} } func (m *ReadResponse) String() string { return proto.CompactTextString(m) } func (*ReadResponse) ProtoMessage() {} func (*ReadResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_eefc82927d57d89b, []int{6} + return fileDescriptor_eefc82927d57d89b, []int{7} } func (m *ReadResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -470,7 +527,7 @@ func (m *Query) Reset() { *m = Query{} } func (m *Query) String() string { return proto.CompactTextString(m) } func (*Query) ProtoMessage() {} func (*Query) Descriptor() ([]byte, []int) { - return fileDescriptor_eefc82927d57d89b, []int{7} + return fileDescriptor_eefc82927d57d89b, []int{8} } func (m *Query) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -539,7 +596,7 @@ func (m *QueryResult) Reset() { *m = QueryResult{} } func (m *QueryResult) String() string { return proto.CompactTextString(m) } func (*QueryResult) ProtoMessage() {} func (*QueryResult) Descriptor() ([]byte, []int) { - return fileDescriptor_eefc82927d57d89b, []int{8} + return fileDescriptor_eefc82927d57d89b, []int{9} } func (m *QueryResult) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -592,7 +649,7 @@ func (m *ChunkedReadResponse) Reset() { *m = ChunkedReadResponse{} } func (m *ChunkedReadResponse) String() string { return proto.CompactTextString(m) } func (*ChunkedReadResponse) ProtoMessage() {} func (*ChunkedReadResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_eefc82927d57d89b, []int{9} + return fileDescriptor_eefc82927d57d89b, []int{10} } func (m *ChunkedReadResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -640,6 +697,7 @@ func init() { proto.RegisterType((*WriteRequest)(nil), "prometheus.WriteRequest") proto.RegisterType((*MinimizedWriteRequestPacking)(nil), "prometheus.MinimizedWriteRequestPacking") proto.RegisterType((*MinimizedWriteRequest)(nil), "prometheus.MinimizedWriteRequest") + proto.RegisterType((*MinimizedWriteRequestBytes)(nil), "prometheus.MinimizedWriteRequestBytes") proto.RegisterType((*MinimizedWriteRequestFixed32)(nil), "prometheus.MinimizedWriteRequestFixed32") proto.RegisterType((*MinimizedWriteRequestFixed64)(nil), "prometheus.MinimizedWriteRequestFixed64") proto.RegisterType((*ReadRequest)(nil), "prometheus.ReadRequest") @@ -652,45 +710,46 @@ func init() { func init() { proto.RegisterFile("remote.proto", fileDescriptor_eefc82927d57d89b) } var fileDescriptor_eefc82927d57d89b = []byte{ - // 601 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x94, 0xdf, 0x4e, 0x13, 0x41, - 0x14, 0xc6, 0x19, 0x5a, 0x68, 0x3d, 0x8b, 0x64, 0x1d, 0x40, 0x56, 0x62, 0x80, 0x6c, 0x8c, 0x69, - 0x82, 0xa9, 0xb1, 0x34, 0x5c, 0x71, 0x21, 0x60, 0x0d, 0x22, 0xab, 0x38, 0xad, 0xd1, 0x18, 0x93, - 0xcd, 0x76, 0xf7, 0x84, 0x4e, 0x60, 0xff, 0xb0, 0x33, 0x9b, 0x50, 0xaf, 0x7d, 0x00, 0xe3, 0x23, - 0x79, 0xc5, 0x95, 0xf1, 0x09, 0x8c, 0xe1, 0x49, 0xcc, 0xfe, 0x83, 0xa9, 0xd6, 0x68, 0x9a, 0x78, - 0xb7, 0x7b, 0xce, 0xf7, 0x7d, 0xfb, 0x9b, 0x33, 0x3b, 0x03, 0x73, 0x31, 0xfa, 0xa1, 0xc4, 0x66, - 0x14, 0x87, 0x32, 0xa4, 0x10, 0xc5, 0xa1, 0x8f, 0x72, 0x80, 0x89, 0x58, 0xd1, 0xe4, 0x30, 0x42, - 0x91, 0x37, 0x56, 0x16, 0x8f, 0xc3, 0xe3, 0x30, 0x7b, 0x7c, 0x98, 0x3e, 0xe5, 0x55, 0xf3, 0x13, - 0x81, 0xb9, 0x37, 0x31, 0x97, 0xc8, 0xf0, 0x2c, 0x41, 0x21, 0xe9, 0x36, 0x80, 0xe4, 0x3e, 0x0a, - 0x8c, 0x39, 0x0a, 0x83, 0xac, 0x57, 0x1a, 0x5a, 0xeb, 0x76, 0xf3, 0x3a, 0xb4, 0xd9, 0xe3, 0x3e, - 0x76, 0xb3, 0xee, 0x6e, 0xf5, 0xe2, 0xfb, 0xda, 0x14, 0x53, 0xf4, 0x74, 0x1b, 0xea, 0x3e, 0x4a, - 0xc7, 0x73, 0xa4, 0x63, 0x54, 0x32, 0xef, 0x8a, 0xea, 0xb5, 0x50, 0xc6, 0xdc, 0xb5, 0x0a, 0x45, - 0xe1, 0xbf, 0x72, 0x1c, 0x54, 0xeb, 0xd3, 0x7a, 0xc5, 0xfc, 0x4c, 0xe0, 0xae, 0xc5, 0x03, 0xee, - 0xf3, 0x0f, 0xe8, 0xa9, 0x6c, 0x47, 0x8e, 0x7b, 0xc2, 0x83, 0x63, 0x7a, 0x38, 0x06, 0xf1, 0xfe, - 0xc8, 0x67, 0x4a, 0xf7, 0x35, 0x6b, 0xe1, 0x1d, 0x83, 0x6c, 0x40, 0x4d, 0x0c, 0xfd, 0x7e, 0x78, - 0x2a, 0x8c, 0xea, 0x3a, 0x69, 0xdc, 0x60, 0xe5, 0x6b, 0x8e, 0x73, 0x50, 0xad, 0x57, 0xf4, 0xaa, - 0xf9, 0x91, 0xc0, 0xd2, 0x58, 0x28, 0xda, 0x19, 0x43, 0xb3, 0xf6, 0x17, 0x9a, 0x89, 0x31, 0xfe, - 0x38, 0x9b, 0xa7, 0xfc, 0x1c, 0xbd, 0xcd, 0xd6, 0x44, 0xb3, 0x29, 0xbc, 0xff, 0x09, 0x6a, 0xab, - 0x3d, 0x39, 0xd4, 0x56, 0x7b, 0x62, 0xa8, 0xaf, 0x04, 0x34, 0x86, 0x8e, 0x57, 0x6e, 0xd3, 0x06, - 0xd4, 0xce, 0x12, 0x15, 0xe0, 0x96, 0x0a, 0xf0, 0x2a, 0xc1, 0x78, 0xc8, 0x4a, 0x05, 0x7d, 0x0f, - 0xcb, 0x8e, 0xeb, 0x62, 0x24, 0xd1, 0xb3, 0x63, 0x14, 0x51, 0x18, 0x08, 0xb4, 0xb3, 0xc3, 0x64, - 0x4c, 0xaf, 0x57, 0x1a, 0xf3, 0xad, 0x7b, 0xaa, 0x59, 0xf9, 0x4c, 0x93, 0x15, 0xea, 0xde, 0x30, - 0x42, 0xb6, 0x54, 0x86, 0xa8, 0x55, 0x61, 0xb6, 0x61, 0x4e, 0x2d, 0x50, 0x0d, 0x6a, 0xdd, 0x1d, - 0xeb, 0xe8, 0xb0, 0xd3, 0xd5, 0xa7, 0xe8, 0x32, 0x2c, 0x74, 0x7b, 0xac, 0xb3, 0x63, 0x75, 0x9e, - 0xd8, 0x6f, 0x5f, 0x32, 0x7b, 0x6f, 0xff, 0xf5, 0x8b, 0xe7, 0x5d, 0x9d, 0x98, 0x3b, 0xa9, 0xcb, - 0xb9, 0x8a, 0xa2, 0x8f, 0xa0, 0x16, 0xa3, 0x48, 0x4e, 0x65, 0xb9, 0xa0, 0xe5, 0xdf, 0x17, 0x94, - 0xf5, 0x59, 0xa9, 0x33, 0xbf, 0x10, 0x98, 0xc9, 0x1a, 0xf4, 0x01, 0x50, 0x21, 0x9d, 0x58, 0xda, - 0xd9, 0x5c, 0xa5, 0xe3, 0x47, 0xb6, 0x9f, 0xe6, 0x90, 0x46, 0x85, 0xe9, 0x59, 0xa7, 0x57, 0x36, - 0x2c, 0x41, 0x1b, 0xa0, 0x63, 0xe0, 0x8d, 0x6a, 0xa7, 0x33, 0xed, 0x3c, 0x06, 0x9e, 0xaa, 0x6c, - 0x43, 0xdd, 0x77, 0xa4, 0x3b, 0xc0, 0x58, 0x14, 0xe7, 0xdf, 0x50, 0xa9, 0x0e, 0x9d, 0x3e, 0x9e, - 0x5a, 0xb9, 0x80, 0x5d, 0x29, 0xe9, 0x06, 0xcc, 0x0c, 0x78, 0x20, 0xf3, 0xfd, 0xd4, 0x5a, 0x4b, - 0xbf, 0x0e, 0x77, 0x3f, 0x6d, 0xb2, 0x5c, 0x63, 0x76, 0x40, 0x53, 0x16, 0x47, 0xb7, 0xfe, 0xfd, - 0xbe, 0x52, 0xff, 0x22, 0xf3, 0x1c, 0x16, 0xf6, 0x06, 0x49, 0x70, 0x92, 0x6e, 0x8e, 0x32, 0xd5, - 0xc7, 0x30, 0xef, 0xe6, 0x65, 0x7b, 0x24, 0xf2, 0x8e, 0x1a, 0x59, 0x18, 0x8b, 0xd4, 0x9b, 0xae, - 0xfa, 0x4a, 0xd7, 0x40, 0x4b, 0x7f, 0xa3, 0xa1, 0xcd, 0x03, 0x0f, 0xcf, 0x8b, 0x39, 0x41, 0x56, - 0x7a, 0x96, 0x56, 0x76, 0x17, 0x2f, 0x2e, 0x57, 0xc9, 0xb7, 0xcb, 0x55, 0xf2, 0xe3, 0x72, 0x95, - 0xbc, 0x9b, 0x4d, 0x73, 0xa3, 0x7e, 0x7f, 0x36, 0xbb, 0x8f, 0x37, 0x7f, 0x06, 0x00, 0x00, 0xff, - 0xff, 0x13, 0xe9, 0xfd, 0x50, 0xce, 0x05, 0x00, 0x00, + // 621 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x95, 0xdf, 0x4e, 0x13, 0x41, + 0x14, 0xc6, 0x19, 0x5a, 0x68, 0x3d, 0x8b, 0xa4, 0x0e, 0x20, 0xb5, 0x31, 0x40, 0x36, 0xc6, 0x34, + 0xc1, 0xd4, 0x08, 0x0d, 0x57, 0x5c, 0x08, 0x58, 0x83, 0xc8, 0x2a, 0x4e, 0x6b, 0x34, 0xc6, 0x64, + 0xb3, 0xdd, 0x3d, 0xa1, 0x13, 0xd8, 0x3f, 0xec, 0x4c, 0x13, 0xd6, 0x6b, 0xaf, 0x8d, 0xf1, 0x91, + 0xbc, 0xe2, 0xca, 0xf8, 0x04, 0xc6, 0xf0, 0x24, 0x66, 0xff, 0x95, 0xa9, 0xae, 0xc1, 0x34, 0xf1, + 0x6e, 0xf7, 0x9c, 0xef, 0xfb, 0xf6, 0x37, 0x73, 0xa6, 0x53, 0x98, 0x0b, 0xd1, 0xf5, 0x25, 0xb6, + 0x82, 0xd0, 0x97, 0x3e, 0x85, 0x20, 0xf4, 0x5d, 0x94, 0x03, 0x1c, 0x8a, 0x86, 0x26, 0xa3, 0x00, + 0x45, 0xda, 0x68, 0x2c, 0x1e, 0xfb, 0xc7, 0x7e, 0xf2, 0xf8, 0x30, 0x7e, 0x4a, 0xab, 0xfa, 0x67, + 0x02, 0x73, 0x6f, 0x42, 0x2e, 0x91, 0xe1, 0xd9, 0x10, 0x85, 0xa4, 0xdb, 0x00, 0x92, 0xbb, 0x28, + 0x30, 0xe4, 0x28, 0xea, 0x64, 0xad, 0xd4, 0xd4, 0x36, 0x6e, 0xb7, 0xae, 0x42, 0x5b, 0x3d, 0xee, + 0x62, 0x37, 0xe9, 0xee, 0x96, 0x2f, 0x7e, 0xac, 0x4e, 0x31, 0x45, 0x4f, 0xb7, 0xa1, 0xea, 0xa2, + 0xb4, 0x1c, 0x4b, 0x5a, 0xf5, 0x52, 0xe2, 0x6d, 0xa8, 0x5e, 0x03, 0x65, 0xc8, 0x6d, 0x23, 0x53, + 0x64, 0xfe, 0x91, 0xe3, 0xa0, 0x5c, 0x9d, 0xae, 0x95, 0xf4, 0x2f, 0x04, 0xee, 0x1a, 0xdc, 0xe3, + 0x2e, 0xff, 0x80, 0x8e, 0xca, 0x76, 0x64, 0xd9, 0x27, 0xdc, 0x3b, 0xa6, 0x87, 0x05, 0x88, 0xf7, + 0xc7, 0x3e, 0x93, 0xbb, 0xaf, 0x58, 0x33, 0x6f, 0x01, 0x72, 0x1d, 0x2a, 0x22, 0x72, 0xfb, 0xfe, + 0xa9, 0xa8, 0x97, 0xd7, 0x48, 0xf3, 0x06, 0xcb, 0x5f, 0x53, 0x9c, 0x83, 0x72, 0xb5, 0x54, 0x2b, + 0xeb, 0x1f, 0x09, 0x2c, 0x15, 0x42, 0xd1, 0x4e, 0x01, 0xcd, 0xea, 0x35, 0x34, 0x13, 0x63, 0x7c, + 0x22, 0xd0, 0x28, 0xc4, 0xd8, 0x8d, 0x24, 0x0a, 0x7a, 0x50, 0xc0, 0x72, 0xef, 0x3a, 0x96, 0xd8, + 0x39, 0x31, 0xd0, 0x5f, 0x87, 0xf5, 0x94, 0x9f, 0xa3, 0xb3, 0xb9, 0x31, 0xd1, 0xb0, 0x32, 0xef, + 0x7f, 0x82, 0xda, 0x6a, 0x4f, 0x0e, 0xb5, 0xd5, 0x9e, 0x18, 0xea, 0x1b, 0x01, 0x8d, 0xa1, 0xe5, + 0xe4, 0xe7, 0x66, 0x1d, 0x2a, 0x67, 0x43, 0x15, 0xe0, 0x96, 0x0a, 0xf0, 0x6a, 0x88, 0x61, 0xc4, + 0x72, 0x05, 0x7d, 0x0f, 0xcb, 0x96, 0x6d, 0x63, 0x20, 0xd1, 0x31, 0x43, 0x14, 0x81, 0xef, 0x09, + 0x34, 0x93, 0x5f, 0x77, 0x7d, 0x7a, 0xad, 0xd4, 0x9c, 0x1f, 0x9f, 0xb2, 0xf2, 0x99, 0x16, 0xcb, + 0xd4, 0xbd, 0x28, 0x40, 0xb6, 0x94, 0x87, 0xa8, 0x55, 0xa1, 0xb7, 0x61, 0x4e, 0x2d, 0x50, 0x0d, + 0x2a, 0xdd, 0x1d, 0xe3, 0xe8, 0xb0, 0xd3, 0xad, 0x4d, 0xd1, 0x65, 0x58, 0xe8, 0xf6, 0x58, 0x67, + 0xc7, 0xe8, 0x3c, 0x31, 0xdf, 0xbe, 0x64, 0xe6, 0xde, 0xfe, 0xeb, 0x17, 0xcf, 0xbb, 0x35, 0xa2, + 0xef, 0xc4, 0x2e, 0x6b, 0x14, 0x45, 0x1f, 0x41, 0x25, 0x44, 0x31, 0x3c, 0x95, 0xf9, 0x82, 0x96, + 0xff, 0x5c, 0x50, 0xd2, 0x67, 0xb9, 0x4e, 0xff, 0x4a, 0x60, 0x26, 0x69, 0xd0, 0x07, 0x40, 0x85, + 0xb4, 0x42, 0x69, 0x26, 0xfb, 0x2a, 0x2d, 0x37, 0x30, 0xdd, 0x38, 0x87, 0x34, 0x4b, 0xac, 0x96, + 0x74, 0x7a, 0x79, 0xc3, 0x10, 0xb4, 0x09, 0x35, 0xf4, 0x9c, 0x71, 0xed, 0x74, 0xa2, 0x9d, 0x47, + 0xcf, 0x51, 0x95, 0x6d, 0xa8, 0xba, 0x96, 0xb4, 0x07, 0x18, 0x8a, 0xec, 0x42, 0xaa, 0xab, 0x54, + 0x87, 0x56, 0x1f, 0x4f, 0x8d, 0x54, 0xc0, 0x46, 0x4a, 0xba, 0x0e, 0x33, 0x03, 0xee, 0xc9, 0x74, + 0x9e, 0xda, 0xc6, 0xd2, 0xef, 0x9b, 0xbb, 0x1f, 0x37, 0x59, 0xaa, 0xd1, 0x3b, 0xa0, 0x29, 0x8b, + 0xa3, 0x5b, 0xff, 0x7e, 0x81, 0xaa, 0xa7, 0x48, 0x3f, 0x87, 0x85, 0xbd, 0xc1, 0xd0, 0x3b, 0x89, + 0x87, 0xa3, 0xec, 0xea, 0x63, 0x98, 0xb7, 0xd3, 0xb2, 0x39, 0x16, 0x79, 0x47, 0x8d, 0xcc, 0x8c, + 0x59, 0xea, 0x4d, 0x5b, 0x7d, 0xa5, 0xab, 0xa0, 0xc5, 0xc7, 0x28, 0x32, 0xb9, 0xe7, 0xe0, 0x79, + 0xb6, 0x4f, 0x90, 0x94, 0x9e, 0xc5, 0x95, 0xdd, 0xc5, 0x8b, 0xcb, 0x15, 0xf2, 0xfd, 0x72, 0x85, + 0xfc, 0xbc, 0x5c, 0x21, 0xef, 0x66, 0xe3, 0xdc, 0xa0, 0xdf, 0x9f, 0x4d, 0xfe, 0x20, 0x36, 0x7f, + 0x05, 0x00, 0x00, 0xff, 0xff, 0xf7, 0x0a, 0xf4, 0xd7, 0x5f, 0x06, 0x00, 0x00, } func (m *WriteRequest) Marshal() (dAtA []byte, err error) { @@ -844,6 +903,54 @@ func (m *MinimizedWriteRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *MinimizedWriteRequestBytes) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *MinimizedWriteRequestBytes) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *MinimizedWriteRequestBytes) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Symbols) > 0 { + i -= len(m.Symbols) + copy(dAtA[i:], m.Symbols) + i = encodeVarintRemote(dAtA, i, uint64(len(m.Symbols))) + i-- + dAtA[i] = 0x22 + } + if len(m.Timeseries) > 0 { + for iNdEx := len(m.Timeseries) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Timeseries[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintRemote(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + func (m *MinimizedWriteRequestFixed32) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -1269,6 +1376,28 @@ func (m *MinimizedWriteRequest) Size() (n int) { return n } +func (m *MinimizedWriteRequestBytes) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Timeseries) > 0 { + for _, e := range m.Timeseries { + l = e.Size() + n += 1 + l + sovRemote(uint64(l)) + } + } + l = len(m.Symbols) + if l > 0 { + n += 1 + l + sovRemote(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func (m *MinimizedWriteRequestFixed32) Size() (n int) { if m == nil { return 0 @@ -1782,6 +1911,123 @@ func (m *MinimizedWriteRequest) Unmarshal(dAtA []byte) error { } return nil } +func (m *MinimizedWriteRequestBytes) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRemote + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: MinimizedWriteRequestBytes: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: MinimizedWriteRequestBytes: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Timeseries", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRemote + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRemote + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthRemote + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Timeseries = append(m.Timeseries, MinimizedTimeSeriesBytes{}) + if err := m.Timeseries[len(m.Timeseries)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Symbols", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRemote + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRemote + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthRemote + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Symbols = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipRemote(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthRemote + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *MinimizedWriteRequestFixed32) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/prompb/remote.proto b/prompb/remote.proto index ae65cbd89..c28c3f36b 100644 --- a/prompb/remote.proto +++ b/prompb/remote.proto @@ -51,6 +51,18 @@ message MinimizedWriteRequest { string symbols = 4; } +message MinimizedWriteRequestBytes { + repeated MinimizedTimeSeriesBytes timeseries = 1 [(gogoproto.nullable) = false]; + // Cortex uses this field to determine the source of the write request. + // We reserve it to avoid any compatibility issues. + reserved 2; + // Metadata (3) has moved to be part of the TimeSeries type + reserved 3; + // The symbols table. All symbols are concatenated strings. To read the symbols table, it's required + // to know the offset:length range of the actual symbol to read from this string. + string symbols = 4; +} + message MinimizedWriteRequestFixed32 { repeated MinimizedTimeSeriesFixed32 timeseries = 1 [(gogoproto.nullable) = false]; // Cortex uses this field to determine the source of the write request. diff --git a/prompb/types.pb.go b/prompb/types.pb.go index 8084a9ec6..d764a9c4b 100644 --- a/prompb/types.pb.go +++ b/prompb/types.pb.go @@ -127,7 +127,7 @@ func (x LabelMatcher_Type) String() string { } func (LabelMatcher_Type) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_d938547f84707355, []int{12, 0} + return fileDescriptor_d938547f84707355, []int{13, 0} } // We require this to match chunkenc.Encoding. @@ -159,7 +159,7 @@ func (x Chunk_Encoding) String() string { } func (Chunk_Encoding) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_d938547f84707355, []int{14, 0} + return fileDescriptor_d938547f84707355, []int{15, 0} } type MetricMetadata struct { @@ -1034,6 +1034,82 @@ func (m *MinimizedTimeSeriesPacking) GetHistograms() []Histogram { return nil } +type MinimizedTimeSeriesBytes struct { + // Sorted list of label name-value pair references, encoded as varints. This + // list's real len is always multiple of 4, packing tuples of (label name offset, + // label name length, label value offset, label value length). + LabelSymbols []byte `protobuf:"bytes,1,opt,name=label_symbols,json=labelSymbols,proto3" json:"label_symbols,omitempty"` + // Sorted by time, oldest sample first. + // TODO: support references for other types + Samples []Sample `protobuf:"bytes,2,rep,name=samples,proto3" json:"samples"` + Exemplars []Exemplar `protobuf:"bytes,3,rep,name=exemplars,proto3" json:"exemplars"` + Histograms []Histogram `protobuf:"bytes,4,rep,name=histograms,proto3" json:"histograms"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *MinimizedTimeSeriesBytes) Reset() { *m = MinimizedTimeSeriesBytes{} } +func (m *MinimizedTimeSeriesBytes) String() string { return proto.CompactTextString(m) } +func (*MinimizedTimeSeriesBytes) ProtoMessage() {} +func (*MinimizedTimeSeriesBytes) Descriptor() ([]byte, []int) { + return fileDescriptor_d938547f84707355, []int{10} +} +func (m *MinimizedTimeSeriesBytes) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *MinimizedTimeSeriesBytes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_MinimizedTimeSeriesBytes.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *MinimizedTimeSeriesBytes) XXX_Merge(src proto.Message) { + xxx_messageInfo_MinimizedTimeSeriesBytes.Merge(m, src) +} +func (m *MinimizedTimeSeriesBytes) XXX_Size() int { + return m.Size() +} +func (m *MinimizedTimeSeriesBytes) XXX_DiscardUnknown() { + xxx_messageInfo_MinimizedTimeSeriesBytes.DiscardUnknown(m) +} + +var xxx_messageInfo_MinimizedTimeSeriesBytes proto.InternalMessageInfo + +func (m *MinimizedTimeSeriesBytes) GetLabelSymbols() []byte { + if m != nil { + return m.LabelSymbols + } + return nil +} + +func (m *MinimizedTimeSeriesBytes) GetSamples() []Sample { + if m != nil { + return m.Samples + } + return nil +} + +func (m *MinimizedTimeSeriesBytes) GetExemplars() []Exemplar { + if m != nil { + return m.Exemplars + } + return nil +} + +func (m *MinimizedTimeSeriesBytes) GetHistograms() []Histogram { + if m != nil { + return m.Histograms + } + return nil +} + type Label struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` @@ -1046,7 +1122,7 @@ func (m *Label) Reset() { *m = Label{} } func (m *Label) String() string { return proto.CompactTextString(m) } func (*Label) ProtoMessage() {} func (*Label) Descriptor() ([]byte, []int) { - return fileDescriptor_d938547f84707355, []int{10} + return fileDescriptor_d938547f84707355, []int{11} } func (m *Label) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1100,7 +1176,7 @@ func (m *Labels) Reset() { *m = Labels{} } func (m *Labels) String() string { return proto.CompactTextString(m) } func (*Labels) ProtoMessage() {} func (*Labels) Descriptor() ([]byte, []int) { - return fileDescriptor_d938547f84707355, []int{11} + return fileDescriptor_d938547f84707355, []int{12} } func (m *Labels) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1150,7 +1226,7 @@ func (m *LabelMatcher) Reset() { *m = LabelMatcher{} } func (m *LabelMatcher) String() string { return proto.CompactTextString(m) } func (*LabelMatcher) ProtoMessage() {} func (*LabelMatcher) Descriptor() ([]byte, []int) { - return fileDescriptor_d938547f84707355, []int{12} + return fileDescriptor_d938547f84707355, []int{13} } func (m *LabelMatcher) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1217,7 +1293,7 @@ func (m *ReadHints) Reset() { *m = ReadHints{} } func (m *ReadHints) String() string { return proto.CompactTextString(m) } func (*ReadHints) ProtoMessage() {} func (*ReadHints) Descriptor() ([]byte, []int) { - return fileDescriptor_d938547f84707355, []int{13} + return fileDescriptor_d938547f84707355, []int{14} } func (m *ReadHints) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1311,7 +1387,7 @@ func (m *Chunk) Reset() { *m = Chunk{} } func (m *Chunk) String() string { return proto.CompactTextString(m) } func (*Chunk) ProtoMessage() {} func (*Chunk) Descriptor() ([]byte, []int) { - return fileDescriptor_d938547f84707355, []int{14} + return fileDescriptor_d938547f84707355, []int{15} } func (m *Chunk) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1383,7 +1459,7 @@ func (m *ChunkedSeries) Reset() { *m = ChunkedSeries{} } func (m *ChunkedSeries) String() string { return proto.CompactTextString(m) } func (*ChunkedSeries) ProtoMessage() {} func (*ChunkedSeries) Descriptor() ([]byte, []int) { - return fileDescriptor_d938547f84707355, []int{15} + return fileDescriptor_d938547f84707355, []int{16} } func (m *ChunkedSeries) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1441,6 +1517,7 @@ func init() { proto.RegisterType((*MinimizedTimeSeriesFixed32)(nil), "prometheus.MinimizedTimeSeriesFixed32") proto.RegisterType((*MinimizedTimeSeriesFixed64)(nil), "prometheus.MinimizedTimeSeriesFixed64") proto.RegisterType((*MinimizedTimeSeriesPacking)(nil), "prometheus.MinimizedTimeSeriesPacking") + proto.RegisterType((*MinimizedTimeSeriesBytes)(nil), "prometheus.MinimizedTimeSeriesBytes") proto.RegisterType((*Label)(nil), "prometheus.Label") proto.RegisterType((*Labels)(nil), "prometheus.Labels") proto.RegisterType((*LabelMatcher)(nil), "prometheus.LabelMatcher") @@ -1452,81 +1529,82 @@ func init() { func init() { proto.RegisterFile("types.proto", fileDescriptor_d938547f84707355) } var fileDescriptor_d938547f84707355 = []byte{ - // 1174 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xdc, 0x57, 0xcb, 0x6e, 0xdb, 0x46, - 0x17, 0x36, 0x49, 0x89, 0x12, 0x8f, 0x2e, 0xa6, 0x27, 0x4e, 0x7e, 0xfe, 0x46, 0xe3, 0xa8, 0x04, - 0xd2, 0xaa, 0x45, 0x21, 0x23, 0x4e, 0x50, 0x34, 0x68, 0x50, 0xc0, 0x76, 0xe4, 0x0b, 0x6a, 0x4a, - 0xcd, 0x48, 0x46, 0x9b, 0x6e, 0x84, 0x91, 0x34, 0x96, 0x08, 0x8b, 0x97, 0x72, 0x46, 0x81, 0x95, - 0xf7, 0xe8, 0xae, 0x2f, 0xd1, 0xb7, 0x08, 0xd0, 0x4d, 0xfb, 0x02, 0x45, 0xe1, 0x55, 0x81, 0xee, - 0xfa, 0x04, 0xc5, 0x0c, 0x49, 0x91, 0x8a, 0x5d, 0xa0, 0xe9, 0xae, 0xde, 0xcd, 0xf9, 0xce, 0xed, - 0x3b, 0x73, 0x66, 0x38, 0x87, 0x50, 0xe1, 0x8b, 0x90, 0xb2, 0x56, 0x18, 0x05, 0x3c, 0x40, 0x10, - 0x46, 0x81, 0x47, 0xf9, 0x94, 0xce, 0xd9, 0xd6, 0xe6, 0x24, 0x98, 0x04, 0x12, 0xde, 0x11, 0xab, - 0xd8, 0xc2, 0xfe, 0x41, 0x85, 0xba, 0x43, 0x79, 0xe4, 0x8e, 0x1c, 0xca, 0xc9, 0x98, 0x70, 0x82, - 0x9e, 0x42, 0x41, 0xc4, 0xb0, 0x94, 0x86, 0xd2, 0xac, 0xef, 0x3e, 0x6c, 0x65, 0x31, 0x5a, 0xab, - 0x96, 0x89, 0xd8, 0x5f, 0x84, 0x14, 0x4b, 0x17, 0xf4, 0x09, 0x20, 0x4f, 0x62, 0x83, 0x73, 0xe2, - 0xb9, 0xb3, 0xc5, 0xc0, 0x27, 0x1e, 0xb5, 0xd4, 0x86, 0xd2, 0x34, 0xb0, 0x19, 0x6b, 0x0e, 0xa5, - 0xa2, 0x43, 0x3c, 0x8a, 0x10, 0x14, 0xa6, 0x74, 0x16, 0x5a, 0x05, 0xa9, 0x97, 0x6b, 0x81, 0xcd, - 0x7d, 0x97, 0x5b, 0xc5, 0x18, 0x13, 0x6b, 0x7b, 0x01, 0x90, 0x65, 0x42, 0x15, 0x28, 0x9d, 0x75, - 0xbe, 0xec, 0x74, 0xbf, 0xee, 0x98, 0x6b, 0x42, 0x38, 0xe8, 0x9e, 0x75, 0xfa, 0x6d, 0x6c, 0x2a, - 0xc8, 0x80, 0xe2, 0xd1, 0xde, 0xd9, 0x51, 0xdb, 0x54, 0x51, 0x0d, 0x8c, 0xe3, 0x93, 0x5e, 0xbf, - 0x7b, 0x84, 0xf7, 0x1c, 0x53, 0x43, 0x08, 0xea, 0x52, 0x93, 0x61, 0x05, 0xe1, 0xda, 0x3b, 0x73, - 0x9c, 0x3d, 0xfc, 0xd2, 0x2c, 0xa2, 0x32, 0x14, 0x4e, 0x3a, 0x87, 0x5d, 0x53, 0x47, 0x55, 0x28, - 0xf7, 0xfa, 0x7b, 0xfd, 0x76, 0xaf, 0xdd, 0x37, 0x4b, 0xf6, 0x33, 0xd0, 0x7b, 0xc4, 0x0b, 0x67, - 0x14, 0x6d, 0x42, 0xf1, 0x15, 0x99, 0xcd, 0xe3, 0x6d, 0x51, 0x70, 0x2c, 0xa0, 0xf7, 0xc0, 0xe0, - 0xae, 0x47, 0x19, 0x27, 0x5e, 0x28, 0xeb, 0xd4, 0x70, 0x06, 0xd8, 0x01, 0x94, 0xdb, 0x97, 0xd4, - 0x0b, 0x67, 0x24, 0x42, 0x3b, 0xa0, 0xcf, 0xc8, 0x90, 0xce, 0x98, 0xa5, 0x34, 0xb4, 0x66, 0x65, - 0x77, 0x23, 0xbf, 0xaf, 0xa7, 0x42, 0xb3, 0x5f, 0x78, 0xf3, 0xeb, 0x83, 0x35, 0x9c, 0x98, 0x65, - 0x09, 0xd5, 0xbf, 0x4d, 0xa8, 0xbd, 0x9d, 0xf0, 0xa7, 0x22, 0x18, 0xc7, 0x2e, 0xe3, 0xc1, 0x24, - 0x22, 0x1e, 0xba, 0x0f, 0xc6, 0x28, 0x98, 0xfb, 0x7c, 0xe0, 0xfa, 0x5c, 0xd2, 0x2e, 0x1c, 0xaf, - 0xe1, 0xb2, 0x84, 0x4e, 0x7c, 0x8e, 0xde, 0x87, 0x4a, 0xac, 0x3e, 0x9f, 0x05, 0x84, 0xc7, 0x69, - 0x8e, 0xd7, 0x30, 0x48, 0xf0, 0x50, 0x60, 0xc8, 0x04, 0x8d, 0xcd, 0x3d, 0x99, 0x47, 0xc1, 0x62, - 0x89, 0xee, 0x81, 0xce, 0x46, 0x53, 0xea, 0x11, 0xd9, 0xb5, 0x0d, 0x9c, 0x48, 0xe8, 0x21, 0xd4, - 0x5f, 0xd3, 0x28, 0x18, 0xf0, 0x69, 0x44, 0xd9, 0x34, 0x98, 0x8d, 0x65, 0x07, 0x15, 0x5c, 0x13, - 0x68, 0x3f, 0x05, 0xd1, 0x07, 0x89, 0x59, 0xc6, 0x4b, 0x97, 0xbc, 0x14, 0x5c, 0x15, 0xf8, 0x41, - 0xca, 0xed, 0x63, 0x30, 0x73, 0x76, 0x31, 0xc1, 0x92, 0x24, 0xa8, 0xe0, 0xfa, 0xd2, 0x32, 0x26, - 0x79, 0x00, 0x75, 0x9f, 0x4e, 0x08, 0x77, 0x5f, 0xd1, 0x01, 0x0b, 0x89, 0xcf, 0xac, 0xb2, 0xdc, - 0xe1, 0x7b, 0xf9, 0x1d, 0xde, 0x9f, 0x8f, 0x2e, 0x28, 0xef, 0x85, 0xc4, 0x4f, 0xb6, 0xb9, 0x96, - 0xfa, 0x08, 0x8c, 0xa1, 0x0f, 0x61, 0x7d, 0x19, 0x64, 0x4c, 0x67, 0x9c, 0x30, 0xcb, 0x68, 0x68, - 0x4d, 0x84, 0x97, 0xb1, 0x9f, 0x4b, 0x74, 0xc5, 0x50, 0xb2, 0x63, 0x16, 0x34, 0xb4, 0xa6, 0x92, - 0x19, 0x4a, 0x6a, 0x4c, 0xd0, 0x0a, 0x03, 0xe6, 0xe6, 0x68, 0x55, 0xfe, 0x09, 0xad, 0xd4, 0x67, - 0x49, 0x6b, 0x19, 0x24, 0xa1, 0x55, 0x8d, 0x69, 0xa5, 0x70, 0x46, 0x6b, 0x69, 0x98, 0xd0, 0xaa, - 0xc5, 0xb4, 0x52, 0x38, 0xa1, 0xf5, 0x05, 0x40, 0x44, 0x19, 0xe5, 0x83, 0xa9, 0xd8, 0xfd, 0xba, - 0xbc, 0xe3, 0x0f, 0xf2, 0x94, 0x96, 0xe7, 0xa7, 0x85, 0x85, 0xdd, 0xb1, 0xeb, 0x73, 0x6c, 0x44, - 0xe9, 0x72, 0xf5, 0x00, 0xae, 0xbf, 0x7d, 0x00, 0x9f, 0x80, 0xb1, 0xf4, 0x5a, 0xbd, 0xa9, 0x25, - 0xd0, 0x5e, 0xb6, 0x7b, 0xa6, 0x82, 0x74, 0x50, 0x3b, 0x5d, 0x53, 0xcd, 0x6e, 0xab, 0xb6, 0x5f, - 0x82, 0xa2, 0xe4, 0xbc, 0x5f, 0x05, 0xc8, 0xda, 0x6e, 0x3f, 0x03, 0xc8, 0xf6, 0x47, 0x9c, 0xbc, - 0xe0, 0xfc, 0x9c, 0xd1, 0xf8, 0x28, 0x6f, 0xe0, 0x44, 0x12, 0xf8, 0x8c, 0xfa, 0x13, 0x3e, 0x95, - 0x27, 0xb8, 0x86, 0x13, 0xc9, 0xfe, 0x5d, 0x01, 0xe8, 0xbb, 0x1e, 0xed, 0xd1, 0xc8, 0xa5, 0xec, - 0xdd, 0xef, 0xdf, 0x2e, 0x94, 0x98, 0xbc, 0xfa, 0xcc, 0x52, 0xa5, 0x07, 0xca, 0x7b, 0xc4, 0x5f, - 0x85, 0xc4, 0x25, 0x35, 0x44, 0x9f, 0x81, 0x41, 0x93, 0x0b, 0xcf, 0x2c, 0x4d, 0x7a, 0x6d, 0xe6, - 0xbd, 0xd2, 0xaf, 0x41, 0xe2, 0x97, 0x19, 0xa3, 0xcf, 0x01, 0xa6, 0xe9, 0xc6, 0x33, 0xab, 0x20, - 0x5d, 0xef, 0xde, 0xd8, 0x96, 0xc4, 0x37, 0x67, 0x6e, 0xff, 0xa1, 0xc0, 0x1d, 0xc7, 0xf5, 0x5d, - 0xcf, 0x7d, 0x4d, 0xc7, 0xb9, 0x9a, 0x3f, 0x82, 0x9a, 0x2c, 0x66, 0xc0, 0x16, 0xde, 0x30, 0x48, - 0x4a, 0xaf, 0x25, 0x01, 0xaa, 0x52, 0xd5, 0x8b, 0x35, 0xff, 0xa5, 0x6a, 0xff, 0x54, 0x60, 0xeb, - 0x86, 0x6a, 0x0f, 0xdd, 0x4b, 0x3a, 0x7e, 0xbc, 0x7b, 0x73, 0xd1, 0xa5, 0xdb, 0x5c, 0xf4, 0xa7, - 0x4f, 0x6e, 0x2e, 0x5a, 0xbf, 0xa5, 0x45, 0x7f, 0x45, 0x46, 0x17, 0xae, 0x3f, 0xb9, 0xa5, 0x9d, - 0x7e, 0x04, 0x45, 0xf9, 0x39, 0x12, 0xa3, 0x90, 0x1c, 0x9f, 0x94, 0x78, 0x14, 0x12, 0xeb, 0xd5, - 0xa1, 0xc0, 0x48, 0x86, 0x02, 0xfb, 0x29, 0xe8, 0xa7, 0xf1, 0x47, 0xeb, 0x5d, 0xbf, 0x72, 0xf6, - 0xf7, 0x0a, 0x54, 0x25, 0xee, 0x10, 0x3e, 0x9a, 0xd2, 0x08, 0x3d, 0x5a, 0x99, 0xfe, 0xee, 0x5f, - 0xf3, 0x4f, 0xec, 0x5a, 0xb9, 0xa9, 0x2f, 0x25, 0xaa, 0xde, 0x44, 0x54, 0xcb, 0x13, 0x6d, 0x42, - 0x41, 0xce, 0x70, 0x3a, 0xa8, 0xed, 0x17, 0xf1, 0xa3, 0xd0, 0x69, 0xbf, 0x88, 0x1f, 0x05, 0x2c, - 0xe6, 0x36, 0x01, 0xe0, 0xb6, 0xa9, 0xd9, 0x3f, 0x2a, 0xe2, 0x25, 0x21, 0x63, 0xf1, 0x90, 0x30, - 0xf4, 0x3f, 0x28, 0x31, 0x4e, 0xc3, 0x81, 0xc7, 0x24, 0x2f, 0x0d, 0xeb, 0x42, 0x74, 0x98, 0x48, - 0x7d, 0x3e, 0xf7, 0x47, 0x69, 0x6a, 0xb1, 0x46, 0xff, 0x87, 0x32, 0xe3, 0x24, 0xe2, 0xc2, 0x3a, - 0x9e, 0x90, 0x4a, 0x52, 0x76, 0x18, 0xba, 0x0b, 0x3a, 0xf5, 0xc7, 0x03, 0xd9, 0x14, 0xa1, 0x28, - 0x52, 0x7f, 0xec, 0x30, 0xb4, 0x05, 0xe5, 0x49, 0x14, 0xcc, 0x43, 0xd7, 0x9f, 0x58, 0xc5, 0x86, - 0xd6, 0x34, 0xf0, 0x52, 0x46, 0x75, 0x50, 0x87, 0x0b, 0x39, 0xa5, 0x94, 0xb1, 0x3a, 0x5c, 0x88, - 0xe8, 0x11, 0xf1, 0x27, 0x54, 0x04, 0x29, 0xc5, 0xd1, 0xa5, 0xec, 0x30, 0xfb, 0x17, 0x05, 0x8a, - 0x07, 0xd3, 0xb9, 0x7f, 0x81, 0xb6, 0xa1, 0xe2, 0xb9, 0xfe, 0x40, 0xbc, 0x8b, 0x19, 0x67, 0xc3, - 0x73, 0x7d, 0x71, 0x88, 0x1d, 0x26, 0xf5, 0xe4, 0x72, 0xa9, 0x4f, 0x06, 0x47, 0x8f, 0x5c, 0x26, - 0xfa, 0x56, 0xd2, 0x04, 0x4d, 0x36, 0x61, 0x2b, 0xdf, 0x04, 0x99, 0xa0, 0xd5, 0xf6, 0x47, 0xc1, - 0xd8, 0xf5, 0x27, 0x59, 0x07, 0xc4, 0x40, 0x2e, 0xab, 0xaa, 0x62, 0xb9, 0xb6, 0x9f, 0x43, 0x39, - 0xb5, 0xba, 0xf6, 0x12, 0x7f, 0xd3, 0x15, 0xf3, 0xf2, 0xca, 0x90, 0xac, 0xa2, 0x3b, 0xb0, 0x7e, - 0x78, 0xda, 0xdd, 0xeb, 0x0f, 0x72, 0x93, 0xb3, 0xfd, 0x1d, 0xd4, 0x64, 0x46, 0x3a, 0xfe, 0xb7, - 0xef, 0xe8, 0x0e, 0xe8, 0x23, 0x11, 0x21, 0xbd, 0x79, 0x1b, 0xd7, 0xaa, 0x49, 0x1d, 0x62, 0xb3, - 0xfd, 0xcd, 0x37, 0x57, 0xdb, 0xca, 0xcf, 0x57, 0xdb, 0xca, 0x6f, 0x57, 0xdb, 0xca, 0xb7, 0xba, - 0xb0, 0x0e, 0x87, 0x43, 0x5d, 0xfe, 0xaf, 0x3c, 0xfe, 0x2b, 0x00, 0x00, 0xff, 0xff, 0xc8, 0x59, - 0x21, 0x0f, 0xe0, 0x0c, 0x00, 0x00, + // 1193 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xdc, 0x57, 0xdd, 0x8e, 0xdb, 0x44, + 0x14, 0xde, 0xb1, 0x13, 0x27, 0x3e, 0x9b, 0xa4, 0xde, 0xe9, 0xb6, 0x98, 0x15, 0xdd, 0x2e, 0x46, + 0x85, 0x80, 0x50, 0xaa, 0x6e, 0x2b, 0x44, 0x45, 0x85, 0xb4, 0xbb, 0xcd, 0x76, 0x2b, 0xea, 0x84, + 0x4e, 0xb2, 0x82, 0x72, 0x13, 0x4d, 0x92, 0xd9, 0xc4, 0x6a, 0xfc, 0x83, 0x67, 0x52, 0x6d, 0xfa, + 0x1e, 0xdc, 0xf1, 0x12, 0xbc, 0x45, 0x25, 0x6e, 0xe0, 0x05, 0x10, 0xea, 0x15, 0x82, 0x3b, 0x9e, + 0x00, 0xcd, 0xd8, 0x8e, 0x9d, 0x6e, 0x90, 0x28, 0x77, 0xed, 0xdd, 0x9c, 0xef, 0xfc, 0x7d, 0x67, + 0xce, 0x8c, 0x7d, 0x06, 0x36, 0xc5, 0x22, 0x62, 0xbc, 0x15, 0xc5, 0xa1, 0x08, 0x31, 0x44, 0x71, + 0xe8, 0x33, 0x31, 0x65, 0x73, 0xbe, 0xb3, 0x3d, 0x09, 0x27, 0xa1, 0x82, 0x6f, 0xca, 0x55, 0x62, + 0xe1, 0xfc, 0xa8, 0x41, 0xc3, 0x65, 0x22, 0xf6, 0x46, 0x2e, 0x13, 0x74, 0x4c, 0x05, 0xc5, 0x77, + 0xa1, 0x24, 0x63, 0xd8, 0x68, 0x0f, 0x35, 0x1b, 0xfb, 0x37, 0x5a, 0x79, 0x8c, 0xd6, 0xaa, 0x65, + 0x2a, 0xf6, 0x17, 0x11, 0x23, 0xca, 0x05, 0x7f, 0x0a, 0xd8, 0x57, 0xd8, 0xe0, 0x8c, 0xfa, 0xde, + 0x6c, 0x31, 0x08, 0xa8, 0xcf, 0x6c, 0x6d, 0x0f, 0x35, 0x4d, 0x62, 0x25, 0x9a, 0x63, 0xa5, 0xe8, + 0x50, 0x9f, 0x61, 0x0c, 0xa5, 0x29, 0x9b, 0x45, 0x76, 0x49, 0xe9, 0xd5, 0x5a, 0x62, 0xf3, 0xc0, + 0x13, 0x76, 0x39, 0xc1, 0xe4, 0xda, 0x59, 0x00, 0xe4, 0x99, 0xf0, 0x26, 0x54, 0x4e, 0x3b, 0x5f, + 0x75, 0xba, 0xdf, 0x74, 0xac, 0x0d, 0x29, 0x1c, 0x75, 0x4f, 0x3b, 0xfd, 0x36, 0xb1, 0x10, 0x36, + 0xa1, 0xfc, 0xe0, 0xe0, 0xf4, 0x41, 0xdb, 0xd2, 0x70, 0x1d, 0xcc, 0x93, 0x87, 0xbd, 0x7e, 0xf7, + 0x01, 0x39, 0x70, 0x2d, 0x1d, 0x63, 0x68, 0x28, 0x4d, 0x8e, 0x95, 0xa4, 0x6b, 0xef, 0xd4, 0x75, + 0x0f, 0xc8, 0x13, 0xab, 0x8c, 0xab, 0x50, 0x7a, 0xd8, 0x39, 0xee, 0x5a, 0x06, 0xae, 0x41, 0xb5, + 0xd7, 0x3f, 0xe8, 0xb7, 0x7b, 0xed, 0xbe, 0x55, 0x71, 0xee, 0x81, 0xd1, 0xa3, 0x7e, 0x34, 0x63, + 0x78, 0x1b, 0xca, 0xcf, 0xe8, 0x6c, 0x9e, 0x6c, 0x0b, 0x22, 0x89, 0x80, 0xdf, 0x03, 0x53, 0x78, + 0x3e, 0xe3, 0x82, 0xfa, 0x91, 0xaa, 0x53, 0x27, 0x39, 0xe0, 0x84, 0x50, 0x6d, 0x9f, 0x33, 0x3f, + 0x9a, 0xd1, 0x18, 0xdf, 0x04, 0x63, 0x46, 0x87, 0x6c, 0xc6, 0x6d, 0xb4, 0xa7, 0x37, 0x37, 0xf7, + 0xb7, 0x8a, 0xfb, 0xfa, 0x48, 0x6a, 0x0e, 0x4b, 0x2f, 0x7e, 0xbb, 0xbe, 0x41, 0x52, 0xb3, 0x3c, + 0xa1, 0xf6, 0xaf, 0x09, 0xf5, 0x57, 0x13, 0xfe, 0x5c, 0x06, 0xf3, 0xc4, 0xe3, 0x22, 0x9c, 0xc4, + 0xd4, 0xc7, 0xd7, 0xc0, 0x1c, 0x85, 0xf3, 0x40, 0x0c, 0xbc, 0x40, 0x28, 0xda, 0xa5, 0x93, 0x0d, + 0x52, 0x55, 0xd0, 0xc3, 0x40, 0xe0, 0xf7, 0x61, 0x33, 0x51, 0x9f, 0xcd, 0x42, 0x2a, 0x92, 0x34, + 0x27, 0x1b, 0x04, 0x14, 0x78, 0x2c, 0x31, 0x6c, 0x81, 0xce, 0xe7, 0xbe, 0xca, 0x83, 0x88, 0x5c, + 0xe2, 0xab, 0x60, 0xf0, 0xd1, 0x94, 0xf9, 0x54, 0x75, 0x6d, 0x8b, 0xa4, 0x12, 0xbe, 0x01, 0x8d, + 0xe7, 0x2c, 0x0e, 0x07, 0x62, 0x1a, 0x33, 0x3e, 0x0d, 0x67, 0x63, 0xd5, 0x41, 0x44, 0xea, 0x12, + 0xed, 0x67, 0x20, 0xfe, 0x30, 0x35, 0xcb, 0x79, 0x19, 0x8a, 0x17, 0x22, 0x35, 0x89, 0x1f, 0x65, + 0xdc, 0x3e, 0x01, 0xab, 0x60, 0x97, 0x10, 0xac, 0x28, 0x82, 0x88, 0x34, 0x96, 0x96, 0x09, 0xc9, + 0x23, 0x68, 0x04, 0x6c, 0x42, 0x85, 0xf7, 0x8c, 0x0d, 0x78, 0x44, 0x03, 0x6e, 0x57, 0xd5, 0x0e, + 0x5f, 0x2d, 0xee, 0xf0, 0xe1, 0x7c, 0xf4, 0x94, 0x89, 0x5e, 0x44, 0x83, 0x74, 0x9b, 0xeb, 0x99, + 0x8f, 0xc4, 0x38, 0xfe, 0x08, 0x2e, 0x2d, 0x83, 0x8c, 0xd9, 0x4c, 0x50, 0x6e, 0x9b, 0x7b, 0x7a, + 0x13, 0x93, 0x65, 0xec, 0xfb, 0x0a, 0x5d, 0x31, 0x54, 0xec, 0xb8, 0x0d, 0x7b, 0x7a, 0x13, 0xe5, + 0x86, 0x8a, 0x1a, 0x97, 0xb4, 0xa2, 0x90, 0x7b, 0x05, 0x5a, 0x9b, 0xff, 0x85, 0x56, 0xe6, 0xb3, + 0xa4, 0xb5, 0x0c, 0x92, 0xd2, 0xaa, 0x25, 0xb4, 0x32, 0x38, 0xa7, 0xb5, 0x34, 0x4c, 0x69, 0xd5, + 0x13, 0x5a, 0x19, 0x9c, 0xd2, 0xfa, 0x12, 0x20, 0x66, 0x9c, 0x89, 0xc1, 0x54, 0xee, 0x7e, 0x43, + 0xdd, 0xf1, 0xeb, 0x45, 0x4a, 0xcb, 0xf3, 0xd3, 0x22, 0xd2, 0xee, 0xc4, 0x0b, 0x04, 0x31, 0xe3, + 0x6c, 0xb9, 0x7a, 0x00, 0x2f, 0xbd, 0x7a, 0x00, 0xef, 0x80, 0xb9, 0xf4, 0x5a, 0xbd, 0xa9, 0x15, + 0xd0, 0x9f, 0xb4, 0x7b, 0x16, 0xc2, 0x06, 0x68, 0x9d, 0xae, 0xa5, 0xe5, 0xb7, 0x55, 0x3f, 0xac, + 0x40, 0x59, 0x71, 0x3e, 0xac, 0x01, 0xe4, 0x6d, 0x77, 0xee, 0x01, 0xe4, 0xfb, 0x23, 0x4f, 0x5e, + 0x78, 0x76, 0xc6, 0x59, 0x72, 0x94, 0xb7, 0x48, 0x2a, 0x49, 0x7c, 0xc6, 0x82, 0x89, 0x98, 0xaa, + 0x13, 0x5c, 0x27, 0xa9, 0xe4, 0xfc, 0x81, 0x00, 0xfa, 0x9e, 0xcf, 0x7a, 0x2c, 0xf6, 0x18, 0x7f, + 0xfd, 0xfb, 0xb7, 0x0f, 0x15, 0xae, 0xae, 0x3e, 0xb7, 0x35, 0xe5, 0x81, 0x8b, 0x1e, 0xc9, 0x57, + 0x21, 0x75, 0xc9, 0x0c, 0xf1, 0xe7, 0x60, 0xb2, 0xf4, 0xc2, 0x73, 0x5b, 0x57, 0x5e, 0xdb, 0x45, + 0xaf, 0xec, 0x6b, 0x90, 0xfa, 0xe5, 0xc6, 0xf8, 0x0b, 0x80, 0x69, 0xb6, 0xf1, 0xdc, 0x2e, 0x29, + 0xd7, 0x2b, 0x6b, 0xdb, 0x92, 0xfa, 0x16, 0xcc, 0x9d, 0xbf, 0x10, 0x5c, 0x76, 0xbd, 0xc0, 0xf3, + 0xbd, 0xe7, 0x6c, 0x5c, 0xa8, 0xf9, 0x63, 0xa8, 0xab, 0x62, 0x06, 0x7c, 0xe1, 0x0f, 0xc3, 0xb4, + 0xf4, 0x7a, 0x1a, 0xa0, 0xa6, 0x54, 0xbd, 0x44, 0xf3, 0x26, 0x55, 0xfb, 0x37, 0x82, 0x9d, 0x35, + 0xd5, 0x1e, 0x7b, 0xe7, 0x6c, 0x7c, 0x7b, 0x7f, 0x7d, 0xd1, 0x95, 0xb7, 0xb9, 0xe8, 0xcf, 0xee, + 0xac, 0x2f, 0xda, 0x78, 0x4b, 0x8b, 0xfe, 0x9a, 0x8e, 0x9e, 0x7a, 0xc1, 0xe4, 0x2d, 0xed, 0xf4, + 0x9f, 0x08, 0xec, 0x35, 0x45, 0x1f, 0x2e, 0x04, 0xe3, 0xf8, 0x83, 0x8b, 0x25, 0xa3, 0x66, 0xed, + 0xcd, 0x2d, 0xf6, 0x16, 0x94, 0xd5, 0xb7, 0x57, 0xce, 0x7d, 0x6a, 0x56, 0x44, 0xc9, 0xdc, 0x27, + 0xd7, 0xab, 0x13, 0x90, 0x99, 0x4e, 0x40, 0xce, 0x5d, 0x30, 0x1e, 0x25, 0x5f, 0xe8, 0xd7, 0xfd, + 0xa4, 0x3b, 0x3f, 0x20, 0xa8, 0x29, 0xdc, 0xa5, 0x62, 0x34, 0x65, 0x31, 0xbe, 0xb5, 0x32, 0xea, + 0x5e, 0xbb, 0xe0, 0x9f, 0xda, 0xb5, 0x0a, 0x23, 0x6e, 0x46, 0x54, 0x5b, 0x47, 0x54, 0x2f, 0x12, + 0x6d, 0x42, 0x49, 0x0d, 0xac, 0x06, 0x68, 0xed, 0xc7, 0xc9, 0x1f, 0xb0, 0xd3, 0x7e, 0x9c, 0xfc, + 0x01, 0x89, 0x1c, 0x52, 0x25, 0x40, 0xda, 0x96, 0xee, 0xfc, 0x84, 0xe4, 0x6f, 0x93, 0x8e, 0xe5, + 0x5f, 0x93, 0xe3, 0x77, 0xa0, 0xc2, 0x05, 0x8b, 0x06, 0x7e, 0xd2, 0x5d, 0x9d, 0x18, 0x52, 0x74, + 0xb9, 0x4c, 0x7d, 0x36, 0x0f, 0x46, 0x59, 0x6a, 0xb9, 0xc6, 0xef, 0x42, 0x95, 0x0b, 0x1a, 0x0b, + 0x69, 0x9d, 0x8c, 0x83, 0x15, 0x25, 0xbb, 0x1c, 0x5f, 0x01, 0x83, 0x05, 0xe3, 0x81, 0x6a, 0x8a, + 0x54, 0x94, 0x59, 0x30, 0x76, 0x39, 0xde, 0x81, 0xea, 0x24, 0x0e, 0xe7, 0x91, 0x17, 0x4c, 0xec, + 0xf2, 0x9e, 0xde, 0x34, 0xc9, 0x52, 0xc6, 0x0d, 0xd0, 0x86, 0x0b, 0x35, 0x92, 0x55, 0x89, 0x36, + 0x5c, 0xc8, 0xe8, 0x31, 0x0d, 0x26, 0x4c, 0x06, 0xa9, 0x24, 0xd1, 0x95, 0xec, 0x72, 0xe7, 0x57, + 0x04, 0xe5, 0xa3, 0xe9, 0x3c, 0x78, 0x8a, 0x77, 0x61, 0xd3, 0xf7, 0x82, 0x81, 0x1c, 0x02, 0x72, + 0xce, 0xa6, 0xef, 0x05, 0xf2, 0xf0, 0xba, 0x5c, 0xe9, 0xe9, 0xf9, 0x52, 0x9f, 0x4e, 0xc9, 0x3e, + 0x3d, 0x4f, 0xf5, 0xad, 0xb4, 0x09, 0xba, 0x6a, 0xc2, 0x4e, 0xb1, 0x09, 0x2a, 0x41, 0xab, 0x1d, + 0x8c, 0xc2, 0xb1, 0x17, 0x4c, 0xf2, 0x0e, 0xc8, 0xd7, 0x87, 0xaa, 0xaa, 0x46, 0xd4, 0xda, 0xb9, + 0x0f, 0xd5, 0xcc, 0xea, 0xc2, 0xd8, 0xf1, 0x6d, 0x57, 0x3e, 0x0e, 0x56, 0x5e, 0x04, 0x1a, 0xbe, + 0x0c, 0x97, 0x8e, 0x1f, 0x75, 0x0f, 0xfa, 0x83, 0xc2, 0x33, 0xc1, 0xf9, 0x1e, 0xea, 0x2a, 0x23, + 0x1b, 0xff, 0xdf, 0xa1, 0xe1, 0x26, 0x18, 0x23, 0x19, 0x21, 0xbb, 0x79, 0x5b, 0x17, 0xaa, 0xc9, + 0x1c, 0x12, 0xb3, 0xc3, 0xed, 0x17, 0x2f, 0x77, 0xd1, 0x2f, 0x2f, 0x77, 0xd1, 0xef, 0x2f, 0x77, + 0xd1, 0x77, 0x86, 0xb4, 0x8e, 0x86, 0x43, 0x43, 0x3d, 0xce, 0x6e, 0xff, 0x13, 0x00, 0x00, 0xff, + 0xff, 0xda, 0xb3, 0x16, 0xfd, 0xcd, 0x0d, 0x00, 0x00, } func (m *MetricMetadata) Marshal() (dAtA []byte, err error) { @@ -2321,6 +2399,82 @@ func (m *MinimizedTimeSeriesPacking) MarshalToSizedBuffer(dAtA []byte) (int, err return len(dAtA) - i, nil } +func (m *MinimizedTimeSeriesBytes) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *MinimizedTimeSeriesBytes) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *MinimizedTimeSeriesBytes) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Histograms) > 0 { + for iNdEx := len(m.Histograms) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Histograms[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintTypes(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x22 + } + } + if len(m.Exemplars) > 0 { + for iNdEx := len(m.Exemplars) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Exemplars[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintTypes(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a + } + } + if len(m.Samples) > 0 { + for iNdEx := len(m.Samples) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Samples[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintTypes(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + } + if len(m.LabelSymbols) > 0 { + i -= len(m.LabelSymbols) + copy(dAtA[i:], m.LabelSymbols) + i = encodeVarintTypes(dAtA, i, uint64(len(m.LabelSymbols))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func (m *Label) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -2997,6 +3151,40 @@ func (m *MinimizedTimeSeriesPacking) Size() (n int) { return n } +func (m *MinimizedTimeSeriesBytes) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.LabelSymbols) + if l > 0 { + n += 1 + l + sovTypes(uint64(l)) + } + if len(m.Samples) > 0 { + for _, e := range m.Samples { + l = e.Size() + n += 1 + l + sovTypes(uint64(l)) + } + } + if len(m.Exemplars) > 0 { + for _, e := range m.Exemplars { + l = e.Size() + n += 1 + l + sovTypes(uint64(l)) + } + } + if len(m.Histograms) > 0 { + for _, e := range m.Histograms { + l = e.Size() + n += 1 + l + sovTypes(uint64(l)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func (m *Label) Size() (n int) { if m == nil { return 0 @@ -5160,6 +5348,193 @@ func (m *MinimizedTimeSeriesPacking) Unmarshal(dAtA []byte) error { } return nil } +func (m *MinimizedTimeSeriesBytes) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: MinimizedTimeSeriesBytes: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: MinimizedTimeSeriesBytes: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field LabelSymbols", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.LabelSymbols = append(m.LabelSymbols[:0], dAtA[iNdEx:postIndex]...) + if m.LabelSymbols == nil { + m.LabelSymbols = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Samples", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Samples = append(m.Samples, Sample{}) + if err := m.Samples[len(m.Samples)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Exemplars", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Exemplars = append(m.Exemplars, Exemplar{}) + if err := m.Exemplars[len(m.Exemplars)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Histograms", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Histograms = append(m.Histograms, Histogram{}) + if err := m.Histograms[len(m.Histograms)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipTypes(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *Label) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/prompb/types.proto b/prompb/types.proto index af9902768..21c884d06 100644 --- a/prompb/types.proto +++ b/prompb/types.proto @@ -181,6 +181,20 @@ message MinimizedTimeSeriesPacking { repeated Histogram histograms = 4 [(gogoproto.nullable) = false]; } +message MinimizedTimeSeriesBytes { + // Sorted list of label name-value pair references, encoded as varints. This + // list's real len is always multiple of 4, packing tuples of (label name offset, + // label name length, label value offset, label value length). + bytes label_symbols = 1; + + // Sorted by time, oldest sample first. + // TODO: support references for other types + repeated Sample samples = 2 [(gogoproto.nullable) = false]; + repeated Exemplar exemplars = 3 [(gogoproto.nullable) = false]; + repeated Histogram histograms = 4 [(gogoproto.nullable) = false]; + // TODO: add metadata +} + message Label { string name = 1; string value = 2; diff --git a/scripts/remotewrite11-bench/run.sh b/scripts/remotewrite11-bench/run.sh index 4c39e66f2..ccdba232d 100755 --- a/scripts/remotewrite11-bench/run.sh +++ b/scripts/remotewrite11-bench/run.sh @@ -7,10 +7,11 @@ trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT declare -a INSTANCES # (sender,receiver) pairs to run: (sender_name; sender_flags; receiver_name; receiver_flags) INSTANCES+=('sender-v1;;receiver-v1;') -INSTANCES+=('sender-v11;--remote-write-format 1;receiver-v11;--remote-write-format 1') -INSTANCES+=('sender-v11-min32-optimized-varint;--remote-write-format 2;receiver-v11-min32-optimized-varint;--remote-write-format 2') -INSTANCES+=('sender-v11-min64-fixed;--remote-write-format 3;receiver-v11-min64-fixed;--remote-write-format 3') -INSTANCES+=('sender-v11-min32-fixed;--remote-write-format 4;receiver-v11-min32-fixed;--remote-write-format 4') +INSTANCES+=('sender-v11-min32-optimized-varint;--remote-write-format 1;receiver-v11-min32-optimized-varint;--remote-write-format 1') +INSTANCES+=('sender-v11-min64-fixed;--remote-write-format 2;receiver-v11-min64-fixed;--remote-write-format 2') +INSTANCES+=('sender-v11-min32-fixed;--remote-write-format 3;receiver-v11-min32-fixed;--remote-write-format 3') +INSTANCES+=('sender-v11-min-bytes;--remote-write-format 4;receiver-v11-min-bytes;--remote-write-format 4') + # ~~~~~~~~~~~~~ # append two ports to all instances diff --git a/storage/remote/codec.go b/storage/remote/codec.go index 7127a147e..320d32d2f 100644 --- a/storage/remote/codec.go +++ b/storage/remote/codec.go @@ -15,6 +15,7 @@ package remote import ( "compress/gzip" + "encoding/binary" "errors" "fmt" "io" @@ -812,6 +813,19 @@ func labelsToUint32Packed(lbls labels.Labels, symbolTable *rwSymbolTable, buf [] return result } +func labelsToByteSlice(lbls labels.Labels, symbolTable *rwSymbolTable, buf []byte) []byte { + result := buf[:0] + lbls.Range(func(l labels.Label) { + off, leng := symbolTable.Ref(l.Name) + result = binary.AppendUvarint(result, uint64(off)) + result = binary.AppendUvarint(result, uint64(leng)) + off, leng = symbolTable.Ref(l.Value) + result = binary.AppendUvarint(result, uint64(off)) + result = binary.AppendUvarint(result, uint64(leng)) + }) + return result +} + func Uint32RefToLabels(symbols string, minLabels []uint32) labels.Labels { ls := labels.NewScratchBuilder(len(minLabels) / 2) @@ -835,6 +849,45 @@ func Uint32RefToLabels(symbols string, minLabels []uint32) labels.Labels { return ls.Labels() } +func ByteSliceToLabels(symbols string, minLabels []byte) labels.Labels { + ls := labels.NewScratchBuilder(len(minLabels) / 2) + + for len(minLabels) > 0 { + // todo, check for overflow? + offset, n := binary.Uvarint(minLabels) + minLabels = minLabels[n:] + length, n := binary.Uvarint(minLabels) + minLabels = minLabels[n:] + name := symbols[offset : offset+length] + // todo, check for overflow? + offset, n = binary.Uvarint(minLabels) + minLabels = minLabels[n:] + length, n = binary.Uvarint(minLabels) + minLabels = minLabels[n:] + value := symbols[offset : offset+length] + ls.Add(name, value) + } + + // labelIdx := 0 + // for labelIdx < len(minLabels) { + // // todo, check for overflow? + // offset := minLabels[labelIdx] + // labelIdx++ + // length := minLabels[labelIdx] + // labelIdx++ + // name := symbols[offset : offset+length] + // // todo, check for overflow? + // offset = minLabels[labelIdx] + // labelIdx++ + // length = minLabels[labelIdx] + // labelIdx++ + // value := symbols[offset : offset+length] + // ls.Add(name, value) + // } + + return ls.Labels() +} + func Uint64RefToLabels(symbols string, minLabels []uint64) labels.Labels { ls := labels.NewScratchBuilder(len(minLabels) / 2) labelIdx := 0 @@ -996,6 +1049,25 @@ func DecodeMinimizedWriteRequestFixed32(r io.Reader) (*prompb.MinimizedWriteRequ return &req, nil } +func DecodeMinimizedWriteRequestBytes(r io.Reader) (*prompb.MinimizedWriteRequestBytes, error) { + compressed, err := io.ReadAll(r) + if err != nil { + return nil, err + } + + reqBuf, err := snappy.Decode(nil, compressed) + if err != nil { + return nil, err + } + + var req prompb.MinimizedWriteRequestBytes + if err := proto.Unmarshal(reqBuf, &req); err != nil { + return nil, err + } + + return &req, nil +} + func MinimizedWriteRequestToWriteRequest(redReq *prompb.MinimizedWriteRequest) (*prompb.WriteRequest, error) { req := &prompb.WriteRequest{ Timeseries: make([]prompb.TimeSeries, len(redReq.Timeseries)), @@ -1041,18 +1113,13 @@ func min64WriteRequestToWriteRequest(redReq *prompb.MinimizedWriteRequestFixed64 }) }) - exemplars := make([]prompb.Exemplar, len(rts.Exemplars)) // TODO handle exemplars - //for j, e := range rts.Exemplars { - // exemplars[j].Value = e.Value - // exemplars[j].Timestamp = e.Timestamp - // exemplars[j].Labels = make([]prompb.Label, len(e.Labels)) - // - // for k, l := range e.Labels { - // exemplars[j].Labels[k].Name = redReq.StringSymbolTable[l.NameRef] - // exemplars[j].Labels[k].Value = redReq.StringSymbolTable[l.ValueRef] - // } - //} + exemplars := make([]prompb.Exemplar, len(rts.Exemplars)) + for j, e := range rts.Exemplars { + exemplars[j].Value = e.Value + exemplars[j].Timestamp = e.Timestamp + exemplars[j].Labels = e.Labels + } req.Timeseries[i].Samples = rts.Samples req.Timeseries[i].Exemplars = exemplars diff --git a/storage/remote/queue_manager.go b/storage/remote/queue_manager.go index 64046004c..caf61d981 100644 --- a/storage/remote/queue_manager.go +++ b/storage/remote/queue_manager.go @@ -395,7 +395,8 @@ const ( Base1 RemoteWriteFormat = iota // original map based format Min32Optimized // two 32bit varint plus marshalling optimization Min64Fixed // a single fixed64 bit value, first 32 are offset and 2nd 32 are - Min32Fixed // two 32bit fixed, similar to optimized but not varints + no manual marshalling optimization + Min32Fixed + MinBytes // two 32bit fixed, similar to optimized but not varints + no manual marshalling optimization ) // QueueManager manages a queue of samples to be sent to the Storage @@ -1396,6 +1397,11 @@ func (s *shards) runShard(ctx context.Context, shardID int, queue *queue) { pendingMin32Data[i].Samples = []prompb.Sample{{}} } + pendingMinBytesData := make([]prompb.MinimizedTimeSeriesBytes, max) + for i := range pendingMinBytesData { + pendingMinBytesData[i].Samples = []prompb.Sample{{}} + } + timer := time.NewTimer(time.Duration(s.qm.cfg.BatchSendDeadline)) stop := func() { if !timer.Stop() { @@ -1450,6 +1456,11 @@ func (s *shards) runShard(ctx context.Context, shardID int, queue *queue) { n := nPendingSamples + nPendingExemplars + nPendingHistograms s.sendMin32Samples(ctx, pendingMin32Data[:n], symbolTable.LabelsString(), nPendingSamples, nPendingExemplars, nPendingHistograms, pBuf, &buf) symbolTable.clear() + case MinBytes: + nPendingSamples, nPendingExemplars, nPendingHistograms := populateMinimizedTimeSeriesBytes(&symbolTable, batch, pendingMinBytesData, s.qm.sendExemplars, s.qm.sendNativeHistograms) + n := nPendingSamples + nPendingExemplars + nPendingHistograms + s.sendMin32Samples(ctx, pendingMin32Data[:n], symbolTable.LabelsString(), nPendingSamples, nPendingExemplars, nPendingHistograms, pBuf, &buf) + symbolTable.clear() } queue.ReturnForReuse(batch) @@ -1577,6 +1588,18 @@ func (s *shards) sendMin32Samples(ctx context.Context, samples []prompb.Minimize s.updateMetrics(ctx, err, sampleCount, exemplarCount, histogramCount, time.Since(begin)) } +func (s *shards) sendMinBytes(ctx context.Context, samples []prompb.MinimizedTimeSeriesBytes, labels string, sampleCount, exemplarCount, histogramCount int, pBuf *proto.Buffer, buf *[]byte) { + begin := time.Now() + // Build the ReducedWriteRequest with no metadata. + // Failing to build the write request is non-recoverable, since it will + // only error if marshaling the proto to bytes fails. + req, highest, err := buildMinimizedWriteRequestBytes(samples, labels, pBuf, buf) + if err == nil { + err = s.sendSamplesWithBackoff(ctx, req, sampleCount, exemplarCount, histogramCount, highest) + } + s.updateMetrics(ctx, err, sampleCount, exemplarCount, histogramCount, time.Since(begin)) +} + func (s *shards) updateMetrics(ctx context.Context, err error, sampleCount, exemplarCount, histogramCount int, duration time.Duration) { if err != nil { level.Error(s.qm.logger).Log("msg", "non-recoverable error", "count", sampleCount, "exemplarCount", exemplarCount, "err", err) @@ -1725,21 +1748,15 @@ func populateMinimizedTimeSeriesFixed64(symbolTable *rwSymbolTable, batch []time Timestamp: d.timestamp, }) nPendingSamples++ - // TODO: handle all types - //case tExemplar: - // // TODO(npazosmendez) optimize? - // l := make([]prompb.LabelRef, 0, d.exemplarLabels.Len()) - // d.exemplarLabels.Range(func(el labels.Label) { - // nRef := pool.intern(el.Name) - // vRef := pool.intern(el.Value) - // l = append(l, prompb.LabelRef{NameRef: nRef, ValueRef: vRef}) - // }) - // pendingData[nPending].Exemplars = append(pendingData[nPending].Exemplars, prompb.ExemplarRef{ - // Labels: l, - // Value: d.value, - // Timestamp: d.timestamp, - // }) - // nPendingExemplars++ + // TODO: handle all types + case tExemplar: + // TODO intern exemplars and histograms for the new formats + pendingData[nPending].Exemplars = append(pendingData[nPending].Exemplars, prompb.Exemplar{ + Labels: labelsToLabelsProto(d.exemplarLabels, nil), + Value: d.value, + Timestamp: d.timestamp, + }) + nPendingExemplars++ case tHistogram: pendingData[nPending].Histograms = append(pendingData[nPending].Histograms, HistogramToHistogramProto(d.timestamp, d.histogram)) nPendingHistograms++ @@ -1801,6 +1818,55 @@ func populateMinimizedTimeSeriesFixed32(symbolTable *rwSymbolTable, batch []time return nPendingSamples, nPendingExemplars, nPendingHistograms } +func populateMinimizedTimeSeriesBytes(symbolTable *rwSymbolTable, batch []timeSeries, pendingData []prompb.MinimizedTimeSeriesBytes, sendExemplars, sendNativeHistograms bool) (int, int, int) { + var nPendingSamples, nPendingExemplars, nPendingHistograms int + for nPending, d := range batch { + pendingData[nPending].Samples = pendingData[nPending].Samples[:0] + if sendExemplars { + pendingData[nPending].Exemplars = pendingData[nPending].Exemplars[:0] + } + if sendNativeHistograms { + pendingData[nPending].Histograms = pendingData[nPending].Histograms[:0] + } + + // Number of pending samples is limited by the fact that sendSamples (via sendSamplesWithBackoff) + // retries endlessly, so once we reach max samples, if we can never send to the endpoint we'll + // stop reading from the queue. This makes it safe to reference pendingSamples by index. + // pendingData[nPending].Labels = labelsToLabelsProto(d.seriesLabels, pendingData[nPending].Labels) + + pendingData[nPending].LabelSymbols = labelsToByteSlice(d.seriesLabels, symbolTable, pendingData[nPending].LabelSymbols) + switch d.sType { + case tSample: + pendingData[nPending].Samples = append(pendingData[nPending].Samples, prompb.Sample{ + Value: d.value, + Timestamp: d.timestamp, + }) + nPendingSamples++ + // TODO: handle all types + //case tExemplar: + // l := make([]prompb.LabelRef, 0, d.exemplarLabels.Len()) + // d.exemplarLabels.Range(func(el labels.Label) { + // nRef := pool.intern(el.Name) + // vRef := pool.intern(el.Value) + // l = append(l, prompb.LabelRef{NameRef: nRef, ValueRef: vRef}) + // }) + // pendingData[nPending].Exemplars = append(pendingData[nPending].Exemplars, prompb.ExemplarRef{ + // Labels: l, + // Value: d.value, + // Timestamp: d.timestamp, + // }) + // nPendingExemplars++ + case tHistogram: + pendingData[nPending].Histograms = append(pendingData[nPending].Histograms, HistogramToHistogramProto(d.timestamp, d.histogram)) + nPendingHistograms++ + case tFloatHistogram: + pendingData[nPending].Histograms = append(pendingData[nPending].Histograms, FloatHistogramToHistogramProto(d.timestamp, d.floatHistogram)) + nPendingHistograms++ + } + } + return nPendingSamples, nPendingExemplars, nPendingHistograms +} + func populateMinimizedTimeSeriesPacking(symbolTable *rwSymbolTable, batch []timeSeries, pendingData []prompb.MinimizedTimeSeriesPacking, sendExemplars, sendNativeHistograms bool) (int, int, int) { var nPendingSamples, nPendingExemplars, nPendingHistograms int for nPending, d := range batch { @@ -2163,6 +2229,52 @@ func buildMinimizedWriteRequestFixed32(samples []prompb.MinimizedTimeSeriesFixed return compressed, highest, nil } +func buildMinimizedWriteRequestBytes(samples []prompb.MinimizedTimeSeriesBytes, labels string, pBuf *proto.Buffer, buf *[]byte) ([]byte, int64, error) { + var highest int64 + for _, ts := range samples { + // At the moment we only ever append a TimeSeries with a single sample or exemplar in it. + if len(ts.Samples) > 0 && ts.Samples[0].Timestamp > highest { + highest = ts.Samples[0].Timestamp + } + if len(ts.Exemplars) > 0 && ts.Exemplars[0].Timestamp > highest { + highest = ts.Exemplars[0].Timestamp + } + if len(ts.Histograms) > 0 && ts.Histograms[0].Timestamp > highest { + highest = ts.Histograms[0].Timestamp + } + } + + req := &prompb.MinimizedWriteRequestBytes{ + Symbols: labels, + Timeseries: samples, + } + + if pBuf == nil { + pBuf = proto.NewBuffer(nil) // For convenience in tests. Not efficient. + } else { + pBuf.Reset() + } + err := pBuf.Marshal(req) + if err != nil { + return nil, 0, err + } + + // snappy uses len() to see if it needs to allocate a new slice. Make the + // buffer as long as possible. + if buf != nil { + *buf = (*buf)[0:cap(*buf)] + } else { + buf = &[]byte{} + } + + compressed := snappy.Encode(*buf, pBuf.Bytes()) + if n := snappy.MaxEncodedLen(len(pBuf.Bytes())); buf != nil && n > len(*buf) { + // grow the buffer for the next time + *buf = make([]byte, n) + } + return compressed, highest, nil +} + func buildMinimizedWriteRequestPacking(samples []prompb.MinimizedTimeSeriesPacking, labels string, pBuf *proto.Buffer, buf *[]byte) ([]byte, int64, error) { var highest int64 for _, ts := range samples { diff --git a/storage/remote/queue_manager_test.go b/storage/remote/queue_manager_test.go index 3bdce191b..b7c1b7748 100644 --- a/storage/remote/queue_manager_test.go +++ b/storage/remote/queue_manager_test.go @@ -69,23 +69,23 @@ func TestSampleDelivery(t *testing.T) { floatHistograms bool rwFormat RemoteWriteFormat }{ - //{samples: true, exemplars: false, histograms: false, floatHistograms: false, name: "samples only"}, - //{samples: true, exemplars: true, histograms: true, floatHistograms: true, name: "samples, exemplars, and histograms"}, - //{samples: false, exemplars: true, histograms: false, floatHistograms: false, name: "exemplars only"}, - //{samples: false, exemplars: false, histograms: true, floatHistograms: false, name: "histograms only"}, - //{samples: false, exemplars: false, histograms: false, floatHistograms: true, name: "float histograms only"}, + {samples: true, exemplars: false, histograms: false, floatHistograms: false, name: "samples only"}, + {samples: true, exemplars: true, histograms: true, floatHistograms: true, name: "samples, exemplars, and histograms"}, + {samples: false, exemplars: true, histograms: false, floatHistograms: false, name: "exemplars only"}, + {samples: false, exemplars: false, histograms: true, floatHistograms: false, name: "histograms only"}, + {samples: false, exemplars: false, histograms: false, floatHistograms: true, name: "float histograms only"}, {rwFormat: Min32Optimized, samples: true, exemplars: false, histograms: false, name: "interned samples only"}, - //{rwFormat: Min32Optimized, samples: true, exemplars: true, histograms: true, floatHistograms: true, name: "interned samples, exemplars, and histograms"}, - //{rwFormat: Min32Optimized, samples: false, exemplars: true, histograms: false, name: "interned exemplars only"}, - //{rwFormat: Min32Optimized, samples: false, exemplars: false, histograms: true, name: "interned histograms only"}, - //{rwFormat: Min32Optimized, samples: false, exemplars: false, histograms: false, floatHistograms: true, name: "interned float histograms only"}, + {rwFormat: Min32Optimized, samples: true, exemplars: true, histograms: true, floatHistograms: true, name: "interned samples, exemplars, and histograms"}, + {rwFormat: Min32Optimized, samples: false, exemplars: true, histograms: false, name: "interned exemplars only"}, + {rwFormat: Min32Optimized, samples: false, exemplars: false, histograms: true, name: "interned histograms only"}, + {rwFormat: Min32Optimized, samples: false, exemplars: false, histograms: false, floatHistograms: true, name: "interned float histograms only"}, - //{rwFormat: Min64Fixed, samples: true, exemplars: false, histograms: false, name: "interned samples only"}, - //{rwFormat: Min64Fixed, samples: true, exemplars: true, histograms: true, floatHistograms: true, name: "interned samples, exemplars, and histograms"}, - //{rwFormat: Min64Fixed, samples: false, exemplars: true, histograms: false, name: "interned exemplars only"}, - //{rwFormat: Min64Fixed, samples: false, exemplars: false, histograms: true, name: "interned histograms only"}, - //{rwFormat: Min64Fixed, samples: false, exemplars: false, histograms: false, floatHistograms: true, name: "interned float histograms only"}, + {rwFormat: Min64Fixed, samples: true, exemplars: false, histograms: false, name: "interned samples only"}, + {rwFormat: Min64Fixed, samples: true, exemplars: true, histograms: true, floatHistograms: true, name: "interned samples, exemplars, and histograms"}, + {rwFormat: Min64Fixed, samples: false, exemplars: true, histograms: false, name: "interned exemplars only"}, + {rwFormat: Min64Fixed, samples: false, exemplars: false, histograms: true, name: "interned histograms only"}, + {rwFormat: Min64Fixed, samples: false, exemplars: false, histograms: false, floatHistograms: true, name: "interned float histograms only"}, } // Let's create an even number of send batches so we don't run into the @@ -116,26 +116,26 @@ func TestSampleDelivery(t *testing.T) { defer s.Close() var ( - series []record.RefSeries - samples []record.RefSample - //exemplars []record.RefExemplar - //histograms []record.RefHistogramSample - //floatHistograms []record.RefFloatHistogramSample + series []record.RefSeries + samples []record.RefSample + exemplars []record.RefExemplar + histograms []record.RefHistogramSample + floatHistograms []record.RefFloatHistogramSample ) // Generates same series in both cases. if tc.samples { samples, series = createTimeseries(n, n) } - //if tc.exemplars { - // exemplars, series = createExemplars(n, n) - //} - //if tc.histograms { - // histograms, _, series = createHistograms(n, n, false) - //} - //if tc.floatHistograms { - // _, floatHistograms, series = createHistograms(n, n, true) - //} + if tc.exemplars { + exemplars, series = createExemplars(n, n) + } + if tc.histograms { + histograms, _, series = createHistograms(n, n, false) + } + if tc.floatHistograms { + _, floatHistograms, series = createHistograms(n, n, true) + } // Apply new config. queueConfig.Capacity = len(samples) @@ -152,24 +152,24 @@ func TestSampleDelivery(t *testing.T) { // Send first half of data. c.expectSamples(samples[:len(samples)/2], series) - //c.expectExemplars(exemplars[:len(exemplars)/2], series) - //c.expectHistograms(histograms[:len(histograms)/2], series) - //c.expectFloatHistograms(floatHistograms[:len(floatHistograms)/2], series) + c.expectExemplars(exemplars[:len(exemplars)/2], series) + c.expectHistograms(histograms[:len(histograms)/2], series) + c.expectFloatHistograms(floatHistograms[:len(floatHistograms)/2], series) qm.Append(samples[:len(samples)/2]) - //qm.AppendExemplars(exemplars[:len(exemplars)/2]) - //qm.AppendHistograms(histograms[:len(histograms)/2]) - //qm.AppendFloatHistograms(floatHistograms[:len(floatHistograms)/2]) + qm.AppendExemplars(exemplars[:len(exemplars)/2]) + qm.AppendHistograms(histograms[:len(histograms)/2]) + qm.AppendFloatHistograms(floatHistograms[:len(floatHistograms)/2]) c.waitForExpectedData(t) // Send second half of data. c.expectSamples(samples[len(samples)/2:], series) - //c.expectExemplars(exemplars[len(exemplars)/2:], series) - //c.expectHistograms(histograms[len(histograms)/2:], series) - //c.expectFloatHistograms(floatHistograms[len(floatHistograms)/2:], series) + c.expectExemplars(exemplars[len(exemplars)/2:], series) + c.expectHistograms(histograms[len(histograms)/2:], series) + c.expectFloatHistograms(floatHistograms[len(floatHistograms)/2:], series) qm.Append(samples[len(samples)/2:]) - //qm.AppendExemplars(exemplars[len(exemplars)/2:]) - //qm.AppendHistograms(histograms[len(histograms)/2:]) - //qm.AppendFloatHistograms(floatHistograms[len(floatHistograms)/2:]) + qm.AppendExemplars(exemplars[len(exemplars)/2:]) + qm.AppendHistograms(histograms[len(histograms)/2:]) + qm.AppendFloatHistograms(floatHistograms[len(floatHistograms)/2:]) c.waitForExpectedData(t) }) } diff --git a/storage/remote/write_handler.go b/storage/remote/write_handler.go index 0f74f7b48..e230f3ba3 100644 --- a/storage/remote/write_handler.go +++ b/storage/remote/write_handler.go @@ -76,6 +76,7 @@ func (h *writeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { var reqMin *prompb.MinimizedWriteRequest var reqMin64Fixed *prompb.MinimizedWriteRequestFixed64 var reqMin32Fixed *prompb.MinimizedWriteRequestFixed32 + var reqMinBytes *prompb.MinimizedWriteRequestBytes // TODO: this should eventually be done via content negotiation/looking at the header switch h.rwFormat { @@ -87,6 +88,8 @@ func (h *writeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { reqMin64Fixed, err = DecodeMinimizedWriteRequestFixed64(r.Body) case Min32Fixed: reqMin32Fixed, err = DecodeMinimizedWriteRequestFixed32(r.Body) + case MinBytes: + reqMinBytes, err = DecodeMinimizedWriteRequestBytes(r.Body) } if err != nil { @@ -105,6 +108,8 @@ func (h *writeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { err = h.writeMin64(r.Context(), reqMin64Fixed) case Min32Fixed: err = h.writeMin32(r.Context(), reqMin32Fixed) + case MinBytes: + err = h.writeMinBytes(r.Context(), reqMinBytes) } switch { @@ -415,3 +420,42 @@ func (h *writeHandler) writeMin32(ctx context.Context, req *prompb.MinimizedWrit return nil } + +func (h *writeHandler) writeMinBytes(ctx context.Context, req *prompb.MinimizedWriteRequestBytes) (err error) { + outOfOrderExemplarErrs := 0 + + app := h.appendable.Appender(ctx) + defer func() { + if err != nil { + _ = app.Rollback() + return + } + err = app.Commit() + }() + + for _, ts := range req.Timeseries { + ls := ByteSliceToLabels(req.Symbols, ts.LabelSymbols) + + err := h.appendSamples(app, ts.Samples, ls) + if err != nil { + return err + } + + for _, ep := range ts.Exemplars { + e := exemplarProtoToExemplar(ep) + //e := exemplarRefProtoToExemplar(req.StringSymbolTable, ep) + h.appendExemplar(app, e, ls, &outOfOrderExemplarErrs) + } + + err = h.appendHistograms(app, ts.Histograms, ls) + if err != nil { + return err + } + } + + if outOfOrderExemplarErrs > 0 { + _ = level.Warn(h.logger).Log("msg", "Error on ingesting out-of-order exemplars", "num_dropped", outOfOrderExemplarErrs) + } + + return nil +}