mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2025-03-05 20:59:13 -08:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Optimize zstd decompression memory usage

Browse source
This commit is contained in:
Nicolás Pazos 2023-11-06 17:14:03 -03:00 committed by Callum Styan
parent 1abce2acfc
commit 6f957dba95
1 changed files with 21 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
storage/remote/compression.go
View file

@ -166,6 +166,7 @@ func (s *s2Compression) Decompress(data []byte) ([]byte, error) {
type zstdCompression struct {
level zstd.EncoderLevel
buf bytes.Buffer
rawBuf []byte
r *reZstd.Decoder
w *reZstd.Encoder
}
@ -178,7 +179,8 @@ func (z *zstdCompression) Compress(data []byte) ([]byte, error) {
return nil, err
}
}
// NOTE: from my observations EncodeAll takes a bit less CPU but considerably more memory.
// Taking this decision deliberately.
z.buf.Reset()
z.w.Reset(&z.buf)
_, err = z.w.Write(data)
@ -200,17 +202,20 @@ func (z *zstdCompression) Decompress(data []byte) ([]byte, error) {
return nil, err
}
}
err = z.r.Reset(bytes.NewReader(data))
if err != nil {
return nil, err
}
z.buf.Reset()
_, err = io.Copy(&z.buf, z.r)
if err != nil {
return nil, err
}
return z.buf.Bytes(), nil
// NOTE: interestingly, I'm seeing much better memory usage using DecodeAll, for the same CPU
z.rawBuf = z.rawBuf[:0]
z.rawBuf, err = z.r.DecodeAll(data, z.rawBuf)
return z.rawBuf, err
// err = z.r.Reset(bytes.NewReader(data))
// if err != nil {
// return nil, err
// }
// z.buf.Reset()
// _, err = io.Copy(&z.buf, z.r)
// if err != nil {
// return nil, err
// }
// return z.buf.Bytes(), nil
}
type lzwCompression struct {