mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2025-03-05 20:59:13 -08:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

wal: avoid heap allocation in WAL reader

Browse source 
The buffers we allocated were escaping to the heap, resulting in large
memory usage spikes during startup and checkpointing in Prometheus.
This attaches the buffer to the reader object to prevent this.

Signed-off-by: Fabian Reinartz <freinartz@google.com>
This commit is contained in:
Fabian Reinartz 2018-05-24 15:51:47 -04:00
parent 7841d417b3
commit d951140ab8
1 changed files with 15 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
wal/wal.go
View file

@ -621,14 +621,6 @@ func NewSegmentsRangeReader(dir string, m, n int) (io.ReadCloser, error) {
return newSegmentBufReader(segs...), nil return newSegmentBufReader(segs...), nil
} }
// Reader reads WAL records from an io.Reader.
type Reader struct {
rdr io.Reader
err error
rec []byte
total int // total bytes processed.
}
// segmentBufReader is a buffered reader that reads in multiples of pages. // segmentBufReader is a buffered reader that reads in multiples of pages.
// The main purpose is that we are able to track segment and offset for // The main purpose is that we are able to track segment and offset for
// corruption reporting. // corruption reporting.
@ -683,6 +675,15 @@ func (r *segmentBufReader) Read(b []byte) (n int, err error) {
return n, nil return n, nil
} }
// Reader reads WAL records from an io.Reader.
type Reader struct {
rdr io.Reader
err error
rec []byte
buf [pageSize]byte
total int // total bytes processed.
}
// NewReader returns a new reader. // NewReader returns a new reader.
func NewReader(r io.Reader) *Reader { func NewReader(r io.Reader) *Reader {
return &Reader{rdr: r} return &Reader{rdr: r}
@ -700,8 +701,11 @@ func (r *Reader) Next() bool {
} }
func (r *Reader) next() (err error) { func (r *Reader) next() (err error) {
var hdr [recordHeaderSize]byte // We have to use r.buf since allocating byte arrays here fails escape
var buf [pageSize]byte // analysis and ends up on the heap, even though it seemingly should not.
hdr := r.buf[:7]
buf := r.buf[7:]
r.rec = r.rec[:0] r.rec = r.rec[:0]
i := 0 i := 0
@ -745,7 +749,7 @@ func (r *Reader) next() (err error) {
crc = binary.BigEndian.Uint32(hdr[3:]) crc = binary.BigEndian.Uint32(hdr[3:])
) )
if length > pageSize { if length > pageSize-recordHeaderSize {
return errors.Errorf("invalid record size %d", length) return errors.Errorf("invalid record size %d", length)
} }
n, err = io.ReadFull(r.rdr, buf[:length]) n, err = io.ReadFull(r.rdr, buf[:length])