mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2024-12-26 06:04:05 -08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

storage: Evict unused chunk.Descs in crash recovery

Browse source 
This is in line with the v1.5 change in paradigm to not keep
chunk.Descs without chunks around after a series maintenance.

It's mainly motivated by avoiding excessive amounts of RAM usage
during crash recovery.

The code avoids to create memory time series with zero chunk.Descs as
that is prone to trigger weird effects. (Series maintenance would
archive series with zero chunk.Descs, but we cannot do that here
because the archive indices still have to be checked.)
This commit is contained in:
beorn7 2017-04-03 17:03:59 +02:00
parent eac36d123e
commit 08fc6cbd39
2 changed files with 11 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
storage/local/crashrecovery.go
View file

@ -304,6 +304,8 @@ func (p *persistence) sanitizeSeries(
}
s.persistWatermark = len(cds)
s.modTime = modTime
// Finally, evict again all chunk.Descs except the latest one to save memory.
s.evictChunkDescs(len(cds) - 1)
return fp, true
}
// This is the tricky one: We have chunks from heads.db, but
@ -356,6 +358,8 @@ func (p *persistence) sanitizeSeries(
atomic.AddInt64(&chunk.NumMemChunks, int64(-len(s.chunkDescs)))
s.chunkDescs = cds
s.headChunkClosed = true
// Finally, evict again all chunk.Descs except the latest one to save memory.
s.evictChunkDescs(len(cds) - 1)
return fp, true
}
log.Warnf(
@ -364,11 +368,15 @@ func (p *persistence) sanitizeSeries(
)
chunk.NumMemDescs.Sub(float64(keepIdx))
atomic.AddInt64(&chunk.NumMemChunks, int64(-keepIdx))
chunkDescsToEvict := len(cds)
if keepIdx == len(s.chunkDescs) {
// No chunks from series file left, head chunk is evicted, so declare it closed.
s.headChunkClosed = true
chunkDescsToEvict-- // Keep one chunk.Desc in this case to avoid a series with zero chunk.Descs.
}
s.chunkDescs = append(cds, s.chunkDescs[keepIdx:]...)
// Finally, evict again chunk.Descs without chunk to save memory.
s.evictChunkDescs(chunkDescsToEvict)
return fp, true
}
// This series is supposed to be archived.
@ -458,6 +466,8 @@ func (p *persistence) cleanUpArchiveIndexes(
return err
}
fpToSeries[model.Fingerprint(fp)] = series
// Evict all but one chunk.Desc to save memory.
series.evictChunkDescs(len(cds) - 1)
return nil
}); err != nil {
return err

2
storage/local/storage.go
View file

@ -1587,7 +1587,7 @@ func (s *MemorySeriesStorage) maintainMemorySeries(
}
return
}
// If we are here, the series is not archived, so check for Chunk.Desc
// If we are here, the series is not archived, so check for chunk.Desc
// eviction next.
series.evictChunkDescs(iOldestNotEvicted)