storage: Evict unused chunk.Descs in crash recovery

This is in line with the v1.5 change in paradigm to not keep
chunk.Descs without chunks around after a series maintenance.

It's mainly motivated by avoiding excessive amounts of RAM usage
during crash recovery.

The code avoids to create memory time series with zero chunk.Descs as
that is prone to trigger weird effects. (Series maintenance would
archive series with zero chunk.Descs, but we cannot do that here
because the archive indices still have to be checked.)

storage/local/crashrecovery.go

@@ -304,6 +304,8 @@ func (p *persistence) sanitizeSeries(
 			}
 			s.persistWatermark = len(cds)
 			s.modTime = modTime
+			// Finally, evict again all chunk.Descs except the latest one to save memory.
+			s.evictChunkDescs(len(cds) - 1)
 			return fp, true
 		}
 		// This is the tricky one: We have chunks from heads.db, but
@@ -356,6 +358,8 @@ func (p *persistence) sanitizeSeries(
 			atomic.AddInt64(&chunk.NumMemChunks, int64(-len(s.chunkDescs)))
 			s.chunkDescs = cds
 			s.headChunkClosed = true
+			// Finally, evict again all chunk.Descs except the latest one to save memory.
+			s.evictChunkDescs(len(cds) - 1)
 			return fp, true
 		}
 		log.Warnf(
@@ -364,11 +368,15 @@ func (p *persistence) sanitizeSeries(
 		)
 		chunk.NumMemDescs.Sub(float64(keepIdx))
 		atomic.AddInt64(&chunk.NumMemChunks, int64(-keepIdx))
+		chunkDescsToEvict := len(cds)
 		if keepIdx == len(s.chunkDescs) {
 			// No chunks from series file left, head chunk is evicted, so declare it closed.
 			s.headChunkClosed = true
+			chunkDescsToEvict-- // Keep one chunk.Desc in this case to avoid a series with zero chunk.Descs.
 		}
 		s.chunkDescs = append(cds, s.chunkDescs[keepIdx:]...)
+		// Finally, evict again chunk.Descs without chunk to save memory.
+		s.evictChunkDescs(chunkDescsToEvict)
 		return fp, true
 	}
 	// This series is supposed to be archived.
@@ -458,6 +466,8 @@ func (p *persistence) cleanUpArchiveIndexes(
 			return err
 		}
 		fpToSeries[model.Fingerprint(fp)] = series
+		// Evict all but one chunk.Desc to save memory.
+		series.evictChunkDescs(len(cds) - 1)
 		return nil
 	}); err != nil {
 		return err

storage/local/storage.go

@@ -1587,7 +1587,7 @@ func (s *MemorySeriesStorage) maintainMemorySeries(
 		}
 		return
 	}
-	// If we are here, the series is not archived, so check for Chunk.Desc
+	// If we are here, the series is not archived, so check for chunk.Desc
 	// eviction next.
 	series.evictChunkDescs(iOldestNotEvicted)