TSDB: refactor cleanup of chunks and series

Extract the middle of the loop into a function, so it will be
easier to modify the `seriesHashmap` data structure.

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>

tsdb/head.go

@@ -1777,15 +1777,12 @@ func (s *stripeSeries) gc(mint int64, minOOOMmapRef chunks.ChunkDiskMapperRef) (
 		deletedFromPrevStripe       = 0
 	)
 	minMmapFile = math.MaxInt32
-	// Run through all series and truncate old chunks. Mark those with no
-	// chunks left as deleted and store their ID.
-	for i := 0; i < s.size; i++ {
-		deletedForCallback := make(map[chunks.HeadSeriesRef]labels.Labels, deletedFromPrevStripe)
-		s.locks[i].Lock()
 
-		for hash, all := range s.hashes[i] {
+	// For one series, truncate old chunks and check if any chunks left. If not, mark as deleted and collect the ID.
-			for _, series := range all {
+	check := func(i int, hash uint64, series *memSeries, deletedForCallback map[chunks.HeadSeriesRef]labels.Labels) {
 		series.Lock()
+		defer series.Unlock()
+
 		rmChunks += series.truncateChunksBefore(mint, minOOOMmapRef)
 
 		if len(series.mmappedChunks) > 0 {
@@ -1816,10 +1813,8 @@ func (s *stripeSeries) gc(mint int64, minOOOMmapRef chunks.ChunkDiskMapperRef) (
 			if seriesMint < actualMint {
 				actualMint = seriesMint
 			}
-					series.Unlock()
+			return
-					continue
 		}
-
 		// The series is gone entirely. We need to keep the series lock
 		// and make sure we have acquired the stripe locks for hash and ID of the
 		// series alike.
@@ -1839,8 +1834,16 @@ func (s *stripeSeries) gc(mint int64, minOOOMmapRef chunks.ChunkDiskMapperRef) (
 		if i != j {
 			s.locks[j].Unlock()
 		}
+	}
 
-				series.Unlock()
+	// Run through all series shard by shard, checking which should be deleted.
+	for i := 0; i < s.size; i++ {
+		deletedForCallback := make(map[chunks.HeadSeriesRef]labels.Labels, deletedFromPrevStripe)
+		s.locks[i].Lock()
+
+		for hash, all := range s.hashes[i] {
+			for _, series := range all {
+				check(i, hash, series, deletedForCallback)
 			}
 		}