Compute WAL size and use it during retention size checks (#5886)

* Compute WAL size and account for it when applying the retention settings.

Signed-off-by: Dipack P Panjabi <dpanjabi@hudson-trading.com>

tsdb/block_test.go

@@ -17,6 +17,8 @@ import (
 	"context"
 	"encoding/binary"
 
+	"github.com/prometheus/prometheus/tsdb/fileutil"
+
 	"errors"
 	"io/ioutil"
 	"math/rand"
@@ -175,7 +177,8 @@ func TestBlockSize(t *testing.T) {
 			testutil.Ok(t, blockInit.Close())
 		}()
 		expSizeInit = blockInit.Size()
-		actSizeInit := testutil.DirSize(t, blockInit.Dir())
+		actSizeInit, err := fileutil.DirSize(blockInit.Dir())
+		testutil.Ok(t, err)
 		testutil.Equals(t, expSizeInit, actSizeInit)
 	}
 
@@ -184,7 +187,7 @@ func TestBlockSize(t *testing.T) {
 		testutil.Ok(t, blockInit.Delete(1, 10, labels.NewMustRegexpMatcher("", ".*")))
 		expAfterDelete := blockInit.Size()
 		testutil.Assert(t, expAfterDelete > expSizeInit, "after a delete the block size should be bigger as the tombstone file should grow %v > %v", expAfterDelete, expSizeInit)
-		actAfterDelete := testutil.DirSize(t, blockDirInit)
+		actAfterDelete, err := fileutil.DirSize(blockDirInit)
 		testutil.Ok(t, err)
 		testutil.Equals(t, expAfterDelete, actAfterDelete, "after a delete reported block size doesn't match actual disk size")
 
@@ -198,7 +201,8 @@ func TestBlockSize(t *testing.T) {
 			testutil.Ok(t, blockAfterCompact.Close())
 		}()
 		expAfterCompact := blockAfterCompact.Size()
-		actAfterCompact := testutil.DirSize(t, blockAfterCompact.Dir())
+		actAfterCompact, err := fileutil.DirSize(blockAfterCompact.Dir())
+		testutil.Ok(t, err)
 		testutil.Assert(t, actAfterDelete > actAfterCompact, "after a delete and compaction the block size should be smaller %v,%v", actAfterDelete, actAfterCompact)
 		testutil.Equals(t, expAfterCompact, actAfterCompact, "after a delete and compaction reported block size doesn't match actual disk size")
 	}

tsdb/db.go

@@ -933,7 +933,11 @@ func (db *DB) beyondSizeRetention(blocks []*Block) (deleteable map[ulid.ULID]*Bl
 	}
 
 	deleteable = make(map[ulid.ULID]*Block)
-	blocksSize := int64(0)
+
+	walSize, _ := db.Head().wal.Size()
+	// Initializing size counter with WAL size,
+	// as that is part of the retention strategy.
+	blocksSize := walSize
 	for i, block := range blocks {
 		blocksSize += block.Size()
 		if blocksSize > db.opts.MaxBytes {

tsdb/db_test.go

@@ -25,6 +25,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/prometheus/prometheus/tsdb/fileutil"
+
 	"github.com/go-kit/kit/log"
 	"github.com/oklog/ulid"
 	"github.com/pkg/errors"
@@ -1111,11 +1113,49 @@ func TestSizeRetention(t *testing.T) {
 		createBlock(t, db.Dir(), genSeries(100, 10, m.MinTime, m.MaxTime))
 	}
 
+	headBlocks := []*BlockMeta{
+		{MinTime: 700, MaxTime: 800},
+	}
+
+	// Add some data to the WAL.
+	headApp := db.Head().Appender()
+	for _, m := range headBlocks {
+		series := genSeries(100, 10, m.MinTime, m.MaxTime)
+		for _, s := range series {
+			it := s.Iterator()
+			for it.Next() {
+				tim, v := it.At()
+				_, err := headApp.Add(s.Labels(), tim, v)
+				testutil.Ok(t, err)
+			}
+			testutil.Ok(t, it.Err())
+		}
+	}
+	testutil.Ok(t, headApp.Commit())
+
 	// Test that registered size matches the actual disk size.
 	testutil.Ok(t, db.reload())                                         // Reload the db to register the new db size.
 	testutil.Equals(t, len(blocks), len(db.Blocks()))                   // Ensure all blocks are registered.
-	expSize := int64(prom_testutil.ToFloat64(db.metrics.blocksBytes)) // Use the actual internal metrics.
-	actSize := testutil.DirSize(t, db.Dir())
+	blockSize := int64(prom_testutil.ToFloat64(db.metrics.blocksBytes)) // Use the the actual internal metrics.
+	walSize, err := db.Head().wal.Size()
+	testutil.Ok(t, err)
+	// Expected size should take into account block size + WAL size
+	expSize := blockSize + walSize
+	actSize, err := fileutil.DirSize(db.Dir())
+	testutil.Ok(t, err)
+	testutil.Equals(t, expSize, actSize, "registered size doesn't match actual disk size")
+
+	// Create a WAL checkpoint, and compare sizes.
+	first, last, err := db.Head().wal.Segments()
+	testutil.Ok(t, err)
+	_, err = wal.Checkpoint(db.Head().wal, first, last-1, func(x uint64) bool { return false }, 0)
+	testutil.Ok(t, err)
+	blockSize = int64(prom_testutil.ToFloat64(db.metrics.blocksBytes)) // Use the the actual internal metrics.
+	walSize, err = db.Head().wal.Size()
+	testutil.Ok(t, err)
+	expSize = blockSize + walSize
+	actSize, err = fileutil.DirSize(db.Dir())
+	testutil.Ok(t, err)
 	testutil.Equals(t, expSize, actSize, "registered size doesn't match actual disk size")
 
 	// Decrease the max bytes limit so that a delete is triggered.
@@ -1127,9 +1167,14 @@ func TestSizeRetention(t *testing.T) {
 
 	expBlocks := blocks[1:]
 	actBlocks := db.Blocks()
-	expSize = int64(prom_testutil.ToFloat64(db.metrics.blocksBytes))
+	blockSize = int64(prom_testutil.ToFloat64(db.metrics.blocksBytes))
+	walSize, err = db.Head().wal.Size()
+	testutil.Ok(t, err)
+	// Expected size should take into account block size + WAL size
+	expSize = blockSize + walSize
 	actRetentCount := int(prom_testutil.ToFloat64(db.metrics.sizeRetentionCount))
-	actSize = testutil.DirSize(t, db.Dir())
+	actSize, err = fileutil.DirSize(db.Dir())
+	testutil.Ok(t, err)
 
 	testutil.Equals(t, 1, actRetentCount, "metric retention count mismatch")
 	testutil.Equals(t, actSize, expSize, "metric db size doesn't match actual disk size")
@@ -1137,7 +1182,6 @@ func TestSizeRetention(t *testing.T) {
 	testutil.Equals(t, len(blocks)-1, len(actBlocks), "new block count should be decreased from:%v to:%v", len(blocks), len(blocks)-1)
 	testutil.Equals(t, expBlocks[0].MaxTime, actBlocks[0].meta.MaxTime, "maxT mismatch of the first block")
 	testutil.Equals(t, expBlocks[len(expBlocks)-1].MaxTime, actBlocks[len(actBlocks)-1].meta.MaxTime, "maxT mismatch of the last block")
-
 }
 
 func TestSizeRetentionMetric(t *testing.T) {
@@ -2298,7 +2342,8 @@ func TestDBReadOnly(t *testing.T) {
 		testutil.Ok(t, err)
 		dbWritable.DisableCompactions()
 
-		dbSizeBeforeAppend := testutil.DirSize(t, dbWritable.Dir())
+		dbSizeBeforeAppend, err := fileutil.DirSize(dbWritable.Dir())
+		testutil.Ok(t, err)
 		app := dbWritable.Appender()
 		_, err = app.Add(labels.FromStrings("foo", "bar"), dbWritable.Head().MaxTime()+1, 0)
 		testutil.Ok(t, err)
@@ -2306,7 +2351,8 @@ func TestDBReadOnly(t *testing.T) {
 		expSeriesCount++
 
 		expBlocks = dbWritable.Blocks()
-		expDbSize := testutil.DirSize(t, dbWritable.Dir())
+		expDbSize, err := fileutil.DirSize(dbWritable.Dir())
+		testutil.Ok(t, err)
 		testutil.Assert(t, expDbSize > dbSizeBeforeAppend, "db size didn't increase after an append")
 
 		q, err := dbWritable.Querier(math.MinInt64, math.MaxInt64)

tsdb/fileutil/dir.go

@@ -0,0 +1,33 @@
+// Copyright 2019 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fileutil
+
+import (
+	"os"
+	"path/filepath"
+)
+
+func DirSize(dir string) (int64, error) {
+	var size int64
+	err := filepath.Walk(dir, func(filePath string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			size += info.Size()
+		}
+		return nil
+	})
+	return size, err
+}

tsdb/wal/wal.go

@@ -876,3 +876,9 @@ func (r *segmentBufReader) Read(b []byte) (n int, err error) {
 	r.buf.Reset(r.segs[r.cur])
 	return n, nil
 }
+
+// Computing size of the WAL.
+// We do this by adding the sizes of all the files under the WAL dir.
+func (w *WAL) Size() (int64, error) {
+	return fileutil.DirSize(w.Dir())
+}

tsdb/wal/wal_test.go

@@ -23,6 +23,8 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/prometheus/prometheus/tsdb/fileutil"
+
 	client_testutil "github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/prometheus/prometheus/util/testutil"
 )
@@ -408,8 +410,10 @@ func TestCompression(t *testing.T) {
 		testutil.Ok(t, os.RemoveAll(dirUnCompressed))
 	}()
 
-	uncompressedSize := testutil.DirSize(t, dirUnCompressed)
-	compressedSize := testutil.DirSize(t, dirCompressed)
+	uncompressedSize, err := fileutil.DirSize(dirUnCompressed)
+	testutil.Ok(t, err)
+	compressedSize, err := fileutil.DirSize(dirCompressed)
+	testutil.Ok(t, err)
 
 	testutil.Assert(t, float64(uncompressedSize)*0.75 > float64(compressedSize), "Compressing zeroes should save at least 25%% space - uncompressedSize: %d, compressedSize: %d", uncompressedSize, compressedSize)
 }

util/testutil/directory.go

@@ -133,20 +133,6 @@ func NewTemporaryDirectory(name string, t T) (handler TemporaryDirectory) {
 	return
 }
 
-// DirSize returns the size in bytes of all files in a directory.
-func DirSize(t *testing.T, path string) int64 {
-	var size int64
-	err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
-		Ok(t, err)
-		if !info.IsDir() {
-			size += info.Size()
-		}
-		return nil
-	})
-	Ok(t, err)
-	return size
-}
-
 // DirHash returns a hash of all files attribites and their content within a directory.
 func DirHash(t *testing.T, path string) []byte {
 	hash := sha256.New()