From 1291ec71851a7383de30b089f456fdb6202d037a Mon Sep 17 00:00:00 2001 From: Howie Date: Thu, 24 Mar 2022 18:44:14 +0800 Subject: [PATCH] deleting *.tmp WAL files on startup (#10317) * fix issue #10245 Signed-off-by: lihaowei * minor changes Signed-off-by: lihaowei * review changes Signed-off-by: lihaowei * minor changes Signed-off-by: lihaowei --- tsdb/db.go | 20 ++++++++++++++------ tsdb/db_test.go | 5 +++++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/tsdb/db.go b/tsdb/db.go index 93639146a7..d65c0246bf 100644 --- a/tsdb/db.go +++ b/tsdb/db.go @@ -639,9 +639,11 @@ func open(dir string, l log.Logger, r prometheus.Registerer, opts *Options, rngs if err := MigrateWAL(l, walDir); err != nil { return nil, errors.Wrap(err, "migrate WAL") } - // Remove garbage, tmp blocks. - if err := removeBestEffortTmpDirs(l, dir); err != nil { - return nil, errors.Wrap(err, "remove tmp dirs") + for _, tmpDir := range []string{walDir, dir} { + // Remove tmp dirs. + if err := removeBestEffortTmpDirs(l, tmpDir); err != nil { + return nil, errors.Wrap(err, "remove tmp dirs") + } } db := &DB{ @@ -760,11 +762,14 @@ func open(dir string, l log.Logger, r prometheus.Registerer, opts *Options, rngs func removeBestEffortTmpDirs(l log.Logger, dir string) error { files, err := ioutil.ReadDir(dir) + if os.IsNotExist(err) { + return nil + } if err != nil { return err } for _, fi := range files { - if isTmpBlockDir(fi) { + if isTmpDir(fi) { if err := os.RemoveAll(filepath.Join(dir, fi.Name())); err != nil { level.Error(l).Log("msg", "failed to delete tmp block dir", "dir", filepath.Join(dir, fi.Name()), "err", err) continue @@ -1720,8 +1725,8 @@ func isBlockDir(fi os.FileInfo) bool { return err == nil } -// isTmpBlockDir returns dir that consists of block dir ULID and tmp extension. -func isTmpBlockDir(fi os.FileInfo) bool { +// isTmpDir returns true if the given file-info contains a block ULID or checkpoint prefix and a tmp extension. +func isTmpDir(fi os.FileInfo) bool { if !fi.IsDir() { return false } @@ -1729,6 +1734,9 @@ func isTmpBlockDir(fi os.FileInfo) bool { fn := fi.Name() ext := filepath.Ext(fn) if ext == tmpForDeletionBlockDirSuffix || ext == tmpForCreationBlockDirSuffix || ext == tmpLegacy { + if strings.HasPrefix(fn, "checkpoint.") { + return true + } if _, err := ulid.ParseStrict(fn[:len(fn)-len(ext)]); err == nil { return true } diff --git a/tsdb/db_test.go b/tsdb/db_test.go index 66ff861281..ea3ae4d678 100644 --- a/tsdb/db_test.go +++ b/tsdb/db_test.go @@ -2974,6 +2974,9 @@ func TestOpen_VariousBlockStates(t *testing.T) { _, err = writeMetaFile(log.NewLogfmtLogger(os.Stderr), dir, m) require.NoError(t, err) } + tmpCheckpointDir := path.Join(tmpDir, "wal/checkpoint.00000001.tmp") + err := os.MkdirAll(tmpCheckpointDir, 0o777) + require.NoError(t, err) opts := DefaultOptions() opts.RetentionDuration = 0 @@ -3005,6 +3008,8 @@ func TestOpen_VariousBlockStates(t *testing.T) { } } require.Equal(t, len(expectedIgnoredDirs), ignored) + _, err = os.Stat(tmpCheckpointDir) + require.True(t, os.IsNotExist(err)) } func TestOneCheckpointPerCompactCall(t *testing.T) {