clean up after running repair tests (#372)

implement a `CopyDirs` helper to walk all dirs and subdirs and copy all files within to a destination folder.

Refactor `TestRepairBadIndexVersion` to use the `CopyDirs` so it works on a copy db and remove the copy after the test.
This allows running the test more than once and doesn't confuse git.

Signed-off-by: Callum Stytan <callumstyan@gmail.com>

fileutil/fileutil.go

@@ -5,11 +5,80 @@
 package fileutil
 
 import (
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sort"
+	"strings"
 )
 
+// CopyDirs copies all directories, subdirectories and files recursively including the empty folders.
+// Source and destination must be full paths.
+func CopyDirs(src, dest string) error {
+	if err := os.MkdirAll(dest, 0777); err != nil {
+		return err
+	}
+	files, err := readDirs(src)
+	if err != nil {
+		return err
+	}
+
+	for _, f := range files {
+		dp := filepath.Join(dest, f)
+		sp := filepath.Join(src, f)
+
+		stat, err := os.Stat(sp)
+		if err != nil {
+			return err
+		}
+
+		// Empty directories are also created.
+		if stat.IsDir() {
+			if err := os.MkdirAll(dp, 0777); err != nil {
+				return err
+			}
+			continue
+		}
+
+		if err := copyFile(sp, dp); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func copyFile(src, dest string) error {
+	data, err := ioutil.ReadFile(src)
+	if err != nil {
+		return err
+	}
+
+	err = ioutil.WriteFile(dest, data, 0644)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// readDirs reads the source directory recursively and
+// returns relative paths to all files and empty directories.
+func readDirs(src string) ([]string, error) {
+	var files []string
+	var err error
+
+	err = filepath.Walk(src, func(path string, f os.FileInfo, err error) error {
+		relativePath := strings.TrimPrefix(path, src)
+		if len(relativePath) > 0 {
+			files = append(files, relativePath)
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return files, nil
+}
+
 // ReadDir returns the filenames in the given directory in sorted order.
 func ReadDir(dirpath string) ([]string, error) {
 	dir, err := os.Open(dirpath)

repair_test.go

@@ -5,10 +5,10 @@ import (
 	"testing"
 
 	"github.com/prometheus/tsdb/chunks"
-	"github.com/prometheus/tsdb/testutil"
+	"github.com/prometheus/tsdb/fileutil"
-
 	"github.com/prometheus/tsdb/index"
 	"github.com/prometheus/tsdb/labels"
+	"github.com/prometheus/tsdb/testutil"
 )
 
 func TestRepairBadIndexVersion(t *testing.T) {
@@ -45,15 +45,20 @@ func TestRepairBadIndexVersion(t *testing.T) {
 	// 		panic(err)
 	// 	}
 	// }
+	const dbDir = "testdata/repair_index_version/01BZJ9WJQPWHGNC2W4J9TA62KC"
+	tmpDir := "testdata/repair_index_version/copy"
+	tmpDbDir := tmpDir + "/3MCNSQ8S31EHGJYWK5E1GPJWJZ"
 
+	// Check the current db.
 	// In its current state, lookups should fail with the fixed code.
-	const dir = "testdata/repair_index_version/01BZJ9WJQPWHGNC2W4J9TA62KC/"
+	meta, err := readMetaFile(dbDir)
-	meta, err := readMetaFile(dir)
 	testutil.NotOk(t, err)
-	// Touch chunks dir in block.
-	os.MkdirAll(dir+"chunks", 0777)
 
-	r, err := index.NewFileReader(dir + "index")
+	// Touch chunks dir in block.
+	os.MkdirAll(dbDir+"/chunks", 0777)
+	defer os.RemoveAll(dbDir + "/chunks")
+
+	r, err := index.NewFileReader(dbDir + "/index")
 	testutil.Ok(t, err)
 	p, err := r.Postings("b", "1")
 	testutil.Ok(t, err)
@@ -66,12 +71,17 @@ func TestRepairBadIndexVersion(t *testing.T) {
 	testutil.Ok(t, p.Err())
 	testutil.Ok(t, r.Close())
 
+	// Create a copy DB to run test against.
+	if err = fileutil.CopyDirs(dbDir, tmpDbDir); err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpDir)
 	// On DB opening all blocks in the base dir should be repaired.
-	db, err := Open("testdata/repair_index_version", nil, nil, nil)
+	db, err := Open(tmpDir, nil, nil, nil)
 	testutil.Ok(t, err)
 	db.Close()
 
-	r, err = index.NewFileReader(dir + "index")
+	r, err = index.NewFileReader(tmpDbDir + "/index")
 	testutil.Ok(t, err)
 	p, err = r.Postings("b", "1")
 	testutil.Ok(t, err)
@@ -92,7 +102,7 @@ func TestRepairBadIndexVersion(t *testing.T) {
 		{{"a", "2"}, {"b", "1"}},
 	}, res)
 
-	meta, err = readMetaFile(dir)
+	meta, err = readMetaFile(tmpDbDir)
 	testutil.Ok(t, err)
 	testutil.Assert(t, meta.Version == 1, "unexpected meta version %d", meta.Version)
 }