From 32324f79c746a733f8c76bc1eace3a664b8ea5c5 Mon Sep 17 00:00:00 2001 From: Fabian Reinartz Date: Tue, 5 Jun 2018 05:33:04 -0400 Subject: [PATCH] vendour: update prometheus/tsdb Signed-off-by: Fabian Reinartz --- vendor/github.com/prometheus/tsdb/README.md | 2 ++ vendor/github.com/prometheus/tsdb/block.go | 15 ++++----- vendor/github.com/prometheus/tsdb/db.go | 34 ++++++++++++++------- vendor/vendor.json | 26 ++++++++-------- 4 files changed, 46 insertions(+), 31 deletions(-) diff --git a/vendor/github.com/prometheus/tsdb/README.md b/vendor/github.com/prometheus/tsdb/README.md index 2d72fcf47..737020c4d 100644 --- a/vendor/github.com/prometheus/tsdb/README.md +++ b/vendor/github.com/prometheus/tsdb/README.md @@ -7,4 +7,6 @@ This repository contains the Prometheus storage layer that is used in its 2.x re A writeup of its design can be found [here](https://fabxc.org/blog/2017-04-10-writing-a-tsdb/). +Video: [Storing 16 Bytes at Scale](https://youtu.be/b_pEevMAC3I) from [PromCon 2017](https://promcon.io/2017-munich/). + See also the [format documentation](docs/format/README.md). diff --git a/vendor/github.com/prometheus/tsdb/block.go b/vendor/github.com/prometheus/tsdb/block.go index e3760df76..9ae2adbde 100644 --- a/vendor/github.com/prometheus/tsdb/block.go +++ b/vendor/github.com/prometheus/tsdb/block.go @@ -468,9 +468,9 @@ Outer: return writeMetaFile(pb.dir, &pb.meta) } -// CleanTombstones will rewrite the block if there any tombstones to remove them -// and returns if there was a re-write. -func (pb *Block) CleanTombstones(dest string, c Compactor) (bool, error) { +// CleanTombstones will remove the tombstones and rewrite the block (only if there are any tombstones). +// If there was a rewrite, then it returns the ULID of the new block written, else nil. +func (pb *Block) CleanTombstones(dest string, c Compactor) (*ulid.ULID, error) { numStones := 0 pb.tombstones.Iter(func(id uint64, ivs Intervals) error { @@ -480,14 +480,15 @@ func (pb *Block) CleanTombstones(dest string, c Compactor) (bool, error) { }) if numStones == 0 { - return false, nil + return nil, nil } - if _, err := c.Write(dest, pb, pb.meta.MinTime, pb.meta.MaxTime); err != nil { - return false, err + uid, err := c.Write(dest, pb, pb.meta.MinTime, pb.meta.MaxTime) + if err != nil { + return nil, err } - return true, nil + return &uid, nil } // Snapshot creates snapshot of the block into dir. diff --git a/vendor/github.com/prometheus/tsdb/db.go b/vendor/github.com/prometheus/tsdb/db.go index db9564375..28cb14f33 100644 --- a/vendor/github.com/prometheus/tsdb/db.go +++ b/vendor/github.com/prometheus/tsdb/db.go @@ -835,34 +835,46 @@ func (db *DB) Delete(mint, maxt int64, ms ...labels.Matcher) error { } // CleanTombstones re-writes any blocks with tombstones. -func (db *DB) CleanTombstones() error { +func (db *DB) CleanTombstones() (err error) { db.cmtx.Lock() defer db.cmtx.Unlock() start := time.Now() defer db.metrics.tombCleanTimer.Observe(time.Since(start).Seconds()) + newUIDs := []ulid.ULID{} + defer func() { + // If any error is caused, we need to delete all the new directory created. + if err != nil { + for _, uid := range newUIDs { + dir := filepath.Join(db.Dir(), uid.String()) + if err := os.RemoveAll(dir); err != nil { + level.Error(db.logger).Log("msg", "failed to delete block after failed `CleanTombstones`", "dir", dir, "err", err) + } + } + } + }() + db.mtx.RLock() blocks := db.blocks[:] db.mtx.RUnlock() - deleted := []string{} + deletable := []string{} for _, b := range blocks { - ok, err := b.CleanTombstones(db.Dir(), db.compactor) - if err != nil { - return errors.Wrapf(err, "clean tombstones: %s", b.Dir()) - } - - if ok { - deleted = append(deleted, b.Dir()) + if uid, er := b.CleanTombstones(db.Dir(), db.compactor); er != nil { + err = errors.Wrapf(er, "clean tombstones: %s", b.Dir()) + return err + } else if uid != nil { // New block was created. + deletable = append(deletable, b.Dir()) + newUIDs = append(newUIDs, *uid) } } - if len(deleted) == 0 { + if len(deletable) == 0 { return nil } - return errors.Wrap(db.reload(deleted...), "reload blocks") + return errors.Wrap(db.reload(deletable...), "reload blocks") } func intervalOverlap(amin, amax, bmin, bmax int64) bool { diff --git a/vendor/vendor.json b/vendor/vendor.json index 75e095d00..1c5c47c51 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -820,40 +820,40 @@ "revisionTime": "2016-04-11T19:08:41Z" }, { - "checksumSHA1": "e7QdIY1+bs+75qObh1MLVdGNLvE=", + "checksumSHA1": "eohOTRwnox/+qrSrgYmnxeJB2yM=", "path": "github.com/prometheus/tsdb", - "revision": "ae33d7873d94cec6b3da3ffbf903b3b35a2db9ee", - "revisionTime": "2018-05-29T19:14:13Z" + "revision": "c848349f07c83bd38d5d19faa5ea71c7fd8923ea", + "revisionTime": "2018-06-05T09:24:13Z" }, { "checksumSHA1": "QI0UME2olSr4kH6Z8UkpffM59Mc=", "path": "github.com/prometheus/tsdb/chunkenc", - "revision": "ae33d7873d94cec6b3da3ffbf903b3b35a2db9ee", - "revisionTime": "2018-05-29T19:14:13Z" + "revision": "c848349f07c83bd38d5d19faa5ea71c7fd8923ea", + "revisionTime": "2018-06-05T09:24:13Z" }, { "checksumSHA1": "746Mjy2y6wdsGjY/FcGhc8tI4w8=", "path": "github.com/prometheus/tsdb/chunks", - "revision": "ae33d7873d94cec6b3da3ffbf903b3b35a2db9ee", - "revisionTime": "2018-05-29T19:14:13Z" + "revision": "c848349f07c83bd38d5d19faa5ea71c7fd8923ea", + "revisionTime": "2018-06-05T09:24:13Z" }, { "checksumSHA1": "dnyelqeik/xHDRCvCmKFv/Op9XQ=", "path": "github.com/prometheus/tsdb/fileutil", - "revision": "ae33d7873d94cec6b3da3ffbf903b3b35a2db9ee", - "revisionTime": "2018-05-29T19:14:13Z" + "revision": "c848349f07c83bd38d5d19faa5ea71c7fd8923ea", + "revisionTime": "2018-06-05T09:24:13Z" }, { "checksumSHA1": "A2uIFwIgeHmXGBzOpna95kM80RY=", "path": "github.com/prometheus/tsdb/index", - "revision": "ae33d7873d94cec6b3da3ffbf903b3b35a2db9ee", - "revisionTime": "2018-05-29T19:14:13Z" + "revision": "c848349f07c83bd38d5d19faa5ea71c7fd8923ea", + "revisionTime": "2018-06-05T09:24:13Z" }, { "checksumSHA1": "Va8HWvOFTwFeewZFadMAOzNGDps=", "path": "github.com/prometheus/tsdb/labels", - "revision": "ae33d7873d94cec6b3da3ffbf903b3b35a2db9ee", - "revisionTime": "2018-05-29T19:14:13Z" + "revision": "c848349f07c83bd38d5d19faa5ea71c7fd8923ea", + "revisionTime": "2018-06-05T09:24:13Z" }, { "checksumSHA1": "5SYLEhADhdBVZAGPVHWggQl7H8k=",