2017-04-10 11:59:45 -07:00
|
|
|
// Copyright 2017 The Prometheus Authors
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
2016-11-15 01:34:25 -08:00
|
|
|
// Package tsdb implements a time series storage for float64 sample data.
|
|
|
|
package tsdb
|
|
|
|
|
|
|
|
import (
|
2016-12-10 09:08:50 -08:00
|
|
|
"bytes"
|
2016-12-04 04:16:11 -08:00
|
|
|
"fmt"
|
2017-02-27 01:46:15 -08:00
|
|
|
"io"
|
2017-01-06 00:26:39 -08:00
|
|
|
"io/ioutil"
|
2018-04-05 05:51:33 -07:00
|
|
|
"math"
|
2016-12-04 04:16:11 -08:00
|
|
|
"os"
|
2016-12-08 08:43:10 -08:00
|
|
|
"path/filepath"
|
2017-09-08 06:09:24 -07:00
|
|
|
"runtime"
|
2017-05-18 07:09:30 -07:00
|
|
|
"sort"
|
2016-12-14 23:31:26 -08:00
|
|
|
"strconv"
|
2018-04-05 05:51:33 -07:00
|
|
|
"strings"
|
2016-12-08 08:43:10 -08:00
|
|
|
"sync"
|
2017-01-06 06:18:06 -08:00
|
|
|
"time"
|
2016-12-14 23:31:26 -08:00
|
|
|
|
2016-12-14 09:38:46 -08:00
|
|
|
"github.com/go-kit/kit/log"
|
2017-09-28 00:19:34 -07:00
|
|
|
"github.com/go-kit/kit/log/level"
|
2017-05-18 07:09:30 -07:00
|
|
|
"github.com/oklog/ulid"
|
2017-01-03 06:43:26 -08:00
|
|
|
"github.com/pkg/errors"
|
2016-12-31 00:48:49 -08:00
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
2017-11-30 06:34:49 -08:00
|
|
|
"github.com/prometheus/tsdb/chunkenc"
|
2017-10-09 06:21:46 -07:00
|
|
|
"github.com/prometheus/tsdb/fileutil"
|
2017-04-04 02:27:26 -07:00
|
|
|
"github.com/prometheus/tsdb/labels"
|
2018-04-05 05:51:33 -07:00
|
|
|
"golang.org/x/sync/errgroup"
|
2016-11-15 01:34:25 -08:00
|
|
|
)
|
|
|
|
|
2016-12-09 01:00:14 -08:00
|
|
|
// DefaultOptions used for the DB. They are sane for setups using
|
2018-03-02 03:12:32 -08:00
|
|
|
// millisecond precision timestamps.
|
2016-11-15 01:34:25 -08:00
|
|
|
var DefaultOptions = &Options{
|
2017-02-09 17:54:26 -08:00
|
|
|
WALFlushInterval: 5 * time.Second,
|
|
|
|
RetentionDuration: 15 * 24 * 60 * 60 * 1000, // 15 days in milliseconds
|
2017-07-18 02:17:17 -07:00
|
|
|
BlockRanges: ExponentialBlockRanges(int64(2*time.Hour)/1e6, 3, 5),
|
2017-05-09 03:52:47 -07:00
|
|
|
NoLockfile: false,
|
2016-11-15 01:34:25 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
// Options of the DB storage.
|
|
|
|
type Options struct {
|
2017-11-12 21:10:23 -08:00
|
|
|
// The interval at which the write ahead log is flushed to disk.
|
2017-01-06 06:18:06 -08:00
|
|
|
WALFlushInterval time.Duration
|
2017-01-28 23:11:47 -08:00
|
|
|
|
2017-02-09 17:54:26 -08:00
|
|
|
// Duration of persisted data to keep.
|
|
|
|
RetentionDuration uint64
|
|
|
|
|
2017-07-07 04:46:41 -07:00
|
|
|
// The sizes of the Blocks.
|
|
|
|
BlockRanges []int64
|
2017-01-28 23:11:47 -08:00
|
|
|
|
2017-05-09 03:52:47 -07:00
|
|
|
// NoLockfile disables creation and consideration of a lock file.
|
|
|
|
NoLockfile bool
|
2016-11-15 01:34:25 -08:00
|
|
|
}
|
|
|
|
|
2017-01-12 11:17:49 -08:00
|
|
|
// Appender allows appending a batch of data. It must be completed with a
|
|
|
|
// call to Commit or Rollback and must not be reused afterwards.
|
2017-04-14 13:37:28 -07:00
|
|
|
//
|
|
|
|
// Operations on the Appender interface are not goroutine-safe.
|
2016-12-10 09:08:50 -08:00
|
|
|
type Appender interface {
|
2017-02-01 06:29:48 -08:00
|
|
|
// Add adds a sample pair for the given series. A reference number is
|
|
|
|
// returned which can be used to add further samples in the same or later
|
|
|
|
// transactions.
|
|
|
|
// Returned reference numbers are ephemeral and may be rejected in calls
|
|
|
|
// to AddFast() at any point. Adding the sample via Add() returns a new
|
|
|
|
// reference number.
|
2018-06-26 11:15:58 -07:00
|
|
|
// If the reference is 0 it must not be used for caching.
|
2017-09-05 02:45:18 -07:00
|
|
|
Add(l labels.Labels, t int64, v float64) (uint64, error)
|
2017-02-01 06:29:48 -08:00
|
|
|
|
|
|
|
// Add adds a sample pair for the referenced series. It is generally faster
|
|
|
|
// than adding a sample by providing its full label set.
|
2017-09-05 02:45:18 -07:00
|
|
|
AddFast(ref uint64, t int64, v float64) error
|
2016-12-20 15:02:37 -08:00
|
|
|
|
|
|
|
// Commit submits the collected samples and purges the batch.
|
2016-12-10 09:08:50 -08:00
|
|
|
Commit() error
|
2017-01-12 11:17:49 -08:00
|
|
|
|
|
|
|
// Rollback rolls back all modifications made in the appender so far.
|
|
|
|
Rollback() error
|
2016-12-10 09:08:50 -08:00
|
|
|
}
|
|
|
|
|
2017-01-06 02:40:09 -08:00
|
|
|
// DB handles reads and writes of time series falling into
|
|
|
|
// a hashed partition of a seriedb.
|
|
|
|
type DB struct {
|
2017-03-04 07:50:48 -08:00
|
|
|
dir string
|
2018-05-29 11:35:48 -07:00
|
|
|
lockf fileutil.Releaser
|
2017-03-04 07:50:48 -08:00
|
|
|
|
2017-08-30 09:34:54 -07:00
|
|
|
logger log.Logger
|
|
|
|
metrics *dbMetrics
|
|
|
|
opts *Options
|
2017-11-30 06:34:49 -08:00
|
|
|
chunkPool chunkenc.Pool
|
2017-08-30 09:34:54 -07:00
|
|
|
compactor Compactor
|
2016-12-09 01:00:14 -08:00
|
|
|
|
2017-07-14 00:00:22 -07:00
|
|
|
// Mutex for that must be held when modifying the general block layout.
|
2017-03-20 00:41:56 -07:00
|
|
|
mtx sync.RWMutex
|
2017-10-09 06:21:46 -07:00
|
|
|
blocks []*Block
|
2017-03-04 07:50:48 -08:00
|
|
|
|
2017-08-28 15:39:17 -07:00
|
|
|
head *Head
|
2017-01-06 03:37:28 -08:00
|
|
|
|
|
|
|
compactc chan struct{}
|
|
|
|
donec chan struct{}
|
|
|
|
stopc chan struct{}
|
2017-05-20 00:51:10 -07:00
|
|
|
|
2017-05-23 05:43:30 -07:00
|
|
|
// cmtx is used to control compactions and deletions.
|
2017-07-14 01:06:07 -07:00
|
|
|
cmtx sync.Mutex
|
|
|
|
compactionsEnabled bool
|
2016-12-09 01:00:14 -08:00
|
|
|
}
|
|
|
|
|
2017-01-06 02:40:09 -08:00
|
|
|
type dbMetrics struct {
|
2017-08-30 09:34:54 -07:00
|
|
|
loadedBlocks prometheus.GaugeFunc
|
|
|
|
reloads prometheus.Counter
|
|
|
|
reloadsFailed prometheus.Counter
|
2017-01-06 03:37:28 -08:00
|
|
|
compactionsTriggered prometheus.Counter
|
2018-01-10 02:36:56 -08:00
|
|
|
cutoffs prometheus.Counter
|
|
|
|
cutoffsFailed prometheus.Counter
|
2017-11-22 04:34:50 -08:00
|
|
|
tombCleanTimer prometheus.Histogram
|
2016-12-31 00:48:49 -08:00
|
|
|
}
|
|
|
|
|
2017-05-26 06:13:03 -07:00
|
|
|
func newDBMetrics(db *DB, r prometheus.Registerer) *dbMetrics {
|
2017-01-06 02:40:09 -08:00
|
|
|
m := &dbMetrics{}
|
2017-01-03 06:43:26 -08:00
|
|
|
|
2017-05-26 06:13:03 -07:00
|
|
|
m.loadedBlocks = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
2017-10-20 03:32:32 -07:00
|
|
|
Name: "prometheus_tsdb_blocks_loaded",
|
2017-05-26 06:13:03 -07:00
|
|
|
Help: "Number of currently loaded data blocks",
|
|
|
|
}, func() float64 {
|
|
|
|
db.mtx.RLock()
|
|
|
|
defer db.mtx.RUnlock()
|
|
|
|
return float64(len(db.blocks))
|
|
|
|
})
|
|
|
|
m.reloads = prometheus.NewCounter(prometheus.CounterOpts{
|
2017-10-20 03:32:32 -07:00
|
|
|
Name: "prometheus_tsdb_reloads_total",
|
2017-05-26 06:13:03 -07:00
|
|
|
Help: "Number of times the database reloaded block data from disk.",
|
|
|
|
})
|
|
|
|
m.reloadsFailed = prometheus.NewCounter(prometheus.CounterOpts{
|
2017-10-20 03:32:32 -07:00
|
|
|
Name: "prometheus_tsdb_reloads_failures_total",
|
2017-11-09 05:25:27 -08:00
|
|
|
Help: "Number of times the database failed to reload block data from disk.",
|
2017-05-26 06:13:03 -07:00
|
|
|
})
|
2017-01-06 03:37:28 -08:00
|
|
|
m.compactionsTriggered = prometheus.NewCounter(prometheus.CounterOpts{
|
2017-10-20 03:32:32 -07:00
|
|
|
Name: "prometheus_tsdb_compactions_triggered_total",
|
2017-01-06 03:37:28 -08:00
|
|
|
Help: "Total number of triggered compactions for the partition.",
|
|
|
|
})
|
2018-01-10 02:36:56 -08:00
|
|
|
m.cutoffs = prometheus.NewCounter(prometheus.CounterOpts{
|
|
|
|
Name: "prometheus_tsdb_retention_cutoffs_total",
|
|
|
|
Help: "Number of times the database cut off block data from disk.",
|
|
|
|
})
|
|
|
|
m.cutoffsFailed = prometheus.NewCounter(prometheus.CounterOpts{
|
|
|
|
Name: "prometheus_tsdb_retention_cutoffs_failures_total",
|
|
|
|
Help: "Number of times the database failed to cut off block data from disk.",
|
|
|
|
})
|
2017-11-22 04:34:50 -08:00
|
|
|
m.tombCleanTimer = prometheus.NewHistogram(prometheus.HistogramOpts{
|
|
|
|
Name: "prometheus_tsdb_tombstone_cleanup_seconds",
|
|
|
|
Help: "The time taken to recompact blocks to remove tombstones.",
|
|
|
|
})
|
2016-12-31 00:48:49 -08:00
|
|
|
|
|
|
|
if r != nil {
|
|
|
|
r.MustRegister(
|
2017-05-26 06:13:03 -07:00
|
|
|
m.loadedBlocks,
|
|
|
|
m.reloads,
|
|
|
|
m.reloadsFailed,
|
2018-01-10 02:36:56 -08:00
|
|
|
m.cutoffs,
|
|
|
|
m.cutoffsFailed,
|
2017-01-09 10:14:21 -08:00
|
|
|
m.compactionsTriggered,
|
2017-11-22 04:34:50 -08:00
|
|
|
m.tombCleanTimer,
|
2016-12-31 00:48:49 -08:00
|
|
|
)
|
|
|
|
}
|
|
|
|
return m
|
|
|
|
}
|
|
|
|
|
2017-01-06 02:40:09 -08:00
|
|
|
// Open returns a new DB in the given directory.
|
2017-02-27 22:17:01 -08:00
|
|
|
func Open(dir string, l log.Logger, r prometheus.Registerer, opts *Options) (db *DB, err error) {
|
2017-02-19 04:01:19 -08:00
|
|
|
if err := os.MkdirAll(dir, 0777); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2017-02-19 07:04:37 -08:00
|
|
|
if l == nil {
|
2017-09-13 01:17:20 -07:00
|
|
|
l = log.NewNopLogger()
|
2017-02-19 07:04:37 -08:00
|
|
|
}
|
2017-01-17 21:18:32 -08:00
|
|
|
if opts == nil {
|
|
|
|
opts = DefaultOptions
|
|
|
|
}
|
2018-02-12 02:40:12 -08:00
|
|
|
// Fixup bad format written by Prometheus 2.1.
|
2018-02-09 04:37:10 -08:00
|
|
|
if err := repairBadIndexVersion(l, dir); err != nil {
|
2018-02-09 04:11:03 -08:00
|
|
|
return nil, err
|
|
|
|
}
|
2017-01-17 21:18:32 -08:00
|
|
|
|
2017-01-06 03:37:28 -08:00
|
|
|
db = &DB{
|
2017-07-14 01:06:07 -07:00
|
|
|
dir: dir,
|
|
|
|
logger: l,
|
|
|
|
opts: opts,
|
|
|
|
compactc: make(chan struct{}, 1),
|
|
|
|
donec: make(chan struct{}),
|
|
|
|
stopc: make(chan struct{}),
|
|
|
|
compactionsEnabled: true,
|
2017-11-30 06:34:49 -08:00
|
|
|
chunkPool: chunkenc.NewPool(),
|
2017-01-06 00:26:39 -08:00
|
|
|
}
|
2017-05-26 06:13:03 -07:00
|
|
|
db.metrics = newDBMetrics(db, r)
|
|
|
|
|
2017-05-09 03:52:47 -07:00
|
|
|
if !opts.NoLockfile {
|
2017-05-18 07:09:30 -07:00
|
|
|
absdir, err := filepath.Abs(dir)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2018-05-29 11:35:48 -07:00
|
|
|
lockf, _, err := fileutil.Flock(filepath.Join(absdir, "lock"))
|
2017-05-09 03:52:47 -07:00
|
|
|
if err != nil {
|
2018-05-29 11:35:48 -07:00
|
|
|
return nil, errors.Wrap(err, "lock DB directory")
|
2017-05-09 03:52:47 -07:00
|
|
|
}
|
2018-05-29 11:35:48 -07:00
|
|
|
db.lockf = lockf
|
2017-05-09 03:52:47 -07:00
|
|
|
}
|
|
|
|
|
2017-09-01 02:46:46 -07:00
|
|
|
db.compactor, err = NewLeveledCompactor(r, l, opts.BlockRanges, db.chunkPool)
|
|
|
|
if err != nil {
|
|
|
|
return nil, errors.Wrap(err, "create leveled compactor")
|
2017-07-07 04:46:41 -07:00
|
|
|
}
|
|
|
|
|
2017-10-04 12:51:34 -07:00
|
|
|
wal, err := OpenSegmentWAL(filepath.Join(dir, "wal"), l, opts.WALFlushInterval, r)
|
2017-08-28 15:39:17 -07:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2017-09-01 02:46:46 -07:00
|
|
|
db.head, err = NewHead(r, l, wal, opts.BlockRanges[0])
|
2017-08-30 09:34:54 -07:00
|
|
|
if err != nil {
|
2017-08-30 08:38:25 -07:00
|
|
|
return nil, err
|
|
|
|
}
|
2017-08-30 09:34:54 -07:00
|
|
|
if err := db.reload(); err != nil {
|
2016-12-14 23:31:26 -08:00
|
|
|
return nil, err
|
|
|
|
}
|
2017-09-06 07:20:37 -07:00
|
|
|
if err := db.head.ReadWAL(); err != nil {
|
|
|
|
return nil, errors.Wrap(err, "read WAL")
|
|
|
|
}
|
2017-08-28 15:39:17 -07:00
|
|
|
|
2017-01-06 03:37:28 -08:00
|
|
|
go db.run()
|
|
|
|
|
|
|
|
return db, nil
|
|
|
|
}
|
|
|
|
|
2017-06-08 03:14:13 -07:00
|
|
|
// Dir returns the directory of the database.
|
|
|
|
func (db *DB) Dir() string {
|
|
|
|
return db.dir
|
|
|
|
}
|
|
|
|
|
2017-01-06 03:37:28 -08:00
|
|
|
func (db *DB) run() {
|
|
|
|
defer close(db.donec)
|
|
|
|
|
2017-08-28 15:39:17 -07:00
|
|
|
backoff := time.Duration(0)
|
2017-02-28 06:08:52 -08:00
|
|
|
|
2017-01-19 22:58:19 -08:00
|
|
|
for {
|
|
|
|
select {
|
2017-08-28 15:39:17 -07:00
|
|
|
case <-db.stopc:
|
2017-08-30 09:34:54 -07:00
|
|
|
return
|
2017-08-28 15:39:17 -07:00
|
|
|
case <-time.After(backoff):
|
|
|
|
}
|
|
|
|
|
|
|
|
select {
|
|
|
|
case <-time.After(1 * time.Minute):
|
2017-02-28 06:08:52 -08:00
|
|
|
select {
|
|
|
|
case db.compactc <- struct{}{}:
|
|
|
|
default:
|
|
|
|
}
|
2017-01-06 03:37:28 -08:00
|
|
|
case <-db.compactc:
|
|
|
|
db.metrics.compactionsTriggered.Inc()
|
|
|
|
|
2018-06-27 06:47:11 -07:00
|
|
|
_, err := db.compact()
|
|
|
|
if err != nil {
|
|
|
|
level.Error(db.logger).Log("msg", "compaction failed", "err", err)
|
2017-08-30 09:34:54 -07:00
|
|
|
backoff = exponential(backoff, 1*time.Second, 1*time.Minute)
|
|
|
|
} else {
|
|
|
|
backoff = 0
|
2017-01-06 03:37:28 -08:00
|
|
|
}
|
2017-01-06 06:18:06 -08:00
|
|
|
|
2017-01-06 03:37:28 -08:00
|
|
|
case <-db.stopc:
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-06-27 06:47:11 -07:00
|
|
|
func (db *DB) beyondRetention(meta *BlockMeta) bool {
|
2017-03-17 07:30:05 -07:00
|
|
|
if db.opts.RetentionDuration == 0 {
|
2018-06-27 06:47:11 -07:00
|
|
|
return false
|
2017-03-17 07:30:05 -07:00
|
|
|
}
|
2017-03-17 07:56:19 -07:00
|
|
|
|
|
|
|
db.mtx.RLock()
|
2017-11-23 05:27:10 -08:00
|
|
|
blocks := db.blocks[:]
|
|
|
|
db.mtx.RUnlock()
|
2017-03-17 07:56:19 -07:00
|
|
|
|
2017-11-23 05:27:10 -08:00
|
|
|
if len(blocks) == 0 {
|
2018-06-27 06:47:11 -07:00
|
|
|
return false
|
2017-03-17 07:30:05 -07:00
|
|
|
}
|
|
|
|
|
2017-11-23 05:27:10 -08:00
|
|
|
last := blocks[len(db.blocks)-1]
|
2017-03-17 07:30:05 -07:00
|
|
|
mint := last.Meta().MaxTime - int64(db.opts.RetentionDuration)
|
2018-01-26 06:03:59 -08:00
|
|
|
|
2018-06-27 06:47:11 -07:00
|
|
|
return meta.MaxTime < mint
|
2017-03-17 07:30:05 -07:00
|
|
|
}
|
|
|
|
|
2017-08-30 09:34:54 -07:00
|
|
|
// Appender opens a new appender against the database.
|
|
|
|
func (db *DB) Appender() Appender {
|
|
|
|
return dbAppender{db: db, Appender: db.head.Appender()}
|
|
|
|
}
|
|
|
|
|
|
|
|
// dbAppender wraps the DB's head appender and triggers compactions on commit
|
|
|
|
// if necessary.
|
|
|
|
type dbAppender struct {
|
|
|
|
Appender
|
|
|
|
db *DB
|
|
|
|
}
|
|
|
|
|
|
|
|
func (a dbAppender) Commit() error {
|
|
|
|
err := a.Appender.Commit()
|
|
|
|
|
2017-09-04 06:07:30 -07:00
|
|
|
// We could just run this check every few minutes practically. But for benchmarks
|
|
|
|
// and high frequency use cases this is the safer way.
|
2017-08-30 09:34:54 -07:00
|
|
|
if a.db.head.MaxTime()-a.db.head.MinTime() > a.db.head.chunkRange/2*3 {
|
|
|
|
select {
|
|
|
|
case a.db.compactc <- struct{}{}:
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2018-06-27 09:05:21 -07:00
|
|
|
// Compact data if possible. After successful compaction blocks are reloaded
|
|
|
|
// which will also trigger blocks to be deleted that fall out of the retention
|
|
|
|
// window.
|
|
|
|
// If no blocks are compacted, the retention window state doesn't change. Thus,
|
|
|
|
// this is sufficient to reliably delete old data.
|
|
|
|
// Old blocks are only deleted on reload based on the new block's parent information.
|
|
|
|
// See DB.reload documentation for further information.
|
2017-07-13 07:15:13 -07:00
|
|
|
func (db *DB) compact() (changes bool, err error) {
|
|
|
|
db.cmtx.Lock()
|
|
|
|
defer db.cmtx.Unlock()
|
2017-01-17 21:18:32 -08:00
|
|
|
|
2017-07-14 01:06:07 -07:00
|
|
|
if !db.compactionsEnabled {
|
|
|
|
return false, nil
|
|
|
|
}
|
|
|
|
|
2017-07-13 07:15:13 -07:00
|
|
|
// Check whether we have pending head blocks that are ready to be persisted.
|
|
|
|
// They have the highest priority.
|
2017-08-28 15:39:17 -07:00
|
|
|
for {
|
2017-03-04 07:50:48 -08:00
|
|
|
select {
|
|
|
|
case <-db.stopc:
|
2017-03-20 02:41:43 -07:00
|
|
|
return changes, nil
|
2017-03-04 07:50:48 -08:00
|
|
|
default:
|
2017-02-02 00:32:06 -08:00
|
|
|
}
|
2017-08-28 15:39:17 -07:00
|
|
|
// The head has a compactable range if 1.5 level 0 ranges are between the oldest
|
|
|
|
// and newest timestamp. The 0.5 acts as a buffer of the appendable window.
|
|
|
|
if db.head.MaxTime()-db.head.MinTime() <= db.opts.BlockRanges[0]/2*3 {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
mint, maxt := rangeForTimestamp(db.head.MinTime(), db.opts.BlockRanges[0])
|
2017-02-02 00:32:06 -08:00
|
|
|
|
2017-08-28 15:39:17 -07:00
|
|
|
// Wrap head into a range that bounds all reads to it.
|
|
|
|
head := &rangeHead{
|
|
|
|
head: db.head,
|
|
|
|
mint: mint,
|
|
|
|
maxt: maxt,
|
|
|
|
}
|
2018-06-27 06:47:11 -07:00
|
|
|
if _, err = db.compactor.Write(db.dir, head, mint, maxt, nil); err != nil {
|
2017-03-04 07:50:48 -08:00
|
|
|
return changes, errors.Wrap(err, "persist head block")
|
|
|
|
}
|
|
|
|
changes = true
|
2017-08-09 02:10:29 -07:00
|
|
|
|
2017-09-08 06:09:24 -07:00
|
|
|
runtime.GC()
|
|
|
|
|
2017-08-30 09:34:54 -07:00
|
|
|
if err := db.reload(); err != nil {
|
2017-08-28 15:39:17 -07:00
|
|
|
return changes, errors.Wrap(err, "reload blocks")
|
2017-08-09 02:10:29 -07:00
|
|
|
}
|
2017-09-08 06:09:24 -07:00
|
|
|
runtime.GC()
|
2017-03-04 07:50:48 -08:00
|
|
|
}
|
2017-01-06 03:37:28 -08:00
|
|
|
|
2017-03-02 00:13:29 -08:00
|
|
|
// Check for compactions of multiple blocks.
|
|
|
|
for {
|
2017-08-09 02:10:29 -07:00
|
|
|
plan, err := db.compactor.Plan(db.dir)
|
2017-03-02 00:13:29 -08:00
|
|
|
if err != nil {
|
2017-03-04 07:50:48 -08:00
|
|
|
return changes, errors.Wrap(err, "plan compaction")
|
2017-03-02 00:13:29 -08:00
|
|
|
}
|
2017-08-09 02:10:29 -07:00
|
|
|
if len(plan) == 0 {
|
2017-03-21 04:21:02 -07:00
|
|
|
break
|
|
|
|
}
|
2017-01-06 03:37:28 -08:00
|
|
|
|
2017-03-02 00:13:29 -08:00
|
|
|
select {
|
|
|
|
case <-db.stopc:
|
2017-03-20 02:41:43 -07:00
|
|
|
return changes, nil
|
2017-03-02 00:13:29 -08:00
|
|
|
default:
|
|
|
|
}
|
2017-03-20 02:41:43 -07:00
|
|
|
|
2017-12-05 02:49:22 -08:00
|
|
|
if _, err := db.compactor.Compact(db.dir, plan...); err != nil {
|
2017-08-09 02:10:29 -07:00
|
|
|
return changes, errors.Wrapf(err, "compact %s", plan)
|
|
|
|
}
|
|
|
|
changes = true
|
2017-09-08 06:09:24 -07:00
|
|
|
runtime.GC()
|
2017-08-28 15:39:17 -07:00
|
|
|
|
2018-06-27 06:47:11 -07:00
|
|
|
if err := db.reload(); err != nil {
|
2017-08-28 15:39:17 -07:00
|
|
|
return changes, errors.Wrap(err, "reload blocks")
|
|
|
|
}
|
2017-09-08 06:09:24 -07:00
|
|
|
runtime.GC()
|
2017-02-23 01:50:22 -08:00
|
|
|
}
|
|
|
|
|
2017-03-04 07:50:48 -08:00
|
|
|
return changes, nil
|
2017-02-09 17:54:26 -08:00
|
|
|
}
|
|
|
|
|
2017-10-09 06:21:46 -07:00
|
|
|
func (db *DB) getBlock(id ulid.ULID) (*Block, bool) {
|
2017-03-20 00:41:56 -07:00
|
|
|
for _, b := range db.blocks {
|
2017-05-18 07:09:30 -07:00
|
|
|
if b.Meta().ULID == id {
|
2017-03-20 00:41:56 -07:00
|
|
|
return b, true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil, false
|
|
|
|
}
|
|
|
|
|
2017-11-03 12:34:21 -07:00
|
|
|
// reload on-disk blocks and trigger head truncation if new blocks appeared. It takes
|
|
|
|
// a list of block directories which should be deleted during reload.
|
2018-06-27 06:47:11 -07:00
|
|
|
// Blocks that are obsolete due to replacement or retention will be deleted.
|
|
|
|
func (db *DB) reload() (err error) {
|
2017-08-30 08:38:25 -07:00
|
|
|
defer func() {
|
2017-05-26 06:13:03 -07:00
|
|
|
if err != nil {
|
|
|
|
db.metrics.reloadsFailed.Inc()
|
|
|
|
}
|
|
|
|
db.metrics.reloads.Inc()
|
2017-08-30 08:38:25 -07:00
|
|
|
}()
|
2017-05-26 06:13:03 -07:00
|
|
|
|
2017-03-02 00:13:29 -08:00
|
|
|
dirs, err := blockDirs(db.dir)
|
|
|
|
if err != nil {
|
|
|
|
return errors.Wrap(err, "find blocks")
|
|
|
|
}
|
2018-06-27 06:47:11 -07:00
|
|
|
// We delete old blocks that have been superseded by new ones by gathering all parents
|
|
|
|
// from existing blocks. Those parents all have newer replacements and can be safely deleted
|
|
|
|
// after we loaded the other blocks.
|
2018-06-27 09:05:21 -07:00
|
|
|
// This makes us resilient against the process crashing towards the end of a compaction.
|
|
|
|
// Creation of a new block and deletion of its parents cannot happen atomically. By creating
|
2018-06-28 02:55:01 -07:00
|
|
|
// blocks with their parents, we can pick up the deletion where it left off during a crash.
|
2017-02-09 17:54:26 -08:00
|
|
|
var (
|
2018-06-27 06:47:11 -07:00
|
|
|
blocks []*Block
|
2018-06-27 09:05:21 -07:00
|
|
|
corrupted = map[ulid.ULID]error{}
|
2018-06-27 06:47:11 -07:00
|
|
|
opened = map[ulid.ULID]struct{}{}
|
|
|
|
deleteable = map[ulid.ULID]struct{}{}
|
2017-02-09 17:54:26 -08:00
|
|
|
)
|
2017-03-02 00:13:29 -08:00
|
|
|
for _, dir := range dirs {
|
|
|
|
meta, err := readMetaFile(dir)
|
|
|
|
if err != nil {
|
2018-06-27 06:47:11 -07:00
|
|
|
// The block was potentially in the middle of being deleted during a crash.
|
|
|
|
// Skip it since we may delete it properly further down again.
|
|
|
|
level.Warn(db.logger).Log("msg", "read meta information", "err", err, "dir", dir)
|
2018-06-27 09:05:21 -07:00
|
|
|
|
|
|
|
ulid, err2 := ulid.Parse(filepath.Base(dir))
|
|
|
|
if err2 != nil {
|
|
|
|
level.Error(db.logger).Log("msg", "not a block dir", "dir", dir)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
corrupted[ulid] = err
|
2018-06-27 06:47:11 -07:00
|
|
|
continue
|
2017-02-09 17:54:26 -08:00
|
|
|
}
|
2018-06-27 06:47:11 -07:00
|
|
|
if db.beyondRetention(meta) {
|
|
|
|
deleteable[meta.ULID] = struct{}{}
|
2017-11-03 12:34:21 -07:00
|
|
|
continue
|
|
|
|
}
|
2018-06-27 06:47:11 -07:00
|
|
|
for _, b := range meta.Compaction.Parents {
|
|
|
|
deleteable[b.ULID] = struct{}{}
|
|
|
|
}
|
|
|
|
}
|
2018-06-27 09:05:21 -07:00
|
|
|
// Blocks we failed to open should all be those we are want to delete anyway.
|
|
|
|
for c, err := range corrupted {
|
|
|
|
if _, ok := deleteable[c]; !ok {
|
|
|
|
return errors.Wrapf(err, "unexpected corrupted block %s", c)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// Load new blocks into memory.
|
2018-06-27 06:47:11 -07:00
|
|
|
for _, dir := range dirs {
|
|
|
|
meta, err := readMetaFile(dir)
|
|
|
|
if err != nil {
|
2018-06-27 09:05:21 -07:00
|
|
|
return errors.Wrapf(err, "read meta information %s", dir)
|
2018-06-27 06:47:11 -07:00
|
|
|
}
|
|
|
|
// Don't load blocks that are scheduled for deletion.
|
|
|
|
if _, ok := deleteable[meta.ULID]; ok {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
// See if we already have the block in memory or open it otherwise.
|
2017-05-18 07:09:30 -07:00
|
|
|
b, ok := db.getBlock(meta.ULID)
|
|
|
|
if !ok {
|
2017-10-09 06:21:46 -07:00
|
|
|
b, err = OpenBlock(dir, db.chunkPool)
|
2017-05-18 07:09:30 -07:00
|
|
|
if err != nil {
|
|
|
|
return errors.Wrapf(err, "open block %s", dir)
|
2017-03-02 00:13:29 -08:00
|
|
|
}
|
|
|
|
}
|
2017-03-20 00:41:56 -07:00
|
|
|
blocks = append(blocks, b)
|
2018-06-27 06:47:11 -07:00
|
|
|
opened[meta.ULID] = struct{}{}
|
2016-12-09 01:00:14 -08:00
|
|
|
}
|
2018-05-28 13:00:36 -07:00
|
|
|
sort.Slice(blocks, func(i, j int) bool {
|
|
|
|
return blocks[i].Meta().MinTime < blocks[j].Meta().MinTime
|
|
|
|
})
|
2017-01-06 00:26:39 -08:00
|
|
|
|
2017-05-18 07:09:30 -07:00
|
|
|
if err := validateBlockSequence(blocks); err != nil {
|
|
|
|
return errors.Wrap(err, "invalid block sequence")
|
|
|
|
}
|
2017-05-26 04:01:45 -07:00
|
|
|
|
2017-10-23 11:30:03 -07:00
|
|
|
// Swap in new blocks first for subsequently created readers to be seen.
|
|
|
|
// Then close previous blocks, which may block for pending readers to complete.
|
2017-08-28 15:39:17 -07:00
|
|
|
db.mtx.Lock()
|
2017-10-23 11:30:03 -07:00
|
|
|
oldBlocks := db.blocks
|
2017-03-20 00:41:56 -07:00
|
|
|
db.blocks = blocks
|
2017-08-28 15:39:17 -07:00
|
|
|
db.mtx.Unlock()
|
2017-05-18 07:09:30 -07:00
|
|
|
|
2018-06-27 06:47:11 -07:00
|
|
|
// Drop old blocks from memory.
|
2017-10-23 11:30:03 -07:00
|
|
|
for _, b := range oldBlocks {
|
2018-06-27 06:47:11 -07:00
|
|
|
if _, ok := opened[b.Meta().ULID]; ok {
|
2017-11-03 12:34:21 -07:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
if err := b.Close(); err != nil {
|
|
|
|
level.Warn(db.logger).Log("msg", "closing block failed", "err", err)
|
|
|
|
}
|
2018-06-27 06:47:11 -07:00
|
|
|
}
|
|
|
|
// Delete all obsolete blocks. None of them are opened any longer.
|
|
|
|
for ulid := range deleteable {
|
|
|
|
if err := os.RemoveAll(filepath.Join(db.dir, ulid.String())); err != nil {
|
|
|
|
return errors.Wrapf(err, "delete obsolete block %s", ulid)
|
2017-10-23 11:30:03 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-08-28 15:39:17 -07:00
|
|
|
// Garbage collect data in the head if the most recent persisted block
|
|
|
|
// covers data of its current time range.
|
|
|
|
if len(blocks) == 0 {
|
2017-08-30 09:34:54 -07:00
|
|
|
return nil
|
2017-08-28 15:39:17 -07:00
|
|
|
}
|
2017-10-23 11:30:03 -07:00
|
|
|
maxt := blocks[len(blocks)-1].Meta().MaxTime
|
2017-08-31 02:39:22 -07:00
|
|
|
|
2017-09-01 05:38:49 -07:00
|
|
|
return errors.Wrap(db.head.Truncate(maxt), "head truncate failed")
|
2017-05-18 07:09:30 -07:00
|
|
|
}
|
2017-01-03 06:43:26 -08:00
|
|
|
|
2018-04-13 08:50:15 -07:00
|
|
|
// validateBlockSequence returns error if given block meta files indicate that some blocks overlaps within sequence.
|
2017-10-09 06:21:46 -07:00
|
|
|
func validateBlockSequence(bs []*Block) error {
|
2018-03-28 10:33:41 -07:00
|
|
|
if len(bs) <= 1 {
|
2017-05-18 07:09:30 -07:00
|
|
|
return nil
|
|
|
|
}
|
2018-03-28 07:50:52 -07:00
|
|
|
|
|
|
|
var metas []BlockMeta
|
|
|
|
for _, b := range bs {
|
|
|
|
metas = append(metas, b.meta)
|
|
|
|
}
|
|
|
|
|
2018-03-28 10:33:41 -07:00
|
|
|
overlaps := OverlappingBlocks(metas)
|
2018-03-28 15:50:42 -07:00
|
|
|
if len(overlaps) > 0 {
|
2018-04-05 06:15:24 -07:00
|
|
|
return errors.Errorf("block time ranges overlap: %s", overlaps)
|
2018-03-28 07:50:52 -07:00
|
|
|
}
|
|
|
|
|
2018-03-28 15:50:42 -07:00
|
|
|
return nil
|
2018-03-28 07:50:52 -07:00
|
|
|
}
|
|
|
|
|
2018-04-05 06:15:24 -07:00
|
|
|
// TimeRange specifies minTime and maxTime range.
|
2018-03-29 04:50:46 -07:00
|
|
|
type TimeRange struct {
|
2018-04-05 05:51:33 -07:00
|
|
|
Min, Max int64
|
2018-03-29 04:50:46 -07:00
|
|
|
}
|
2018-04-05 05:51:33 -07:00
|
|
|
|
2018-04-05 06:15:24 -07:00
|
|
|
// Overlaps contains overlapping blocks aggregated by overlapping range.
|
|
|
|
type Overlaps map[TimeRange][]BlockMeta
|
|
|
|
|
|
|
|
// String returns human readable string form of overlapped blocks.
|
|
|
|
func (o Overlaps) String() string {
|
|
|
|
var res []string
|
|
|
|
for r, overlaps := range o {
|
|
|
|
var groups []string
|
|
|
|
for _, m := range overlaps {
|
|
|
|
groups = append(groups, fmt.Sprintf(
|
2018-04-05 08:53:24 -07:00
|
|
|
"<ulid: %s, mint: %d, maxt: %d, range: %s>",
|
2018-04-05 06:15:24 -07:00
|
|
|
m.ULID.String(),
|
|
|
|
m.MinTime,
|
|
|
|
m.MaxTime,
|
|
|
|
(time.Duration((m.MaxTime-m.MinTime)/1000)*time.Second).String(),
|
|
|
|
))
|
|
|
|
}
|
2018-04-05 08:01:16 -07:00
|
|
|
res = append(res, fmt.Sprintf(
|
2018-04-05 08:53:24 -07:00
|
|
|
"[mint: %d, maxt: %d, range: %s, blocks: %d]: %s",
|
2018-04-05 08:01:16 -07:00
|
|
|
r.Min, r.Max,
|
|
|
|
(time.Duration((r.Max-r.Min)/1000)*time.Second).String(),
|
|
|
|
len(overlaps),
|
2018-04-05 08:53:24 -07:00
|
|
|
strings.Join(groups, ", ")),
|
2018-04-05 08:01:16 -07:00
|
|
|
)
|
2018-04-05 06:15:24 -07:00
|
|
|
}
|
2018-04-05 08:01:16 -07:00
|
|
|
return strings.Join(res, "\n")
|
2018-04-05 06:15:24 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
// OverlappingBlocks returns all overlapping blocks from given meta files.
|
|
|
|
func OverlappingBlocks(bm []BlockMeta) Overlaps {
|
2018-03-28 10:33:41 -07:00
|
|
|
if len(bm) <= 1 {
|
2018-03-28 07:50:52 -07:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
sort.Slice(bm, func(i, j int) bool {
|
|
|
|
return bm[i].MinTime < bm[j].MinTime
|
2017-05-18 07:09:30 -07:00
|
|
|
})
|
2018-03-28 07:50:52 -07:00
|
|
|
|
2018-03-28 15:50:42 -07:00
|
|
|
var (
|
2018-04-05 05:51:33 -07:00
|
|
|
overlaps [][]BlockMeta
|
|
|
|
|
2018-03-28 15:50:42 -07:00
|
|
|
// pending contains not ended blocks in regards to "current" timestamp.
|
|
|
|
pending = []BlockMeta{bm[0]}
|
2018-03-29 04:50:46 -07:00
|
|
|
// continuousPending helps to aggregate same overlaps to single group.
|
|
|
|
continuousPending = true
|
2018-03-28 15:50:42 -07:00
|
|
|
)
|
2018-04-05 06:15:24 -07:00
|
|
|
|
|
|
|
// We have here blocks sorted by minTime. We iterate over each block and treat its minTime as our "current" timestamp.
|
|
|
|
// We check if any of the pending block finished (blocks that we have seen before, but their maxTime was still ahead current
|
|
|
|
// timestamp). If not, it means they overlap with our current block. In the same time current block is assumed pending.
|
2018-03-28 15:18:24 -07:00
|
|
|
for _, b := range bm[1:] {
|
2018-03-28 15:50:42 -07:00
|
|
|
var newPending []BlockMeta
|
2018-03-28 10:33:41 -07:00
|
|
|
|
2018-03-28 15:18:24 -07:00
|
|
|
for _, p := range pending {
|
2018-03-28 15:50:42 -07:00
|
|
|
// "b.MinTime" is our current time.
|
2018-03-28 15:18:24 -07:00
|
|
|
if b.MinTime >= p.MaxTime {
|
2018-03-29 04:50:46 -07:00
|
|
|
continuousPending = false
|
2018-03-28 15:18:24 -07:00
|
|
|
continue
|
2018-03-28 10:33:41 -07:00
|
|
|
}
|
|
|
|
|
2018-03-28 15:18:24 -07:00
|
|
|
// "p" overlaps with "b" and "p" is still pending.
|
|
|
|
newPending = append(newPending, p)
|
2017-05-18 07:09:30 -07:00
|
|
|
}
|
2018-03-28 15:50:42 -07:00
|
|
|
|
2018-03-28 15:18:24 -07:00
|
|
|
// Our block "b" is now pending.
|
|
|
|
pending = append(newPending, b)
|
|
|
|
if len(newPending) == 0 {
|
2018-03-28 15:50:42 -07:00
|
|
|
// No overlaps.
|
2018-03-28 15:18:24 -07:00
|
|
|
continue
|
2018-03-28 10:33:41 -07:00
|
|
|
}
|
|
|
|
|
2018-03-29 04:50:46 -07:00
|
|
|
if continuousPending && len(overlaps) > 0 {
|
2018-03-28 15:18:24 -07:00
|
|
|
overlaps[len(overlaps)-1] = append(overlaps[len(overlaps)-1], b)
|
2018-03-28 10:33:41 -07:00
|
|
|
continue
|
2017-05-18 07:09:30 -07:00
|
|
|
}
|
2018-03-28 15:18:24 -07:00
|
|
|
overlaps = append(overlaps, append(newPending, b))
|
2018-03-29 04:50:46 -07:00
|
|
|
// Start new pendings.
|
|
|
|
continuousPending = true
|
2017-05-18 07:09:30 -07:00
|
|
|
}
|
2018-04-05 05:51:33 -07:00
|
|
|
|
|
|
|
// Fetch the critical overlapped time range foreach overlap groups.
|
2018-04-05 06:15:24 -07:00
|
|
|
overlapGroups := Overlaps{}
|
2018-04-05 05:51:33 -07:00
|
|
|
for _, overlap := range overlaps {
|
|
|
|
|
|
|
|
minRange := TimeRange{Min: 0, Max: math.MaxInt64}
|
|
|
|
for _, b := range overlap {
|
|
|
|
if minRange.Max > b.MaxTime {
|
|
|
|
minRange.Max = b.MaxTime
|
|
|
|
}
|
|
|
|
|
|
|
|
if minRange.Min < b.MinTime {
|
|
|
|
minRange.Min = b.MinTime
|
|
|
|
}
|
|
|
|
}
|
|
|
|
overlapGroups[minRange] = overlap
|
|
|
|
}
|
|
|
|
|
|
|
|
return overlapGroups
|
2017-01-02 13:24:35 -08:00
|
|
|
}
|
|
|
|
|
2017-10-09 06:21:46 -07:00
|
|
|
func (db *DB) String() string {
|
|
|
|
return "HEAD"
|
|
|
|
}
|
|
|
|
|
|
|
|
// Blocks returns the databases persisted blocks.
|
|
|
|
func (db *DB) Blocks() []*Block {
|
2017-08-29 06:39:27 -07:00
|
|
|
db.mtx.RLock()
|
|
|
|
defer db.mtx.RUnlock()
|
|
|
|
|
|
|
|
return db.blocks
|
|
|
|
}
|
|
|
|
|
2017-10-09 06:21:46 -07:00
|
|
|
// Head returns the databases's head.
|
2017-09-25 07:45:24 -07:00
|
|
|
func (db *DB) Head() *Head {
|
|
|
|
return db.head
|
|
|
|
}
|
|
|
|
|
2017-01-05 23:08:02 -08:00
|
|
|
// Close the partition.
|
2017-01-06 02:40:09 -08:00
|
|
|
func (db *DB) Close() error {
|
2017-01-06 03:37:28 -08:00
|
|
|
close(db.stopc)
|
|
|
|
<-db.donec
|
|
|
|
|
2017-01-06 02:40:09 -08:00
|
|
|
db.mtx.Lock()
|
2017-03-17 04:12:50 -07:00
|
|
|
defer db.mtx.Unlock()
|
2017-03-04 07:50:48 -08:00
|
|
|
|
2017-03-06 03:13:15 -08:00
|
|
|
var g errgroup.Group
|
2017-01-02 01:34:55 -08:00
|
|
|
|
2017-03-20 00:41:56 -07:00
|
|
|
// blocks also contains all head blocks.
|
|
|
|
for _, pb := range db.blocks {
|
2017-03-06 03:13:15 -08:00
|
|
|
g.Go(pb.Close)
|
2016-12-14 23:31:26 -08:00
|
|
|
}
|
|
|
|
|
2017-03-06 03:13:15 -08:00
|
|
|
var merr MultiError
|
|
|
|
|
|
|
|
merr.Add(g.Wait())
|
2017-08-28 15:39:17 -07:00
|
|
|
|
2017-05-09 03:52:47 -07:00
|
|
|
if db.lockf != nil {
|
2018-05-29 11:35:48 -07:00
|
|
|
merr.Add(db.lockf.Release())
|
2017-05-09 03:52:47 -07:00
|
|
|
}
|
2017-11-10 12:19:39 -08:00
|
|
|
merr.Add(db.head.Close())
|
2017-01-02 13:24:35 -08:00
|
|
|
return merr.Err()
|
2016-12-09 01:00:14 -08:00
|
|
|
}
|
|
|
|
|
2017-06-06 07:53:20 -07:00
|
|
|
// DisableCompactions disables compactions.
|
2017-06-06 11:15:23 -07:00
|
|
|
func (db *DB) DisableCompactions() {
|
2017-07-14 01:06:07 -07:00
|
|
|
db.cmtx.Lock()
|
|
|
|
defer db.cmtx.Unlock()
|
|
|
|
|
|
|
|
db.compactionsEnabled = false
|
2017-09-28 00:19:34 -07:00
|
|
|
level.Info(db.logger).Log("msg", "compactions disabled")
|
2017-06-06 07:53:20 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
// EnableCompactions enables compactions.
|
2017-06-06 11:15:23 -07:00
|
|
|
func (db *DB) EnableCompactions() {
|
2017-07-14 01:06:07 -07:00
|
|
|
db.cmtx.Lock()
|
|
|
|
defer db.cmtx.Unlock()
|
|
|
|
|
|
|
|
db.compactionsEnabled = true
|
2017-09-28 00:19:34 -07:00
|
|
|
level.Info(db.logger).Log("msg", "compactions enabled")
|
2017-06-05 01:18:31 -07:00
|
|
|
}
|
|
|
|
|
2018-02-28 03:04:55 -08:00
|
|
|
// Snapshot writes the current data to the directory. If withHead is set to true it
|
|
|
|
// will create a new block containing all data that's currently in the memory buffer/WAL.
|
|
|
|
func (db *DB) Snapshot(dir string, withHead bool) error {
|
2017-08-30 09:34:54 -07:00
|
|
|
if dir == db.dir {
|
|
|
|
return errors.Errorf("cannot snapshot into base directory")
|
|
|
|
}
|
|
|
|
if _, err := ulid.Parse(dir); err == nil {
|
|
|
|
return errors.Errorf("dir must not be a valid ULID")
|
|
|
|
}
|
2017-06-05 01:18:31 -07:00
|
|
|
|
2017-08-30 09:34:54 -07:00
|
|
|
db.cmtx.Lock()
|
|
|
|
defer db.cmtx.Unlock()
|
|
|
|
|
2017-10-23 11:30:03 -07:00
|
|
|
db.mtx.RLock()
|
|
|
|
defer db.mtx.RUnlock()
|
|
|
|
|
|
|
|
for _, b := range db.blocks {
|
2017-09-28 00:19:34 -07:00
|
|
|
level.Info(db.logger).Log("msg", "snapshotting block", "block", b)
|
2017-08-30 09:34:54 -07:00
|
|
|
|
|
|
|
if err := b.Snapshot(dir); err != nil {
|
2017-11-22 04:28:06 -08:00
|
|
|
return errors.Wrapf(err, "error snapshotting block: %s", b.Dir())
|
2017-08-30 09:34:54 -07:00
|
|
|
}
|
|
|
|
}
|
2018-02-28 03:04:55 -08:00
|
|
|
if !withHead {
|
|
|
|
return nil
|
|
|
|
}
|
2018-06-27 06:47:11 -07:00
|
|
|
_, err := db.compactor.Write(dir, db.head, db.head.MinTime(), db.head.MaxTime(), nil)
|
2017-11-14 06:25:30 -08:00
|
|
|
return errors.Wrap(err, "snapshot head block")
|
2017-08-28 15:39:17 -07:00
|
|
|
}
|
2017-07-14 00:00:22 -07:00
|
|
|
|
2017-08-28 15:39:17 -07:00
|
|
|
// Querier returns a new querier over the data partition for the given time range.
|
|
|
|
// A goroutine must not handle more than one open Querier.
|
2017-10-09 06:21:46 -07:00
|
|
|
func (db *DB) Querier(mint, maxt int64) (Querier, error) {
|
|
|
|
var blocks []BlockReader
|
2017-08-28 15:39:17 -07:00
|
|
|
|
2017-10-23 11:30:03 -07:00
|
|
|
db.mtx.RLock()
|
|
|
|
defer db.mtx.RUnlock()
|
|
|
|
|
|
|
|
for _, b := range db.blocks {
|
2017-10-09 06:21:46 -07:00
|
|
|
m := b.Meta()
|
|
|
|
if intervalOverlap(mint, maxt, m.MinTime, m.MaxTime) {
|
|
|
|
blocks = append(blocks, b)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if maxt >= db.head.MinTime() {
|
|
|
|
blocks = append(blocks, db.head)
|
|
|
|
}
|
2017-08-28 15:39:17 -07:00
|
|
|
|
|
|
|
sq := &querier{
|
|
|
|
blocks: make([]Querier, 0, len(blocks)),
|
|
|
|
}
|
2017-06-06 05:45:54 -07:00
|
|
|
for _, b := range blocks {
|
2017-10-09 06:21:46 -07:00
|
|
|
q, err := NewBlockQuerier(b, mint, maxt)
|
2017-10-23 11:30:03 -07:00
|
|
|
if err == nil {
|
|
|
|
sq.blocks = append(sq.blocks, q)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
// If we fail, all previously opened queriers must be closed.
|
|
|
|
for _, q := range sq.blocks {
|
|
|
|
q.Close()
|
2017-10-09 06:21:46 -07:00
|
|
|
}
|
2017-10-23 11:30:03 -07:00
|
|
|
return nil, errors.Wrapf(err, "open querier for block %s", b)
|
2017-10-09 06:21:46 -07:00
|
|
|
}
|
|
|
|
return sq, nil
|
2017-08-28 15:39:17 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
func rangeForTimestamp(t int64, width int64) (mint, maxt int64) {
|
|
|
|
mint = (t / width) * width
|
|
|
|
return mint, mint + width
|
2017-02-01 06:29:48 -08:00
|
|
|
}
|
|
|
|
|
2017-08-28 15:39:17 -07:00
|
|
|
// Delete implements deletion of metrics. It only has atomicity guarantees on a per-block basis.
|
2017-05-19 12:05:50 -07:00
|
|
|
func (db *DB) Delete(mint, maxt int64, ms ...labels.Matcher) error {
|
2017-05-20 00:51:10 -07:00
|
|
|
db.cmtx.Lock()
|
|
|
|
defer db.cmtx.Unlock()
|
2017-07-14 00:00:22 -07:00
|
|
|
|
2017-05-19 12:05:50 -07:00
|
|
|
var g errgroup.Group
|
|
|
|
|
2017-10-23 11:30:03 -07:00
|
|
|
db.mtx.RLock()
|
|
|
|
defer db.mtx.RUnlock()
|
|
|
|
|
|
|
|
for _, b := range db.blocks {
|
2017-08-28 15:39:17 -07:00
|
|
|
m := b.Meta()
|
|
|
|
if intervalOverlap(mint, maxt, m.MinTime, m.MaxTime) {
|
2017-10-09 06:21:46 -07:00
|
|
|
g.Go(func(b *Block) func() error {
|
2017-08-28 15:39:17 -07:00
|
|
|
return func() error { return b.Delete(mint, maxt, ms...) }
|
|
|
|
}(b))
|
|
|
|
}
|
|
|
|
}
|
2017-08-30 09:34:54 -07:00
|
|
|
g.Go(func() error {
|
|
|
|
return db.head.Delete(mint, maxt, ms...)
|
|
|
|
})
|
2018-03-21 14:23:47 -07:00
|
|
|
return g.Wait()
|
2017-05-19 12:05:50 -07:00
|
|
|
}
|
|
|
|
|
2017-11-22 04:34:50 -08:00
|
|
|
// CleanTombstones re-writes any blocks with tombstones.
|
2018-05-30 19:09:30 -07:00
|
|
|
func (db *DB) CleanTombstones() (err error) {
|
2017-11-22 04:34:50 -08:00
|
|
|
db.cmtx.Lock()
|
|
|
|
defer db.cmtx.Unlock()
|
|
|
|
|
|
|
|
start := time.Now()
|
2018-03-21 13:23:47 -07:00
|
|
|
defer db.metrics.tombCleanTimer.Observe(time.Since(start).Seconds())
|
2017-11-22 04:34:50 -08:00
|
|
|
|
2018-05-30 19:09:30 -07:00
|
|
|
newUIDs := []ulid.ULID{}
|
|
|
|
defer func() {
|
|
|
|
// If any error is caused, we need to delete all the new directory created.
|
|
|
|
if err != nil {
|
|
|
|
for _, uid := range newUIDs {
|
|
|
|
dir := filepath.Join(db.Dir(), uid.String())
|
|
|
|
if err := os.RemoveAll(dir); err != nil {
|
|
|
|
level.Error(db.logger).Log("msg", "failed to delete block after failed `CleanTombstones`", "dir", dir, "err", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
2017-11-22 04:34:50 -08:00
|
|
|
db.mtx.RLock()
|
|
|
|
blocks := db.blocks[:]
|
|
|
|
db.mtx.RUnlock()
|
|
|
|
|
|
|
|
for _, b := range blocks {
|
2018-06-04 12:18:44 -07:00
|
|
|
if uid, er := b.CleanTombstones(db.Dir(), db.compactor); er != nil {
|
2018-05-30 19:09:30 -07:00
|
|
|
err = errors.Wrapf(er, "clean tombstones: %s", b.Dir())
|
|
|
|
return err
|
2018-06-04 12:18:44 -07:00
|
|
|
} else if uid != nil { // New block was created.
|
2018-05-30 19:09:30 -07:00
|
|
|
newUIDs = append(newUIDs, *uid)
|
2017-11-22 04:34:50 -08:00
|
|
|
}
|
|
|
|
}
|
2018-06-27 06:47:11 -07:00
|
|
|
return errors.Wrap(db.reload(), "reload blocks")
|
2017-11-22 04:34:50 -08:00
|
|
|
}
|
|
|
|
|
2016-12-15 07:14:33 -08:00
|
|
|
func intervalOverlap(amin, amax, bmin, bmax int64) bool {
|
2017-05-19 12:05:50 -07:00
|
|
|
// Checks Overlap: http://stackoverflow.com/questions/3269434/
|
2017-05-23 05:43:30 -07:00
|
|
|
return amin <= bmax && bmin <= amax
|
2016-12-15 07:14:33 -08:00
|
|
|
}
|
|
|
|
|
2017-01-19 02:22:47 -08:00
|
|
|
func isBlockDir(fi os.FileInfo) bool {
|
|
|
|
if !fi.IsDir() {
|
|
|
|
return false
|
|
|
|
}
|
2017-05-18 07:09:30 -07:00
|
|
|
_, err := ulid.Parse(fi.Name())
|
|
|
|
return err == nil
|
2017-01-19 02:22:47 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
func blockDirs(dir string) ([]string, error) {
|
|
|
|
files, err := ioutil.ReadDir(dir)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
var dirs []string
|
|
|
|
|
|
|
|
for _, fi := range files {
|
|
|
|
if isBlockDir(fi) {
|
|
|
|
dirs = append(dirs, filepath.Join(dir, fi.Name()))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return dirs, nil
|
2017-01-03 06:43:26 -08:00
|
|
|
}
|
2016-12-09 04:41:38 -08:00
|
|
|
|
2017-08-30 09:34:54 -07:00
|
|
|
func sequenceFiles(dir string) ([]string, error) {
|
2017-02-13 23:53:19 -08:00
|
|
|
files, err := ioutil.ReadDir(dir)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
var res []string
|
|
|
|
|
|
|
|
for _, fi := range files {
|
2017-08-30 09:34:54 -07:00
|
|
|
if _, err := strconv.ParseUint(fi.Name(), 10, 64); err != nil {
|
|
|
|
continue
|
2017-02-13 23:53:19 -08:00
|
|
|
}
|
2017-08-30 09:34:54 -07:00
|
|
|
res = append(res, filepath.Join(dir, fi.Name()))
|
2017-02-13 23:53:19 -08:00
|
|
|
}
|
|
|
|
return res, nil
|
|
|
|
}
|
|
|
|
|
2017-08-30 09:34:54 -07:00
|
|
|
func nextSequenceFile(dir string) (string, int, error) {
|
2017-01-19 02:22:47 -08:00
|
|
|
names, err := fileutil.ReadDir(dir)
|
2017-01-06 00:26:39 -08:00
|
|
|
if err != nil {
|
2017-01-28 23:11:47 -08:00
|
|
|
return "", 0, err
|
2017-01-06 00:26:39 -08:00
|
|
|
}
|
2017-01-06 04:13:22 -08:00
|
|
|
|
2017-01-06 00:26:39 -08:00
|
|
|
i := uint64(0)
|
2017-01-06 04:13:22 -08:00
|
|
|
for _, n := range names {
|
2017-08-30 09:34:54 -07:00
|
|
|
j, err := strconv.ParseUint(n, 10, 64)
|
2017-01-06 04:13:22 -08:00
|
|
|
if err != nil {
|
|
|
|
continue
|
2017-01-06 00:26:39 -08:00
|
|
|
}
|
2017-01-06 04:13:22 -08:00
|
|
|
i = j
|
2017-01-06 00:26:39 -08:00
|
|
|
}
|
2017-08-30 09:34:54 -07:00
|
|
|
return filepath.Join(dir, fmt.Sprintf("%0.6d", i+1)), int(i + 1), nil
|
2017-01-06 00:26:39 -08:00
|
|
|
}
|
2016-12-09 04:41:38 -08:00
|
|
|
|
2016-12-10 09:08:50 -08:00
|
|
|
// The MultiError type implements the error interface, and contains the
|
|
|
|
// Errors used to construct it.
|
|
|
|
type MultiError []error
|
2016-12-07 08:10:49 -08:00
|
|
|
|
2016-12-10 09:08:50 -08:00
|
|
|
// Returns a concatenated string of the contained errors
|
|
|
|
func (es MultiError) Error() string {
|
|
|
|
var buf bytes.Buffer
|
2016-12-07 08:10:49 -08:00
|
|
|
|
2017-01-04 12:11:15 -08:00
|
|
|
if len(es) > 1 {
|
2016-12-10 09:08:50 -08:00
|
|
|
fmt.Fprintf(&buf, "%d errors: ", len(es))
|
2016-12-08 01:04:24 -08:00
|
|
|
}
|
2016-12-07 08:10:49 -08:00
|
|
|
|
2016-12-10 09:08:50 -08:00
|
|
|
for i, err := range es {
|
|
|
|
if i != 0 {
|
|
|
|
buf.WriteString("; ")
|
|
|
|
}
|
|
|
|
buf.WriteString(err.Error())
|
|
|
|
}
|
2016-12-07 08:10:49 -08:00
|
|
|
|
2016-12-10 09:08:50 -08:00
|
|
|
return buf.String()
|
2016-11-15 01:34:25 -08:00
|
|
|
}
|
2016-12-14 23:31:26 -08:00
|
|
|
|
2016-12-15 02:56:41 -08:00
|
|
|
// Add adds the error to the error list if it is not nil.
|
2016-12-31 06:35:08 -08:00
|
|
|
func (es *MultiError) Add(err error) {
|
|
|
|
if err == nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if merr, ok := err.(MultiError); ok {
|
|
|
|
*es = append(*es, merr...)
|
|
|
|
} else {
|
|
|
|
*es = append(*es, err)
|
2016-12-14 23:31:26 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-12-15 02:56:41 -08:00
|
|
|
// Err returns the error list as an error or nil if it is empty.
|
2016-12-14 23:31:26 -08:00
|
|
|
func (es MultiError) Err() error {
|
|
|
|
if len(es) == 0 {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
return es
|
|
|
|
}
|
2016-12-15 02:56:41 -08:00
|
|
|
|
2017-02-27 01:46:15 -08:00
|
|
|
func closeAll(cs ...io.Closer) error {
|
|
|
|
var merr MultiError
|
|
|
|
|
|
|
|
for _, c := range cs {
|
|
|
|
merr.Add(c.Close())
|
|
|
|
}
|
|
|
|
return merr.Err()
|
|
|
|
}
|
2017-08-28 15:39:17 -07:00
|
|
|
|
|
|
|
func exponential(d, min, max time.Duration) time.Duration {
|
|
|
|
d *= 2
|
|
|
|
if d < min {
|
|
|
|
d = min
|
|
|
|
}
|
|
|
|
if d > max {
|
|
|
|
d = max
|
|
|
|
}
|
|
|
|
return d
|
|
|
|
}
|