From d50b9a5619b2efb7559ea1a7187df9fa5c209121 Mon Sep 17 00:00:00 2001 From: Brian Brazil Date: Wed, 28 Nov 2018 09:23:50 +0000 Subject: [PATCH] Reload after reading the WAL. (#460) This causes the head to be GCed at startup, removing any series that were read from the WAL but have since been written to a block. In systems with low ingestion rates, this potentially could be many many hours of data. Signed-off-by: Brian Brazil --- db.go | 6 +++--- db_test.go | 8 +++++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/db.go b/db.go index 9b92823ec..40025bdbc 100644 --- a/db.go +++ b/db.go @@ -271,12 +271,12 @@ func Open(dir string, l log.Logger, r prometheus.Registerer, opts *Options) (db if err != nil { return nil, err } - if err := db.reload(); err != nil { - return nil, err - } if err := db.head.Init(); err != nil { return nil, errors.Wrap(err, "read WAL") } + if err := db.reload(); err != nil { + return nil, err + } go db.run() diff --git a/db_test.go b/db_test.go index 99bde8c5c..6fd59f348 100644 --- a/db_test.go +++ b/db_test.go @@ -27,6 +27,8 @@ import ( "github.com/oklog/ulid" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + prom_testutil "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/tsdb/chunks" "github.com/prometheus/tsdb/index" "github.com/prometheus/tsdb/labels" @@ -1295,11 +1297,15 @@ func TestInitializeHeadTimestamp(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, w.Close()) - db, err := Open(dir, nil, nil, nil) + r := prometheus.NewRegistry() + + db, err := Open(dir, nil, r, nil) testutil.Ok(t, err) testutil.Equals(t, int64(6000), db.head.MinTime()) testutil.Equals(t, int64(15000), db.head.MaxTime()) + // Check that old series has been GCed. + testutil.Equals(t, 1.0, prom_testutil.ToFloat64(db.head.metrics.series)) }) }