From c6f3d4ab339ab80bbbce74c9946237ced01f0509 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar <15064823+codesome@users.noreply.github.com> Date: Mon, 4 Jul 2022 19:05:18 +0530 Subject: [PATCH] Remove temporary patch for out-of-order (#283) * Remove temporary patch for out-of-order Signed-off-by: Ganesh Vernekar * Remove ooo_wbl patch and fix tests Signed-off-by: Ganesh Vernekar --- tsdb/db.go | 16 ------ tsdb/db_test.go | 127 ++++++++++++++++-------------------------------- 2 files changed, 43 insertions(+), 100 deletions(-) diff --git a/tsdb/db.go b/tsdb/db.go index 2670118be..450d8cd93 100644 --- a/tsdb/db.go +++ b/tsdb/db.go @@ -711,14 +711,6 @@ func open(dir string, l log.Logger, r prometheus.Registerer, opts *Options, rngs walDir := filepath.Join(dir, "wal") wblDir := filepath.Join(dir, wal.WblDirName) - // TODO(jesus.vazquez) Remove the block of code below, only necessary until all ooo_wbl dirs in prod have been replaced with wbl - oldWblDir := filepath.Join(dir, "ooo_wbl") - if _, err := os.Stat(oldWblDir); err == nil { - err = fileutil.Rename(oldWblDir, wblDir) - if err != nil { - return nil, errors.Wrap(err, "failed to move old wbl dir to new wbl dir") - } - } // Migrate old WAL if one exists. if err := MigrateWAL(l, walDir); err != nil { @@ -1653,14 +1645,6 @@ func (db *DB) inOrderBlocksMaxTime() (maxt int64, ok bool) { maxt = b.meta.MaxTime } } - if !hasOOO && ok && db.opts.OutOfOrderTimeWindow > 0 { - // Temporary patch. To be removed by mid July 2022. - // Before this patch, blocks did not have "out_of_order" in their meta, so we cannot - // say which block has the out_of_order data. In that case the out-of-order block can be - // up to 2 block ranges ahead of the latest in-order block. - // Note: if hasOOO was true, it means the latest block has the new meta and is taken care in inOrderBlocksMaxTime(). - maxt -= 2 * db.opts.MinBlockDuration - } return maxt, ok } diff --git a/tsdb/db_test.go b/tsdb/db_test.go index 20dea2642..f230a30b8 100644 --- a/tsdb/db_test.go +++ b/tsdb/db_test.go @@ -4433,11 +4433,6 @@ func TestWBLAndMmapReplay(t *testing.T) { require.NoError(t, os.RemoveAll(wblDir)) require.NoError(t, fileutil.CopyDirs(originalWblDir, wblDir)) } - // TODO(jesus.vazquez) Remove this function and the test below using it, only necessary until all ooo_wbl dirs in prod have been replaced with wbl - resetWBLToOldDir := func() { - require.NoError(t, os.RemoveAll(wblDir)) - require.NoError(t, fileutil.CopyDirs(originalWblDir, filepath.Join(db.dir, "ooo_wbl"))) - } resetMmapToOriginal := func() { require.NoError(t, os.RemoveAll(mmapDir)) require.NoError(t, fileutil.CopyDirs(originalMmapDir, mmapDir)) @@ -4534,18 +4529,6 @@ func TestWBLAndMmapReplay(t *testing.T) { require.Equal(t, oooMint, db.head.MinOOOTime()) require.Equal(t, oooMaxt, db.head.MaxOOOTime()) testQuery(expSamples) - require.NoError(t, db.Close()) - }) - - t.Run("Restart DB with Old WBL Dir", func(t *testing.T) { - resetWBLToOldDir() - resetMmapToOriginal() - - db, err = Open(db.dir, nil, nil, opts, nil) - require.NoError(t, err) - require.Equal(t, oooMint, db.head.MinOOOTime()) - require.Equal(t, oooMaxt, db.head.MaxOOOTime()) - testQuery(expSamples) }) } @@ -5233,92 +5216,68 @@ func TestNoGapAfterRestartWithOOO(t *testing.T) { // After compaction. blockRanges [][2]int64 headMint, headMaxt int64 - // Head time ranges after restart for old blocks. - legacyHeadMint, legacyHeadMaxt int64 }{ { 300, 490, 489, 489, [][2]int64{{300, 360}, {480, 600}}, 360, 490, - 360, 490, // OOO blocks is already 2 ranges ahead of the in-order block. }, { 300, 490, 479, 479, [][2]int64{{300, 360}, {360, 480}}, 360, 490, - 240, 490, // OOO block was only 1 range ahead of in-order block. }, } for i, c := range cases { - // legacy = true means the out-of-order blocks don't have the `out_of_order: true` metadata. - for _, legacy := range []bool{false, true} { - t.Run(fmt.Sprintf("case=%d,legacy=%t", i, legacy), func(t *testing.T) { - dir := t.TempDir() + t.Run(fmt.Sprintf("case=%d", i), func(t *testing.T) { + dir := t.TempDir() - opts := DefaultOptions() - opts.OutOfOrderTimeWindow = 30 * time.Minute.Milliseconds() + opts := DefaultOptions() + opts.OutOfOrderTimeWindow = 30 * time.Minute.Milliseconds() - db, err := Open(dir, nil, nil, opts, nil) - require.NoError(t, err) - db.DisableCompactions() - t.Cleanup(func() { - require.NoError(t, db.Close()) - }) - - // 3h10m=190m worth in-order data. - addSamples(t, db, c.inOrderMint, c.inOrderMaxt, true) - verifySamples(t, db, c.inOrderMint, c.inOrderMaxt) - - // One ooo samples. - addSamples(t, db, c.oooMint, c.oooMaxt, true) - verifySamples(t, db, c.inOrderMint, c.inOrderMaxt) - - // We get 2 blocks. 1 from OOO, 1 from in-order. - require.NoError(t, db.Compact()) - verifyBlockRanges := func() { - blocks := db.Blocks() - require.Equal(t, len(c.blockRanges), len(blocks)) - for j, br := range c.blockRanges { - require.Equal(t, br[0]*time.Minute.Milliseconds(), blocks[j].MinTime()) - require.Equal(t, br[1]*time.Minute.Milliseconds(), blocks[j].MaxTime()) - } - } - verifyBlockRanges() - require.Equal(t, c.headMint*time.Minute.Milliseconds(), db.head.MinTime()) - require.Equal(t, c.headMaxt*time.Minute.Milliseconds(), db.head.MaxTime()) - - if legacy { - // In the legacy version, the blocks from out-of-order data did not write a - // "out_of_order: true" to the meta. So we remove it here. - for _, b := range db.Blocks() { - m, _, err := readMetaFile(b.Dir()) - require.NoError(t, err) - m.OutOfOrder = false - _, err = writeMetaFile(log.NewNopLogger(), b.Dir(), m) - require.NoError(t, err) - } - } - - // Restart and expect all samples to be present. + db, err := Open(dir, nil, nil, opts, nil) + require.NoError(t, err) + db.DisableCompactions() + t.Cleanup(func() { require.NoError(t, db.Close()) - - db, err = Open(dir, nil, nil, opts, nil) - require.NoError(t, err) - db.DisableCompactions() - - verifyBlockRanges() - if legacy { - require.Equal(t, c.legacyHeadMint*time.Minute.Milliseconds(), db.head.MinTime()) - require.Equal(t, c.legacyHeadMaxt*time.Minute.Milliseconds(), db.head.MaxTime()) - } else { - require.Equal(t, c.headMint*time.Minute.Milliseconds(), db.head.MinTime()) - require.Equal(t, c.headMaxt*time.Minute.Milliseconds(), db.head.MaxTime()) - } - verifySamples(t, db, c.inOrderMint, c.inOrderMaxt) }) - } + + // 3h10m=190m worth in-order data. + addSamples(t, db, c.inOrderMint, c.inOrderMaxt, true) + verifySamples(t, db, c.inOrderMint, c.inOrderMaxt) + + // One ooo samples. + addSamples(t, db, c.oooMint, c.oooMaxt, true) + verifySamples(t, db, c.inOrderMint, c.inOrderMaxt) + + // We get 2 blocks. 1 from OOO, 1 from in-order. + require.NoError(t, db.Compact()) + verifyBlockRanges := func() { + blocks := db.Blocks() + require.Equal(t, len(c.blockRanges), len(blocks)) + for j, br := range c.blockRanges { + require.Equal(t, br[0]*time.Minute.Milliseconds(), blocks[j].MinTime()) + require.Equal(t, br[1]*time.Minute.Milliseconds(), blocks[j].MaxTime()) + } + } + verifyBlockRanges() + require.Equal(t, c.headMint*time.Minute.Milliseconds(), db.head.MinTime()) + require.Equal(t, c.headMaxt*time.Minute.Milliseconds(), db.head.MaxTime()) + + // Restart and expect all samples to be present. + require.NoError(t, db.Close()) + + db, err = Open(dir, nil, nil, opts, nil) + require.NoError(t, err) + db.DisableCompactions() + + verifyBlockRanges() + require.Equal(t, c.headMint*time.Minute.Milliseconds(), db.head.MinTime()) + require.Equal(t, c.headMaxt*time.Minute.Milliseconds(), db.head.MaxTime()) + verifySamples(t, db, c.inOrderMint, c.inOrderMaxt) + }) } }