mirror of
https://github.com/prometheus/prometheus.git
synced 2024-09-19 23:37:31 -07:00
Remove temporary patch for out-of-order (#283)
* Remove temporary patch for out-of-order Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com> * Remove ooo_wbl patch and fix tests Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com>
This commit is contained in:
Ganesh Vernekar
GitHub
parent
5e8406a1d4
commit
c6f3d4ab33
16
tsdb/db.go
16
tsdb/db.go
|
|
@ -711,14 +711,6 @@ func open(dir string, l log.Logger, r prometheus.Registerer, opts *Options, rngs
|
|||
|
||||
walDir := filepath.Join(dir, "wal")
|
||||
wblDir := filepath.Join(dir, wal.WblDirName)
|
||||
// TODO(jesus.vazquez) Remove the block of code below, only necessary until all ooo_wbl dirs in prod have been replaced with wbl
|
||||
oldWblDir := filepath.Join(dir, "ooo_wbl")
|
||||
if _, err := os.Stat(oldWblDir); err == nil {
|
||||
err = fileutil.Rename(oldWblDir, wblDir)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to move old wbl dir to new wbl dir")
|
||||
}
|
||||
}
|
||||
|
||||
// Migrate old WAL if one exists.
|
||||
if err := MigrateWAL(l, walDir); err != nil {
|
||||
|
|
@ -1653,14 +1645,6 @@ func (db *DB) inOrderBlocksMaxTime() (maxt int64, ok bool) {
|
|||
maxt = b.meta.MaxTime
|
||||
}
|
||||
}
|
||||
if !hasOOO && ok && db.opts.OutOfOrderTimeWindow > 0 {
|
||||
// Temporary patch. To be removed by mid July 2022.
|
||||
// Before this patch, blocks did not have "out_of_order" in their meta, so we cannot
|
||||
// say which block has the out_of_order data. In that case the out-of-order block can be
|
||||
// up to 2 block ranges ahead of the latest in-order block.
|
||||
// Note: if hasOOO was true, it means the latest block has the new meta and is taken care in inOrderBlocksMaxTime().
|
||||
maxt -= 2 * db.opts.MinBlockDuration
|
||||
}
|
||||
return maxt, ok
|
||||
}
|
||||
|
||||
|
|
|
|||
127
tsdb/db_test.go
127
tsdb/db_test.go
|
|
@ -4433,11 +4433,6 @@ func TestWBLAndMmapReplay(t *testing.T) {
|
|||
require.NoError(t, os.RemoveAll(wblDir))
|
||||
require.NoError(t, fileutil.CopyDirs(originalWblDir, wblDir))
|
||||
}
|
||||
// TODO(jesus.vazquez) Remove this function and the test below using it, only necessary until all ooo_wbl dirs in prod have been replaced with wbl
|
||||
resetWBLToOldDir := func() {
|
||||
require.NoError(t, os.RemoveAll(wblDir))
|
||||
require.NoError(t, fileutil.CopyDirs(originalWblDir, filepath.Join(db.dir, "ooo_wbl")))
|
||||
}
|
||||
resetMmapToOriginal := func() {
|
||||
require.NoError(t, os.RemoveAll(mmapDir))
|
||||
require.NoError(t, fileutil.CopyDirs(originalMmapDir, mmapDir))
|
||||
|
|
@ -4534,18 +4529,6 @@ func TestWBLAndMmapReplay(t *testing.T) {
|
|||
require.Equal(t, oooMint, db.head.MinOOOTime())
|
||||
require.Equal(t, oooMaxt, db.head.MaxOOOTime())
|
||||
testQuery(expSamples)
|
||||
require.NoError(t, db.Close())
|
||||
})
|
||||
|
||||
t.Run("Restart DB with Old WBL Dir", func(t *testing.T) {
|
||||
resetWBLToOldDir()
|
||||
resetMmapToOriginal()
|
||||
|
||||
db, err = Open(db.dir, nil, nil, opts, nil)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, oooMint, db.head.MinOOOTime())
|
||||
require.Equal(t, oooMaxt, db.head.MaxOOOTime())
|
||||
testQuery(expSamples)
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -5233,92 +5216,68 @@ func TestNoGapAfterRestartWithOOO(t *testing.T) {
|
|||
// After compaction.
|
||||
blockRanges [][2]int64
|
||||
headMint, headMaxt int64
|
||||
// Head time ranges after restart for old blocks.
|
||||
legacyHeadMint, legacyHeadMaxt int64
|
||||
}{
|
||||
{
|
||||
300, 490,
|
||||
489, 489,
|
||||
[][2]int64{{300, 360}, {480, 600}},
|
||||
360, 490,
|
||||
360, 490, // OOO blocks is already 2 ranges ahead of the in-order block.
|
||||
},
|
||||
{
|
||||
300, 490,
|
||||
479, 479,
|
||||
[][2]int64{{300, 360}, {360, 480}},
|
||||
360, 490,
|
||||
240, 490, // OOO block was only 1 range ahead of in-order block.
|
||||
},
|
||||
}
|
||||
|
||||
for i, c := range cases {
|
||||
// legacy = true means the out-of-order blocks don't have the `out_of_order: true` metadata.
|
||||
for _, legacy := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("case=%d,legacy=%t", i, legacy), func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Run(fmt.Sprintf("case=%d", i), func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
opts := DefaultOptions()
|
||||
opts.OutOfOrderTimeWindow = 30 * time.Minute.Milliseconds()
|
||||
opts := DefaultOptions()
|
||||
opts.OutOfOrderTimeWindow = 30 * time.Minute.Milliseconds()
|
||||
|
||||
db, err := Open(dir, nil, nil, opts, nil)
|
||||
require.NoError(t, err)
|
||||
db.DisableCompactions()
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, db.Close())
|
||||
})
|
||||
|
||||
// 3h10m=190m worth in-order data.
|
||||
addSamples(t, db, c.inOrderMint, c.inOrderMaxt, true)
|
||||
verifySamples(t, db, c.inOrderMint, c.inOrderMaxt)
|
||||
|
||||
// One ooo samples.
|
||||
addSamples(t, db, c.oooMint, c.oooMaxt, true)
|
||||
verifySamples(t, db, c.inOrderMint, c.inOrderMaxt)
|
||||
|
||||
// We get 2 blocks. 1 from OOO, 1 from in-order.
|
||||
require.NoError(t, db.Compact())
|
||||
verifyBlockRanges := func() {
|
||||
blocks := db.Blocks()
|
||||
require.Equal(t, len(c.blockRanges), len(blocks))
|
||||
for j, br := range c.blockRanges {
|
||||
require.Equal(t, br[0]*time.Minute.Milliseconds(), blocks[j].MinTime())
|
||||
require.Equal(t, br[1]*time.Minute.Milliseconds(), blocks[j].MaxTime())
|
||||
}
|
||||
}
|
||||
verifyBlockRanges()
|
||||
require.Equal(t, c.headMint*time.Minute.Milliseconds(), db.head.MinTime())
|
||||
require.Equal(t, c.headMaxt*time.Minute.Milliseconds(), db.head.MaxTime())
|
||||
|
||||
if legacy {
|
||||
// In the legacy version, the blocks from out-of-order data did not write a
|
||||
// "out_of_order: true" to the meta. So we remove it here.
|
||||
for _, b := range db.Blocks() {
|
||||
m, _, err := readMetaFile(b.Dir())
|
||||
require.NoError(t, err)
|
||||
m.OutOfOrder = false
|
||||
_, err = writeMetaFile(log.NewNopLogger(), b.Dir(), m)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Restart and expect all samples to be present.
|
||||
db, err := Open(dir, nil, nil, opts, nil)
|
||||
require.NoError(t, err)
|
||||
db.DisableCompactions()
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
db, err = Open(dir, nil, nil, opts, nil)
|
||||
require.NoError(t, err)
|
||||
db.DisableCompactions()
|
||||
|
||||
verifyBlockRanges()
|
||||
if legacy {
|
||||
require.Equal(t, c.legacyHeadMint*time.Minute.Milliseconds(), db.head.MinTime())
|
||||
require.Equal(t, c.legacyHeadMaxt*time.Minute.Milliseconds(), db.head.MaxTime())
|
||||
} else {
|
||||
require.Equal(t, c.headMint*time.Minute.Milliseconds(), db.head.MinTime())
|
||||
require.Equal(t, c.headMaxt*time.Minute.Milliseconds(), db.head.MaxTime())
|
||||
}
|
||||
verifySamples(t, db, c.inOrderMint, c.inOrderMaxt)
|
||||
})
|
||||
}
|
||||
|
||||
// 3h10m=190m worth in-order data.
|
||||
addSamples(t, db, c.inOrderMint, c.inOrderMaxt, true)
|
||||
verifySamples(t, db, c.inOrderMint, c.inOrderMaxt)
|
||||
|
||||
// One ooo samples.
|
||||
addSamples(t, db, c.oooMint, c.oooMaxt, true)
|
||||
verifySamples(t, db, c.inOrderMint, c.inOrderMaxt)
|
||||
|
||||
// We get 2 blocks. 1 from OOO, 1 from in-order.
|
||||
require.NoError(t, db.Compact())
|
||||
verifyBlockRanges := func() {
|
||||
blocks := db.Blocks()
|
||||
require.Equal(t, len(c.blockRanges), len(blocks))
|
||||
for j, br := range c.blockRanges {
|
||||
require.Equal(t, br[0]*time.Minute.Milliseconds(), blocks[j].MinTime())
|
||||
require.Equal(t, br[1]*time.Minute.Milliseconds(), blocks[j].MaxTime())
|
||||
}
|
||||
}
|
||||
verifyBlockRanges()
|
||||
require.Equal(t, c.headMint*time.Minute.Milliseconds(), db.head.MinTime())
|
||||
require.Equal(t, c.headMaxt*time.Minute.Milliseconds(), db.head.MaxTime())
|
||||
|
||||
// Restart and expect all samples to be present.
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
db, err = Open(dir, nil, nil, opts, nil)
|
||||
require.NoError(t, err)
|
||||
db.DisableCompactions()
|
||||
|
||||
verifyBlockRanges()
|
||||
require.Equal(t, c.headMint*time.Minute.Milliseconds(), db.head.MinTime())
|
||||
require.Equal(t, c.headMaxt*time.Minute.Milliseconds(), db.head.MaxTime())
|
||||
verifySamples(t, db, c.inOrderMint, c.inOrderMaxt)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue