Remove temporary patch for out-of-order (#283)

* Remove temporary patch for out-of-order

Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com>

* Remove ooo_wbl patch and fix tests

Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com>

tsdb/db.go

@@ -711,14 +711,6 @@ func open(dir string, l log.Logger, r prometheus.Registerer, opts *Options, rngs
 
 	walDir := filepath.Join(dir, "wal")
 	wblDir := filepath.Join(dir, wal.WblDirName)
-	// TODO(jesus.vazquez) Remove the block of code below, only necessary until all ooo_wbl dirs in prod have been replaced with wbl
-	oldWblDir := filepath.Join(dir, "ooo_wbl")
-	if _, err := os.Stat(oldWblDir); err == nil {
-		err = fileutil.Rename(oldWblDir, wblDir)
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to move old wbl dir to new wbl dir")
-		}
-	}
 
 	// Migrate old WAL if one exists.
 	if err := MigrateWAL(l, walDir); err != nil {
@@ -1653,14 +1645,6 @@ func (db *DB) inOrderBlocksMaxTime() (maxt int64, ok bool) {
 			maxt = b.meta.MaxTime
 		}
 	}
-	if !hasOOO && ok && db.opts.OutOfOrderTimeWindow > 0 {
-		// Temporary patch. To be removed by mid July 2022.
-		// Before this patch, blocks did not have "out_of_order" in their meta, so we cannot
-		// say which block has the out_of_order data. In that case the out-of-order block can be
-		// up to 2 block ranges ahead of the latest in-order block.
-		// Note: if hasOOO was true, it means the latest block has the new meta and is taken care in inOrderBlocksMaxTime().
-		maxt -= 2 * db.opts.MinBlockDuration
-	}
 	return maxt, ok
 }
 

tsdb/db_test.go

@@ -4433,11 +4433,6 @@ func TestWBLAndMmapReplay(t *testing.T) {
 		require.NoError(t, os.RemoveAll(wblDir))
 		require.NoError(t, fileutil.CopyDirs(originalWblDir, wblDir))
 	}
-	// TODO(jesus.vazquez) Remove this function and the test below using it, only necessary until all ooo_wbl dirs in prod have been replaced with wbl
-	resetWBLToOldDir := func() {
-		require.NoError(t, os.RemoveAll(wblDir))
-		require.NoError(t, fileutil.CopyDirs(originalWblDir, filepath.Join(db.dir, "ooo_wbl")))
-	}
 	resetMmapToOriginal := func() {
 		require.NoError(t, os.RemoveAll(mmapDir))
 		require.NoError(t, fileutil.CopyDirs(originalMmapDir, mmapDir))
@@ -4534,18 +4529,6 @@ func TestWBLAndMmapReplay(t *testing.T) {
 		require.Equal(t, oooMint, db.head.MinOOOTime())
 		require.Equal(t, oooMaxt, db.head.MaxOOOTime())
 		testQuery(expSamples)
-		require.NoError(t, db.Close())
-	})
-
-	t.Run("Restart DB with Old WBL Dir", func(t *testing.T) {
-		resetWBLToOldDir()
-		resetMmapToOriginal()
-
-		db, err = Open(db.dir, nil, nil, opts, nil)
-		require.NoError(t, err)
-		require.Equal(t, oooMint, db.head.MinOOOTime())
-		require.Equal(t, oooMaxt, db.head.MaxOOOTime())
-		testQuery(expSamples)
 	})
 }
 
@@ -5233,92 +5216,68 @@ func TestNoGapAfterRestartWithOOO(t *testing.T) {
 		// After compaction.
 		blockRanges        [][2]int64
 		headMint, headMaxt int64
-		// Head time ranges after restart for old blocks.
-		legacyHeadMint, legacyHeadMaxt int64
 	}{
 		{
 			300, 490,
 			489, 489,
 			[][2]int64{{300, 360}, {480, 600}},
 			360, 490,
-			360, 490, // OOO blocks is already 2 ranges ahead of the in-order block.
 		},
 		{
 			300, 490,
 			479, 479,
 			[][2]int64{{300, 360}, {360, 480}},
 			360, 490,
-			240, 490, // OOO block was only 1 range ahead of in-order block.
 		},
 	}
 
 	for i, c := range cases {
-		// legacy = true means the out-of-order blocks don't have the `out_of_order: true` metadata.
+		t.Run(fmt.Sprintf("case=%d", i), func(t *testing.T) {
-		for _, legacy := range []bool{false, true} {
+			dir := t.TempDir()
-			t.Run(fmt.Sprintf("case=%d,legacy=%t", i, legacy), func(t *testing.T) {
-				dir := t.TempDir()
 
-				opts := DefaultOptions()
+			opts := DefaultOptions()
-				opts.OutOfOrderTimeWindow = 30 * time.Minute.Milliseconds()
+			opts.OutOfOrderTimeWindow = 30 * time.Minute.Milliseconds()
 
-				db, err := Open(dir, nil, nil, opts, nil)
+			db, err := Open(dir, nil, nil, opts, nil)
-				require.NoError(t, err)
+			require.NoError(t, err)
-				db.DisableCompactions()
+			db.DisableCompactions()
-				t.Cleanup(func() {
+			t.Cleanup(func() {
-					require.NoError(t, db.Close())
-				})
-
-				// 3h10m=190m worth in-order data.
-				addSamples(t, db, c.inOrderMint, c.inOrderMaxt, true)
-				verifySamples(t, db, c.inOrderMint, c.inOrderMaxt)
-
-				// One ooo samples.
-				addSamples(t, db, c.oooMint, c.oooMaxt, true)
-				verifySamples(t, db, c.inOrderMint, c.inOrderMaxt)
-
-				// We get 2 blocks. 1 from OOO, 1 from in-order.
-				require.NoError(t, db.Compact())
-				verifyBlockRanges := func() {
-					blocks := db.Blocks()
-					require.Equal(t, len(c.blockRanges), len(blocks))
-					for j, br := range c.blockRanges {
-						require.Equal(t, br[0]*time.Minute.Milliseconds(), blocks[j].MinTime())
-						require.Equal(t, br[1]*time.Minute.Milliseconds(), blocks[j].MaxTime())
-					}
-				}
-				verifyBlockRanges()
-				require.Equal(t, c.headMint*time.Minute.Milliseconds(), db.head.MinTime())
-				require.Equal(t, c.headMaxt*time.Minute.Milliseconds(), db.head.MaxTime())
-
-				if legacy {
-					// In the legacy version, the blocks from out-of-order data did not write a
-					// "out_of_order: true" to the meta. So we remove it here.
-					for _, b := range db.Blocks() {
-						m, _, err := readMetaFile(b.Dir())
-						require.NoError(t, err)
-						m.OutOfOrder = false
-						_, err = writeMetaFile(log.NewNopLogger(), b.Dir(), m)
-						require.NoError(t, err)
-					}
-				}
-
-				// Restart and expect all samples to be present.
 				require.NoError(t, db.Close())
-
-				db, err = Open(dir, nil, nil, opts, nil)
-				require.NoError(t, err)
-				db.DisableCompactions()
-
-				verifyBlockRanges()
-				if legacy {
-					require.Equal(t, c.legacyHeadMint*time.Minute.Milliseconds(), db.head.MinTime())
-					require.Equal(t, c.legacyHeadMaxt*time.Minute.Milliseconds(), db.head.MaxTime())
-				} else {
-					require.Equal(t, c.headMint*time.Minute.Milliseconds(), db.head.MinTime())
-					require.Equal(t, c.headMaxt*time.Minute.Milliseconds(), db.head.MaxTime())
-				}
-				verifySamples(t, db, c.inOrderMint, c.inOrderMaxt)
 			})
-		}
+
+			// 3h10m=190m worth in-order data.
+			addSamples(t, db, c.inOrderMint, c.inOrderMaxt, true)
+			verifySamples(t, db, c.inOrderMint, c.inOrderMaxt)
+
+			// One ooo samples.
+			addSamples(t, db, c.oooMint, c.oooMaxt, true)
+			verifySamples(t, db, c.inOrderMint, c.inOrderMaxt)
+
+			// We get 2 blocks. 1 from OOO, 1 from in-order.
+			require.NoError(t, db.Compact())
+			verifyBlockRanges := func() {
+				blocks := db.Blocks()
+				require.Equal(t, len(c.blockRanges), len(blocks))
+				for j, br := range c.blockRanges {
+					require.Equal(t, br[0]*time.Minute.Milliseconds(), blocks[j].MinTime())
+					require.Equal(t, br[1]*time.Minute.Milliseconds(), blocks[j].MaxTime())
+				}
+			}
+			verifyBlockRanges()
+			require.Equal(t, c.headMint*time.Minute.Milliseconds(), db.head.MinTime())
+			require.Equal(t, c.headMaxt*time.Minute.Milliseconds(), db.head.MaxTime())
+
+			// Restart and expect all samples to be present.
+			require.NoError(t, db.Close())
+
+			db, err = Open(dir, nil, nil, opts, nil)
+			require.NoError(t, err)
+			db.DisableCompactions()
+
+			verifyBlockRanges()
+			require.Equal(t, c.headMint*time.Minute.Milliseconds(), db.head.MinTime())
+			require.Equal(t, c.headMaxt*time.Minute.Milliseconds(), db.head.MaxTime())
+			verifySamples(t, db, c.inOrderMint, c.inOrderMaxt)
+		})
 	}
 }