CompactBlockMetas should produce correct mint/maxt for overlapping blocks. (#10108)

Signed-off-by: Peter Štibraný <pstibrany@gmail.com>
This commit is contained in:
Peter Štibraný 2022-01-05 10:40:00 +01:00 committed by GitHub
parent dfa5cb7462
commit e51a17b501
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 44 additions and 7 deletions

View file

@ -18,7 +18,6 @@ import (
"crypto/rand" "crypto/rand"
"fmt" "fmt"
"io" "io"
"math"
"os" "os"
"path/filepath" "path/filepath"
"sort" "sort"
@ -347,19 +346,20 @@ func splitByRange(ds []dirMeta, tr int64) [][]dirMeta {
} }
// CompactBlockMetas merges many block metas into one, combining it's source blocks together // CompactBlockMetas merges many block metas into one, combining it's source blocks together
// and adjusting compaction level. // and adjusting compaction level. Min/Max time of result block meta covers all input blocks.
func CompactBlockMetas(uid ulid.ULID, blocks ...*BlockMeta) *BlockMeta { func CompactBlockMetas(uid ulid.ULID, blocks ...*BlockMeta) *BlockMeta {
res := &BlockMeta{ res := &BlockMeta{
ULID: uid, ULID: uid,
MinTime: blocks[0].MinTime,
} }
sources := map[ulid.ULID]struct{}{} sources := map[ulid.ULID]struct{}{}
// For overlapping blocks, the Maxt can be mint := blocks[0].MinTime
// in any block so we track it globally. maxt := blocks[0].MaxTime
maxt := int64(math.MinInt64)
for _, b := range blocks { for _, b := range blocks {
if b.MinTime < mint {
mint = b.MinTime
}
if b.MaxTime > maxt { if b.MaxTime > maxt {
maxt = b.MaxTime maxt = b.MaxTime
} }
@ -384,6 +384,7 @@ func CompactBlockMetas(uid ulid.ULID, blocks ...*BlockMeta) *BlockMeta {
return res.Compaction.Sources[i].Compare(res.Compaction.Sources[j]) < 0 return res.Compaction.Sources[i].Compare(res.Compaction.Sources[j]) < 0
}) })
res.MinTime = mint
res.MaxTime = maxt res.MaxTime = maxt
return res return res
} }

View file

@ -25,6 +25,7 @@ import (
"time" "time"
"github.com/go-kit/log" "github.com/go-kit/log"
"github.com/oklog/ulid"
"github.com/pkg/errors" "github.com/pkg/errors"
prom_testutil "github.com/prometheus/client_golang/prometheus/testutil" prom_testutil "github.com/prometheus/client_golang/prometheus/testutil"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@ -1311,3 +1312,38 @@ func TestDeleteCompactionBlockAfterFailedReload(t *testing.T) {
}) })
} }
} }
func TestCompactBlockMetas(t *testing.T) {
parent1 := ulid.MustNew(100, nil)
parent2 := ulid.MustNew(200, nil)
parent3 := ulid.MustNew(300, nil)
parent4 := ulid.MustNew(400, nil)
input := []*BlockMeta{
{ULID: parent1, MinTime: 1000, MaxTime: 2000, Compaction: BlockMetaCompaction{Level: 2, Sources: []ulid.ULID{ulid.MustNew(1, nil), ulid.MustNew(10, nil)}}},
{ULID: parent2, MinTime: 200, MaxTime: 500, Compaction: BlockMetaCompaction{Level: 1}},
{ULID: parent3, MinTime: 500, MaxTime: 2500, Compaction: BlockMetaCompaction{Level: 3, Sources: []ulid.ULID{ulid.MustNew(5, nil), ulid.MustNew(6, nil)}}},
{ULID: parent4, MinTime: 100, MaxTime: 900, Compaction: BlockMetaCompaction{Level: 1}},
}
outUlid := ulid.MustNew(1000, nil)
output := CompactBlockMetas(outUlid, input...)
expected := &BlockMeta{
ULID: outUlid,
MinTime: 100,
MaxTime: 2500,
Stats: BlockStats{},
Compaction: BlockMetaCompaction{
Level: 4,
Sources: []ulid.ULID{ulid.MustNew(1, nil), ulid.MustNew(5, nil), ulid.MustNew(6, nil), ulid.MustNew(10, nil)},
Parents: []BlockDesc{
{ULID: parent1, MinTime: 1000, MaxTime: 2000},
{ULID: parent2, MinTime: 200, MaxTime: 500},
{ULID: parent3, MinTime: 500, MaxTime: 2500},
{ULID: parent4, MinTime: 100, MaxTime: 900},
},
},
}
require.Equal(t, expected, output)
}