mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2025-03-05 20:59:13 -08:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

[CHORE] deleting blocks on startup

Browse source 
Signed-off-by: Nicolas Takashi <nicolas.tcs@hotmail.com>
This commit is contained in:
Nicolas Takashi 2024-12-20 23:43:00 +00:00
parent b7cf351e06
commit 7f2fe8551e
No known key found for this signature in database
GPG key ID: 65BE5B80BBC7A707
2 changed files with 14 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
cmd/promtool/backfill_test.go
View file

@ -745,6 +745,7 @@ after_eof 1 2
require.NoError(t, err) require.NoError(t, err)
options := tsdb.DefaultOptions() options := tsdb.DefaultOptions()
options.RetentionDuration = int64(10 * 365 * 24 * time.Hour / time.Millisecond) // maximum duration tests require a long retention options.RetentionDuration = int64(10 * 365 * 24 * time.Hour / time.Millisecond) // maximum duration tests require a long retention
options.StartupMinRetentionTime = test.Expected.MaxTime * 2
db, err := tsdb.Open(outputDir, nil, nil, options, nil) db, err := tsdb.Open(outputDir, nil, nil, options, nil)
require.NoError(t, err) require.NoError(t, err)
defer func() { defer func() {

29
tsdb/db.go
View file

@ -933,21 +933,6 @@ func open(dir string, l *slog.Logger, r prometheus.Registerer, opts *Options, rn
db.blockChunkQuerierFunc = opts.BlockChunkQuerierFunc db.blockChunkQuerierFunc = opts.BlockChunkQuerierFunc
} }
// store the current blocksToDeleteFunc and replace it with a startup specific one.
originalBlocksToDelete := db.blocksToDelete
db.blocksToDelete = func(blocks []*Block) map[ulid.ULID]struct{} {
return deletableBlocks(db, blocks, beyondStartupTimeRetention, BeyondSizeRetention)
}
// Reload blocks so the retention policy is applied to the blocks.
// based on the current blocksToDeleteFunc.
err = db.reloadBlocks()
if err != nil {
return nil, err
}
// Restore the original blocksToDeleteFunc.
db.blocksToDelete = originalBlocksToDelete
var wal, wbl *wlog.WL var wal, wbl *wlog.WL
segmentSize := wlog.DefaultSegmentSize segmentSize := wlog.DefaultSegmentSize
// Wal is enabled. // Wal is enabled.
@ -1017,9 +1002,21 @@ func open(dir string, l *slog.Logger, r prometheus.Registerer, opts *Options, rn
db.metrics.maxBytes.Set(float64(maxBytes)) db.metrics.maxBytes.Set(float64(maxBytes))
db.metrics.retentionDuration.Set((time.Duration(opts.RetentionDuration) * time.Millisecond).Seconds()) db.metrics.retentionDuration.Set((time.Duration(opts.RetentionDuration) * time.Millisecond).Seconds())
// store the current blocksToDeleteFunc and replace it with a startup specific one.
originalBlocksToDelete := db.blocksToDelete
// Using a custom deletableBlocks so we can delete blocks that are beyond the startup time retention.
db.blocksToDelete = func(blocks []*Block) map[ulid.ULID]struct{} {
return deletableBlocks(db, blocks, beyondStartupTimeRetention, BeyondSizeRetention)
}
if err := db.reload(); err != nil { if err := db.reload(); err != nil {
return nil, err return nil, err
} }
// Restore the original blocksToDeleteFunc after applying the startup specific one.
db.blocksToDelete = originalBlocksToDelete
// Set the min valid time for the ingested samples // Set the min valid time for the ingested samples
// to be no lower than the maxt of the last block. // to be no lower than the maxt of the last block.
minValidTime := int64(math.MinInt64) minValidTime := int64(math.MinInt64)
@ -1771,7 +1768,7 @@ func beyondStartupTimeRetention(db *DB, blocks []*Block) (deletable map[ulid.ULI
deletable = make(map[ulid.ULID]struct{}) deletable = make(map[ulid.ULID]struct{})
for _, block := range blocks { for _, block := range blocks {
if block.Meta().MaxTime >= int64(db.opts.StartupMinRetentionTime) { if block.Meta().MaxTime >= db.opts.StartupMinRetentionTime {
for _, b := range blocks { for _, b := range blocks {
deletable[b.meta.ULID] = struct{}{} deletable[b.meta.ULID] = struct{}{}
} }