Add hidden flag for the delayed compaction random time window

Signed-off-by: Alban HURTAUD <alban.hurtaud@amadeus.com>

cmd/prometheus/main.go

@@ -392,6 +392,9 @@ func main() {
 	serverOnlyFlag(a, "storage.tsdb.samples-per-chunk", "Target number of samples per chunk.").
 		Default("120").Hidden().IntVar(&cfg.tsdb.SamplesPerChunk)
 
+	serverOnlyFlag(a, "storage.tsdb.delayed-compaction.range-percent", "Percentage of the head chunk window to generate a delay for the compaction. 100 can delayed up to the full range (default 10)").
+		Default("10").Hidden().IntVar(&cfg.tsdb.CompactDelayPercentageRange)
+
 	agentOnlyFlag(a, "storage.agent.path", "Base path for metrics storage.").
 		Default("data-agent/").StringVar(&cfg.agentStoragePath)
 
@@ -616,6 +619,12 @@ func main() {
 
 			cfg.tsdb.MaxBlockDuration = maxBlockDuration
 		}
+
+		// Delayed compaction checks
+		if cfg.tsdb.EnableDelayedCompaction && (cfg.tsdb.CompactDelayPercentageRange >= 100 || cfg.tsdb.CompactDelayPercentageRange < 1) {
+			cfg.tsdb.CompactDelayPercentageRange = tsdb.DefaultCompactDelayPercentageRange
+			level.Warn(logger).Log("msg", "The --storage.tsdb.delayed-compaction.range-percent should have a value between 1 and 100. Using default 10%")
+		}
 	}
 
 	noStepSubqueryInterval := &safePromQLNoStepSubqueryInterval{}
@@ -1734,6 +1743,7 @@ type tsdbOptions struct {
 	EnableMemorySnapshotOnShutdown bool
 	EnableNativeHistograms         bool
 	EnableDelayedCompaction        bool
+	CompactDelayPercentageRange    int
 	EnableOverlappingCompaction    bool
 }
 
@@ -1756,6 +1766,7 @@ func (opts tsdbOptions) ToTSDBOptions() tsdb.Options {
 		EnableNativeHistograms:         opts.EnableNativeHistograms,
 		OutOfOrderTimeWindow:           opts.OutOfOrderTimeWindow,
 		EnableDelayedCompaction:        opts.EnableDelayedCompaction,
+		CompactDelayPercentageRange:    opts.CompactDelayPercentageRange,
 		EnableOverlappingCompaction:    opts.EnableOverlappingCompaction,
 	}
 }

tsdb/db.go

@@ -52,6 +52,9 @@ const (
 	// DefaultBlockDuration in milliseconds.
 	DefaultBlockDuration = int64(2 * time.Hour / time.Millisecond)
 
+	// DefaultCompactDelayPercentageRange in percentage.
+	DefaultCompactDelayPercentageRange = 10
+
 	// Block dir suffixes to make deletion and creation operations atomic.
 	// We decided to do suffixes instead of creating meta.json as last (or delete as first) one,
 	// because in error case you still can recover meta.json from the block content within local TSDB dir.
@@ -86,6 +89,7 @@ func DefaultOptions() *Options {
 		EnableOverlappingCompaction: true,
 		EnableSharding:              false,
 		EnableDelayedCompaction:     false,
+		CompactDelayPercentageRange: DefaultCompactDelayPercentageRange,
 		CompactionDelay:             time.Duration(0),
 	}
 }
@@ -198,6 +202,8 @@ type Options struct {
 	// CompactionDelay delays the start time of auto compactions.
 	// It can be increased by up to one minute if the DB does not commit too often.
 	CompactionDelay time.Duration
+	// CompactDelayPercentageRange is the percentage of the chunk range window the random delay will be generated for the compaction.
+	CompactDelayPercentageRange int
 
 	// NewCompactorFunc is a function that returns a TSDB compactor.
 	NewCompactorFunc NewCompactorFunc
@@ -1969,8 +1975,7 @@ func (db *DB) EnableCompactions() {
 }
 
 func (db *DB) generateCompactionDelay() time.Duration {
-	// Up to 10% of the head's chunkRange.
-	return time.Duration(rand.Int63n(db.head.chunkRange.Load()/10)) * time.Millisecond
+	return time.Duration(rand.Int63n(db.head.chunkRange.Load()*int64(db.opts.CompactDelayPercentageRange)/100)) * time.Millisecond
 }
 
 // ForceHeadMMap is intended for use only in tests and benchmarks.