From 9f77d238891f42d8ba3c0144cb7f8d7c08b7e527 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Fri, 17 Jun 2022 06:56:19 +0100 Subject: [PATCH] tsdb: commit data periodically in CreateBlock (#10788) To avoid building up data in memory, commit and make a new appender periodically. The number `commitAfter = 10000` was chosen arbitrarily; testing with 10x more or less gives slightly worse results. Signed-off-by: Bryan Boreham --- tsdb/tsdbblockutil.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tsdb/tsdbblockutil.go b/tsdb/tsdbblockutil.go index 7324463247..a1dcd9a5f3 100644 --- a/tsdb/tsdbblockutil.go +++ b/tsdb/tsdbblockutil.go @@ -44,6 +44,8 @@ func CreateBlock(series []storage.Series, dir string, chunkRange int64, logger l } }() + sampleCount := 0 + const commitAfter = 10000 ctx := context.Background() app := w.Appender(ctx) @@ -57,10 +59,19 @@ func CreateBlock(series []storage.Series, dir string, chunkRange int64, logger l if err != nil { return "", err } + sampleCount++ } if it.Err() != nil { return "", it.Err() } + // Commit and make a new appender periodically, to avoid building up data in memory. + if sampleCount > commitAfter { + if err = app.Commit(); err != nil { + return "", err + } + app = w.Appender(ctx) + sampleCount = 0 + } } if err = app.Commit(); err != nil {