mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2025-01-11 13:57:36 -08:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

tsdb: commit data periodically in CreateBlock (#10788)

Browse source 
To avoid building up data in memory, commit and make a new appender
periodically.

The number `commitAfter = 10000` was chosen arbitrarily; testing with
10x more or less gives slightly worse results.

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
This commit is contained in:
Bryan Boreham 2022-06-17 06:56:19 +01:00 committed by GitHub
parent d65f037def
commit 9f77d23889
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 11 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
tsdb/tsdbblockutil.go
View file

@ -44,6 +44,8 @@ func CreateBlock(series []storage.Series, dir string, chunkRange int64, logger l
} }
}() }()
sampleCount := 0
const commitAfter = 10000
ctx := context.Background() ctx := context.Background()
app := w.Appender(ctx) app := w.Appender(ctx)
@ -57,10 +59,19 @@ func CreateBlock(series []storage.Series, dir string, chunkRange int64, logger l
if err != nil { if err != nil {
return "", err return "", err
} }
sampleCount++
} }
if it.Err() != nil { if it.Err() != nil {
return "", it.Err() return "", it.Err()
} }
// Commit and make a new appender periodically, to avoid building up data in memory.
if sampleCount > commitAfter {
if err = app.Commit(); err != nil {
return "", err
}
app = w.Appender(ctx)
sampleCount = 0
}
} }
if err = app.Commit(); err != nil { if err = app.Commit(); err != nil {