Increase block writer size for backfilling

Signed-off-by: Ganesh Vernekar <cs15btech11018@iith.ac.in>
This commit is contained in:
Ganesh Vernekar 2021-02-16 13:36:02 +05:30
parent 6393b48929
commit c4536fa28c
No known key found for this signature in database
GPG key ID: 0241A11211763456
2 changed files with 52 additions and 1 deletions

View file

@ -93,7 +93,13 @@ func createBlocks(input []byte, mint, maxt int64, maxSamplesInAppender int, outp
nextSampleTs = math.MaxInt64 nextSampleTs = math.MaxInt64
err := func() error { err := func() error {
w, err := tsdb.NewBlockWriter(log.NewNopLogger(), outputDir, blockDuration) // To prevent races with compaction, a block writer only allows appending samples
// that are at most half a block size older than the most recent sample appended so far.
// However, in the way we use the block writer here, compaction doesn't happen, while we
// also need to append samples throughout the whole block range. To allow that, we
// pretend that the block is twice as large here, but only really add sample in the
// original interval later.
w, err := tsdb.NewBlockWriter(log.NewNopLogger(), outputDir, 2*blockDuration)
if err != nil { if err != nil {
return errors.Wrap(err, "block writer") return errors.Wrap(err, "block writer")
} }

View file

@ -365,6 +365,51 @@ http_requests_total{code="400"} 1 1565166113.989
}, },
}, },
}, },
{ // For https://github.com/prometheus/prometheus/issues/8476.
ToParse: `# HELP http_requests_total The total number of HTTP requests.
# TYPE http_requests_total counter
http_requests_total{code="200"} 1021 0
http_requests_total{code="200"} 1022 7199
http_requests_total{code="400"} 1023 0
http_requests_total{code="400"} 1024 7199
# EOF
`,
IsOk: true,
Description: "One series spanning 2h in same block should not cause problems to other series.",
MaxSamplesInAppender: 1,
Expected: struct {
MinTime int64
MaxTime int64
NumBlocks int
Samples []backfillSample
}{
MinTime: 0,
MaxTime: 7199000,
NumBlocks: 1,
Samples: []backfillSample{
{
Timestamp: 0,
Value: 1021,
Labels: labels.FromStrings("__name__", "http_requests_total", "code", "200"),
},
{
Timestamp: 7199000,
Value: 1022,
Labels: labels.FromStrings("__name__", "http_requests_total", "code", "200"),
},
{
Timestamp: 0,
Value: 1023,
Labels: labels.FromStrings("__name__", "http_requests_total", "code", "400"),
},
{
Timestamp: 7199000,
Value: 1024,
Labels: labels.FromStrings("__name__", "http_requests_total", "code", "400"),
},
},
},
},
{ {
ToParse: `no_help_no_type{foo="bar"} 42 6900 ToParse: `no_help_no_type{foo="bar"} 42 6900
# EOF # EOF