mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2024-12-25 05:34:05 -08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

New cases in Test_ChunkQuerier_OOOQuery and Test_Querier_OOOQuery

Browse source 
Case 1: OOO in-memory head chunk overlaps with first mmaped in-order chunk.

Query: |----------------------------------------------------------------|
InO:    |------mmap---------------||---------mem----------------------|
OOO:     |-----mem-----------|

This triggers ChunkOrIterableWithCopy not including OOO head chunks bug.

Similar to #14693 however testing the end of the interval doesn't
trigger the problem because there the in-order head chunk will be
trimmed with a tombstone, causing the code to switch to ChunkOrIterable
which was fixed.
See a36d1a8a92/tsdb/querier.go (L646)
where len(p.bufIter.Intervals) will be non zero, because it includes the
tombstone to trim the result to the query max time.

Thus a new test is added to check the overlap at the beginning of the
interval that has a separate chunk, which does not need trimming.

Note: same test doesn't fail for sample querier in Test_Querier_OOOQuery
as that doesn't use copy, that is copyHeadChunk is false in the if
condition above.

Case 2:

OOO mmaped head chunk overlaps with first mmaped in-order chunk.

Query: |----------------------------------------------------------------|
InO:    |------mmap---------------||---------mem----------------------|
OOO:     |-----mmap-----------|                             |--mem--|

In this case the meta contains the reference of the in-order chunk and
no indication that a merge is needed with the OOO mmaped chunk.

Signed-off-by: György Krajcsovits <gyorgy.krajcsovits@grafana.com>
This commit is contained in:
György Krajcsovits 2024-08-22 11:36:47 +02:00
parent 0538ad3a08
commit 41c076196e
1 changed files with 304 additions and 73 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

377
tsdb/db_test.go
View file

@ -5036,16 +5036,15 @@ func testOOOQueryAfterRestartWithSnapshotAndRemovedWBL(t *testing.T, scenario sa
func Test_Querier_OOOQuery(t *testing.T) {
opts := DefaultOptions()
opts.OutOfOrderCapMax = 30
opts.OutOfOrderTimeWindow = 24 * time.Hour.Milliseconds()
series1 := labels.FromStrings("foo", "bar1")
type filterFunc func(t int64) bool
defaultFilterFunc := func(t int64) bool { return true }
minutes := func(m int64) int64 { return m * time.Minute.Milliseconds() }
addSample := func(db *DB, fromMins, toMins, queryMinT, queryMaxT int64, expSamples []chunks.Sample, filter func(int64) bool) ([]chunks.Sample, int) {
if filter == nil {
filter = func(int64) bool { return true }
}
addSample := func(db *DB, fromMins, toMins, queryMinT, queryMaxT int64, expSamples []chunks.Sample, filter filterFunc) ([]chunks.Sample, int) {
app := db.Appender(context.Background())
totalAppended := 0
for m := fromMins; m <= toMins; m += time.Minute.Milliseconds() {
@ -5060,68 +5059,173 @@ func Test_Querier_OOOQuery(t *testing.T) {
totalAppended++
}
require.NoError(t, app.Commit())
require.Positive(t, totalAppended, 0) // Sanity check that filter is not too zealous.
return expSamples, totalAppended
}
type sampleBatch struct {
minT int64
maxT int64
filter filterFunc
isOOO bool
}
tests := []struct {
name string
queryMinT int64
queryMaxT int64
inOrderMinT int64
inOrderMaxT int64
oooMinT int64
oooMaxT int64
name string
oooCap int64
queryMinT int64
queryMaxT int64
batches []sampleBatch
}{
{
name: "query interval covering ooomint and inordermaxt returns all ingested samples",
queryMinT: minutes(0),
queryMaxT: minutes(200),
inOrderMinT: minutes(100),
inOrderMaxT: minutes(200),
oooMinT: minutes(0),
oooMaxT: minutes(99),
name: "query interval covering ooomint and inordermaxt returns all ingested samples",
oooCap: 30,
queryMinT: minutes(0),
queryMaxT: minutes(200),
batches: []sampleBatch{
{
minT: minutes(100),
maxT: minutes(200),
filter: defaultFilterFunc,
},
{
minT: minutes(0),
maxT: minutes(99),
filter: defaultFilterFunc,
isOOO: true,
},
},
},
{
name: "partial query interval returns only samples within interval",
queryMinT: minutes(20),
queryMaxT: minutes(180),
inOrderMinT: minutes(100),
inOrderMaxT: minutes(200),
oooMinT: minutes(0),
oooMaxT: minutes(99),
name: "partial query interval returns only samples within interval",
oooCap: 30,
queryMinT: minutes(20),
queryMaxT: minutes(180),
batches: []sampleBatch{
{
minT: minutes(100),
maxT: minutes(200),
filter: defaultFilterFunc,
},
{
minT: minutes(0),
maxT: minutes(99),
filter: defaultFilterFunc,
isOOO: true,
},
},
},
{
name: "query overlapping inorder and ooo samples returns all ingested samples",
queryMinT: minutes(0),
queryMaxT: minutes(200),
inOrderMinT: minutes(100),
inOrderMaxT: minutes(200),
oooMinT: minutes(180 - opts.OutOfOrderCapMax/2), // Make sure to fit into the OOO head.
oooMaxT: minutes(180),
name: "query overlapping inorder and ooo samples returns all ingested samples at the end of the interval",
oooCap: 30,
queryMinT: minutes(0),
queryMaxT: minutes(200),
batches: []sampleBatch{
{
minT: minutes(100),
maxT: minutes(200),
filter: func(t int64) bool { return t%2 == 0 },
isOOO: false,
},
{
minT: minutes(170),
maxT: minutes(180),
filter: func(t int64) bool { return t%2 == 1 },
isOOO: true,
},
},
},
{
name: "query overlapping inorder and ooo in-memory samples returns all ingested samples at the beginning of the interval",
oooCap: 30,
queryMinT: minutes(0),
queryMaxT: minutes(200),
batches: []sampleBatch{
{
minT: minutes(100),
maxT: minutes(200),
filter: func(t int64) bool { return t%2 == 0 },
isOOO: false,
},
{
minT: minutes(100),
maxT: minutes(110),
filter: func(t int64) bool { return t%2 == 1 },
isOOO: true,
},
},
},
{
name: "query inorder contain ooo mmaped samples returns all ingested samples at the beginning of the interval",
oooCap: 5,
queryMinT: minutes(0),
queryMaxT: minutes(200),
batches: []sampleBatch{
{
minT: minutes(100),
maxT: minutes(200),
filter: func(t int64) bool { return t%2 == 0 },
isOOO: false,
},
{
minT: minutes(101),
maxT: minutes(101 + (5-1)*2), // Append samples to fit in a single mmmaped OOO chunk and fit inside the first in-order mmaped chunk.
filter: func(t int64) bool { return t%2 == 1 },
isOOO: true,
},
{
minT: minutes(191),
maxT: minutes(193), // Append some more OOO samples to trigger mapping the OOO chunk, but use time 151 to not overlap with in-order head chunk.
filter: func(t int64) bool { return t%2 == 1 },
isOOO: true,
},
},
},
{
name: "query overlapping inorder and ooo mmaped samples returns all ingested samples at the beginning of the interval",
oooCap: 30,
queryMinT: minutes(0),
queryMaxT: minutes(200),
batches: []sampleBatch{
{
minT: minutes(100),
maxT: minutes(200),
filter: func(t int64) bool { return t%2 == 0 },
isOOO: false,
},
{
minT: minutes(101),
maxT: minutes(101 + (30-1)*2), // Append samples to fit in a single mmmaped OOO chunk and overlap the first in-order mmaped chunk.
filter: func(t int64) bool { return t%2 == 1 },
isOOO: true,
},
{
minT: minutes(191),
maxT: minutes(193), // Append some more OOO samples to trigger mapping the OOO chunk, but use time 151 to not overlap with in-order head chunk.
filter: func(t int64) bool { return t%2 == 1 },
isOOO: true,
},
},
},
}
for _, tc := range tests {
t.Run(fmt.Sprintf("name=%s", tc.name), func(t *testing.T) {
opts.OutOfOrderCapMax = tc.oooCap
db := openTestDB(t, opts, nil)
db.DisableCompactions()
defer func() {
require.NoError(t, db.Close())
}()
var (
expSamples []chunks.Sample
inoSamples int
)
var expSamples []chunks.Sample
var oooSamples, appendedCount int
// Add in-order samples (at even minutes).
expSamples, inoSamples = addSample(db, tc.inOrderMinT, tc.inOrderMaxT, tc.queryMinT, tc.queryMaxT, expSamples, func(t int64) bool { return t%2 == 0 })
// Sanity check that filter is not too zealous.
require.Positive(t, inoSamples, 0)
// Add out-of-order samples (at odd minutes).
expSamples, oooSamples := addSample(db, tc.oooMinT, tc.oooMaxT, tc.queryMinT, tc.queryMaxT, expSamples, func(t int64) bool { return t%2 == 1 })
// Sanity check that filter is not too zealous.
require.Positive(t, oooSamples, 0)
for _, batch := range tc.batches {
expSamples, appendedCount = addSample(db, batch.minT, batch.maxT, tc.queryMinT, tc.queryMaxT, expSamples, batch.filter)
if batch.isOOO {
oooSamples += appendedCount
}
}
sort.Slice(expSamples, func(i, j int) bool {
return expSamples[i].T() < expSamples[j].T()
@ -5147,11 +5251,17 @@ func Test_ChunkQuerier_OOOQuery(t *testing.T) {
series1 := labels.FromStrings("foo", "bar1")
type filterFunc func(t int64) bool
defaultFilterFunc := func(t int64) bool { return true }
minutes := func(m int64) int64 { return m * time.Minute.Milliseconds() }
addSample := func(db *DB, fromMins, toMins, queryMinT, queryMaxT int64, expSamples []chunks.Sample) ([]chunks.Sample, int) {
addSample := func(db *DB, fromMins, toMins, queryMinT, queryMaxT int64, expSamples []chunks.Sample, filter filterFunc) ([]chunks.Sample, int) {
app := db.Appender(context.Background())
totalAppended := 0
for m := fromMins; m <= toMins; m += time.Minute.Milliseconds() {
if !filter(m / time.Minute.Milliseconds()) {
continue
}
_, err := app.Append(0, series1, m, float64(m))
if m >= queryMinT && m <= queryMaxT {
expSamples = append(expSamples, sample{t: m, f: float64(m)})
@ -5160,39 +5270,158 @@ func Test_ChunkQuerier_OOOQuery(t *testing.T) {
totalAppended++
}
require.NoError(t, app.Commit())
require.Positive(t, totalAppended) // Sanity check that filter is not too zealous.
return expSamples, totalAppended
}
type sampleBatch struct {
minT int64
maxT int64
filter filterFunc
isOOO bool
}
tests := []struct {
name string
queryMinT int64
queryMaxT int64
inOrderMinT int64
inOrderMaxT int64
oooMinT int64
oooMaxT int64
name string
oooCap int64
queryMinT int64
queryMaxT int64
batches []sampleBatch
}{
{
name: "query interval covering ooomint and inordermaxt returns all ingested samples",
queryMinT: minutes(0),
queryMaxT: minutes(200),
inOrderMinT: minutes(100),
inOrderMaxT: minutes(200),
oooMinT: minutes(0),
oooMaxT: minutes(99),
name: "query interval covering ooomint and inordermaxt returns all ingested samples",
oooCap: 30,
queryMinT: minutes(0),
queryMaxT: minutes(200),
batches: []sampleBatch{
{
minT: minutes(100),
maxT: minutes(200),
filter: defaultFilterFunc,
},
{
minT: minutes(0),
maxT: minutes(99),
filter: defaultFilterFunc,
isOOO: true,
},
},
},
{
name: "partial query interval returns only samples within interval",
queryMinT: minutes(20),
queryMaxT: minutes(180),
inOrderMinT: minutes(100),
inOrderMaxT: minutes(200),
oooMinT: minutes(0),
oooMaxT: minutes(99),
name: "partial query interval returns only samples within interval",
oooCap: 30,
queryMinT: minutes(20),
queryMaxT: minutes(180),
batches: []sampleBatch{
{
minT: minutes(100),
maxT: minutes(200),
filter: defaultFilterFunc,
},
{
minT: minutes(0),
maxT: minutes(99),
filter: defaultFilterFunc,
isOOO: true,
},
},
},
{
name: "query overlapping inorder and ooo samples returns all ingested samples at the end of the interval",
oooCap: 30,
queryMinT: minutes(0),
queryMaxT: minutes(200),
batches: []sampleBatch{
{
minT: minutes(100),
maxT: minutes(200),
filter: func(t int64) bool { return t%2 == 0 },
isOOO: false,
},
{
minT: minutes(170),
maxT: minutes(180),
filter: func(t int64) bool { return t%2 == 1 },
isOOO: true,
},
},
},
{
name: "query overlapping inorder and ooo in-memory samples returns all ingested samples at the beginning of the interval",
oooCap: 30,
queryMinT: minutes(0),
queryMaxT: minutes(200),
batches: []sampleBatch{
{
minT: minutes(100),
maxT: minutes(200),
filter: func(t int64) bool { return t%2 == 0 },
isOOO: false,
},
{
minT: minutes(100),
maxT: minutes(110),
filter: func(t int64) bool { return t%2 == 1 },
isOOO: true,
},
},
},
{
name: "query inorder contain ooo mmaped samples returns all ingested samples at the beginning of the interval",
oooCap: 5,
queryMinT: minutes(0),
queryMaxT: minutes(200),
batches: []sampleBatch{
{
minT: minutes(100),
maxT: minutes(200),
filter: func(t int64) bool { return t%2 == 0 },
isOOO: false,
},
{
minT: minutes(101),
maxT: minutes(101 + (5-1)*2), // Append samples to fit in a single mmmaped OOO chunk and fit inside the first in-order mmaped chunk.
filter: func(t int64) bool { return t%2 == 1 },
isOOO: true,
},
{
minT: minutes(191),
maxT: minutes(193), // Append some more OOO samples to trigger mapping the OOO chunk, but use time 151 to not overlap with in-order head chunk.
filter: func(t int64) bool { return t%2 == 1 },
isOOO: true,
},
},
},
{
name: "query overlapping inorder and ooo mmaped samples returns all ingested samples at the beginning of the interval",
oooCap: 30,
queryMinT: minutes(0),
queryMaxT: minutes(200),
batches: []sampleBatch{
{
minT: minutes(100),
maxT: minutes(200),
filter: func(t int64) bool { return t%2 == 0 },
isOOO: false,
},
{
minT: minutes(101),
maxT: minutes(101 + (30-1)*2), // Append samples to fit in a single mmmaped OOO chunk and overlap the first in-order mmaped chunk.
filter: func(t int64) bool { return t%2 == 1 },
isOOO: true,
},
{
minT: minutes(191),
maxT: minutes(193), // Append some more OOO samples to trigger mapping the OOO chunk, but use time 151 to not overlap with in-order head chunk.
filter: func(t int64) bool { return t%2 == 1 },
isOOO: true,
},
},
},
}
for _, tc := range tests {
t.Run(fmt.Sprintf("name=%s", tc.name), func(t *testing.T) {
opts.OutOfOrderCapMax = tc.oooCap
db := openTestDB(t, opts, nil)
db.DisableCompactions()
defer func() {
@ -5200,12 +5429,14 @@ func Test_ChunkQuerier_OOOQuery(t *testing.T) {
}()
var expSamples []chunks.Sample
var oooSamples, appendedCount int
// Add in-order samples.
expSamples, _ = addSample(db, tc.inOrderMinT, tc.inOrderMaxT, tc.queryMinT, tc.queryMaxT, expSamples)
// Add out-of-order samples.
expSamples, oooSamples := addSample(db, tc.oooMinT, tc.oooMaxT, tc.queryMinT, tc.queryMaxT, expSamples)
for _, batch := range tc.batches {
expSamples, appendedCount = addSample(db, batch.minT, batch.maxT, tc.queryMinT, tc.queryMaxT, expSamples, batch.filter)
if batch.isOOO {
oooSamples += appendedCount
}
}
sort.Slice(expSamples, func(i, j int) bool {
return expSamples[i].T() < expSamples[j].T()