From afb5520bb197af61bfe930154dcd2d7f35d62042 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 30 Apr 2024 21:36:06 +0100 Subject: [PATCH] TSDB: compute size of name and value of each label Signed-off-by: Bryan Boreham --- model/labels/labels.go | 4 ++-- model/labels/labels_dedupelabels.go | 4 ++-- model/labels/labels_stringlabels.go | 4 ++-- tsdb/index/postings.go | 4 ++-- tsdb/index/postings_test.go | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/model/labels/labels.go b/model/labels/labels.go index 8aacd48010..be3eb7d3e5 100644 --- a/model/labels/labels.go +++ b/model/labels/labels.go @@ -490,6 +490,6 @@ func (b *ScratchBuilder) Overwrite(ls *Labels) { } // SizeOfLabels returns the approximate space required for n copies of a label. -func SizeOfLabels(value string, n uint64) uint64 { - return (uint64(len(value)) + uint64(unsafe.Sizeof(value))) * n +func SizeOfLabels(name, value string, n uint64) uint64 { + return (uint64(len(name)) + uint64(unsafe.Sizeof(name)) + uint64(len(value)) + uint64(unsafe.Sizeof(value))) * n } diff --git a/model/labels/labels_dedupelabels.go b/model/labels/labels_dedupelabels.go index 0793292c0a..10b4bfc75f 100644 --- a/model/labels/labels_dedupelabels.go +++ b/model/labels/labels_dedupelabels.go @@ -817,6 +817,6 @@ func (b *ScratchBuilder) Overwrite(ls *Labels) { } // SizeOfLabels returns the approximate space required for n copies of a label. -func SizeOfLabels(value string, n uint64) uint64 { - return uint64(len(value)) + n*2 // Assuming most symbol-table entries are 2 bytes long. +func SizeOfLabels(name, value string, n uint64) uint64 { + return uint64(len(name)+len(value)) + n*4 // Assuming most symbol-table entries are 2 bytes long. } diff --git a/model/labels/labels_stringlabels.go b/model/labels/labels_stringlabels.go index 6da5e31762..341fd362b9 100644 --- a/model/labels/labels_stringlabels.go +++ b/model/labels/labels_stringlabels.go @@ -696,6 +696,6 @@ func (b *ScratchBuilder) SetSymbolTable(_ *SymbolTable) { } // SizeOfLabels returns the approximate space required for n copies of a label. -func SizeOfLabels(value string, n uint64) uint64 { - return uint64(len(value)+sizeVarint(uint64(len(value)))) * n +func SizeOfLabels(name, value string, n uint64) uint64 { + return uint64(len(name)+sizeVarint(uint64(len(name)))+len(value)+sizeVarint(uint64(len(value)))) * n } diff --git a/tsdb/index/postings.go b/tsdb/index/postings.go index 4a01f7152b..aa152ce8c0 100644 --- a/tsdb/index/postings.go +++ b/tsdb/index/postings.go @@ -164,7 +164,7 @@ type PostingsStats struct { // Stats calculates the cardinality statistics from postings. // Caller can pass in a function which computes the space required for n series with a given label. -func (p *MemPostings) Stats(label string, limit int, bytes func(string, uint64) uint64) *PostingsStats { +func (p *MemPostings) Stats(label string, limit int, labelSizeFunc func(string, string, uint64) uint64) *PostingsStats { var size uint64 p.mtx.RLock() @@ -192,7 +192,7 @@ func (p *MemPostings) Stats(label string, limit int, bytes func(string, uint64) } seriesCnt := uint64(len(values)) labelValuePairs.push(Stat{Name: n + "=" + name, Count: seriesCnt}) - size += bytes(name, seriesCnt) + size += labelSizeFunc(n, name, seriesCnt) } labelValueLength.push(Stat{Name: n, Count: size}) } diff --git a/tsdb/index/postings_test.go b/tsdb/index/postings_test.go index ee523a6c84..9f1cd40d80 100644 --- a/tsdb/index/postings_test.go +++ b/tsdb/index/postings_test.go @@ -954,7 +954,7 @@ func TestMemPostingsStats(t *testing.T) { p.Add(2, labels.FromStrings("label", "value1")) // call the Stats method to calculate the cardinality statistics - stats := p.Stats("label", 10, func(s string, n uint64) uint64 { return uint64(len(s)) * n }) + stats := p.Stats("label", 10, func(name, value string, n uint64) uint64 { return uint64(len(name)+len(value)) * n }) // assert that the expected statistics were calculated require.Equal(t, uint64(2), stats.CardinalityMetricsStats[0].Count) @@ -963,7 +963,7 @@ func TestMemPostingsStats(t *testing.T) { require.Equal(t, uint64(3), stats.CardinalityLabelStats[0].Count) require.Equal(t, "label", stats.CardinalityLabelStats[0].Name) - require.Equal(t, uint64(24), stats.LabelValueStats[0].Count) + require.Equal(t, uint64(44), stats.LabelValueStats[0].Count) require.Equal(t, "label", stats.LabelValueStats[0].Name) require.Equal(t, uint64(2), stats.LabelValuePairsStats[0].Count)