diff --git a/cmd/promtool/tsdb.go b/cmd/promtool/tsdb.go index 6a62e2e8bc..d088ec28d1 100644 --- a/cmd/promtool/tsdb.go +++ b/cmd/promtool/tsdb.go @@ -553,7 +553,7 @@ func analyzeBlock(ctx context.Context, path, blockID string, limit int, runExten postingInfos = postingInfos[:0] for _, n := range allLabelNames { - values, err := ir.SortedLabelValues(ctx, n, selectors...) + values, err := ir.SortedLabelValues(ctx, n, &storage.LabelHints{}, selectors...) if err != nil { return err } @@ -569,7 +569,7 @@ func analyzeBlock(ctx context.Context, path, blockID string, limit int, runExten postingInfos = postingInfos[:0] for _, n := range allLabelNames { - lv, err := ir.SortedLabelValues(ctx, n, selectors...) + lv, err := ir.SortedLabelValues(ctx, n, &storage.LabelHints{}, selectors...) if err != nil { return err } @@ -579,7 +579,7 @@ func analyzeBlock(ctx context.Context, path, blockID string, limit int, runExten printInfo(postingInfos) postingInfos = postingInfos[:0] - lv, err := ir.SortedLabelValues(ctx, "__name__", selectors...) + lv, err := ir.SortedLabelValues(ctx, "__name__", &storage.LabelHints{}, selectors...) if err != nil { return err } diff --git a/tsdb/block.go b/tsdb/block.go index ae20c174d6..1c0ee70fff 100644 --- a/tsdb/block.go +++ b/tsdb/block.go @@ -67,10 +67,10 @@ type IndexReader interface { Symbols() index.StringIter // SortedLabelValues returns sorted possible label values. - SortedLabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, error) + SortedLabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) // LabelValues returns possible label values which may not be sorted. - LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, error) + LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) // Postings returns the postings list iterator for the label pairs. // The Postings here contain the offsets to the series inside the index. @@ -476,14 +476,14 @@ func (r blockIndexReader) Symbols() index.StringIter { return r.ir.Symbols() } -func (r blockIndexReader) SortedLabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, error) { +func (r blockIndexReader) SortedLabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) { var st []string var err error if len(matchers) == 0 { - st, err = r.ir.SortedLabelValues(ctx, name) + st, err = r.ir.SortedLabelValues(ctx, name, hints) } else { - st, err = r.LabelValues(ctx, name, matchers...) + st, err = r.LabelValues(ctx, name, hints, matchers...) if err == nil { slices.Sort(st) } @@ -494,16 +494,16 @@ func (r blockIndexReader) SortedLabelValues(ctx context.Context, name string, ma return st, nil } -func (r blockIndexReader) LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, error) { +func (r blockIndexReader) LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) { if len(matchers) == 0 { - st, err := r.ir.LabelValues(ctx, name) + st, err := r.ir.LabelValues(ctx, name, hints) if err != nil { return st, fmt.Errorf("block: %s: %w", r.b.Meta().ULID, err) } return st, nil } - return labelValuesWithMatchers(ctx, r.ir, name, matchers...) + return labelValuesWithMatchers(ctx, r.ir, name, hints, matchers...) } func (r blockIndexReader) LabelNames(ctx context.Context, matchers ...*labels.Matcher) ([]string, error) { diff --git a/tsdb/block_test.go b/tsdb/block_test.go index 776beb4396..f33607f32d 100644 --- a/tsdb/block_test.go +++ b/tsdb/block_test.go @@ -299,11 +299,11 @@ func TestLabelValuesWithMatchers(t *testing.T) { for _, tt := range testCases { t.Run(tt.name, func(t *testing.T) { - actualValues, err := indexReader.SortedLabelValues(ctx, tt.labelName, tt.matchers...) + actualValues, err := indexReader.SortedLabelValues(ctx, tt.labelName, &storage.LabelHints{}, tt.matchers...) require.NoError(t, err) require.Equal(t, tt.expectedValues, actualValues) - actualValues, err = indexReader.LabelValues(ctx, tt.labelName, tt.matchers...) + actualValues, err = indexReader.LabelValues(ctx, tt.labelName, &storage.LabelHints{}, tt.matchers...) sort.Strings(actualValues) require.NoError(t, err) require.Equal(t, tt.expectedValues, actualValues) @@ -459,7 +459,7 @@ func BenchmarkLabelValuesWithMatchers(b *testing.B) { b.ReportAllocs() for benchIdx := 0; benchIdx < b.N; benchIdx++ { - actualValues, err := indexReader.LabelValues(ctx, "b_tens", matchers...) + actualValues, err := indexReader.LabelValues(ctx, "b_tens", &storage.LabelHints{}, matchers...) require.NoError(b, err) require.Len(b, actualValues, 9) } diff --git a/tsdb/head_read.go b/tsdb/head_read.go index f37fd17d60..20495c30b3 100644 --- a/tsdb/head_read.go +++ b/tsdb/head_read.go @@ -61,8 +61,8 @@ func (h *headIndexReader) Symbols() index.StringIter { // specific label name that are within the time range mint to maxt. // If matchers are specified the returned result set is reduced // to label values of metrics matching the matchers. -func (h *headIndexReader) SortedLabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, error) { - values, err := h.LabelValues(ctx, name, matchers...) +func (h *headIndexReader) SortedLabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) { + values, err := h.LabelValues(ctx, name, hints, matchers...) if err == nil { slices.Sort(values) } @@ -73,16 +73,16 @@ func (h *headIndexReader) SortedLabelValues(ctx context.Context, name string, ma // specific label name that are within the time range mint to maxt. // If matchers are specified the returned result set is reduced // to label values of metrics matching the matchers. -func (h *headIndexReader) LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, error) { +func (h *headIndexReader) LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) { if h.maxt < h.head.MinTime() || h.mint > h.head.MaxTime() { return []string{}, nil } if len(matchers) == 0 { - return h.head.postings.LabelValues(ctx, name), nil + return h.head.postings.LabelValues(ctx, name, hints), nil } - return labelValuesWithMatchers(ctx, h, name, matchers...) + return labelValuesWithMatchers(ctx, h, name, hints, matchers...) } // LabelNames returns all the unique label names present in the head diff --git a/tsdb/head_test.go b/tsdb/head_test.go index 065e5ff008..e43e71997d 100644 --- a/tsdb/head_test.go +++ b/tsdb/head_test.go @@ -1007,7 +1007,7 @@ func TestHead_Truncate(t *testing.T) { ss = map[string]struct{}{} values[name] = ss } - for _, value := range h.postings.LabelValues(ctx, name) { + for _, value := range h.postings.LabelValues(ctx, name, &storage.LabelHints{}) { ss[value] = struct{}{} } } @@ -2929,7 +2929,7 @@ func TestHeadLabelNamesValuesWithMinMaxRange(t *testing.T) { require.Equal(t, tt.expectedNames, actualLabelNames) if len(tt.expectedValues) > 0 { for i, name := range expectedLabelNames { - actualLabelValue, err := headIdxReader.SortedLabelValues(ctx, name) + actualLabelValue, err := headIdxReader.SortedLabelValues(ctx, name, &storage.LabelHints{}) require.NoError(t, err) require.Equal(t, []string{tt.expectedValues[i]}, actualLabelValue) } @@ -3002,11 +3002,11 @@ func TestHeadLabelValuesWithMatchers(t *testing.T) { t.Run(tt.name, func(t *testing.T) { headIdxReader := head.indexRange(0, 200) - actualValues, err := headIdxReader.SortedLabelValues(ctx, tt.labelName, tt.matchers...) + actualValues, err := headIdxReader.SortedLabelValues(ctx, tt.labelName, &storage.LabelHints{}, tt.matchers...) require.NoError(t, err) require.Equal(t, tt.expectedValues, actualValues) - actualValues, err = headIdxReader.LabelValues(ctx, tt.labelName, tt.matchers...) + actualValues, err = headIdxReader.LabelValues(ctx, tt.labelName, &storage.LabelHints{}, tt.matchers...) sort.Strings(actualValues) require.NoError(t, err) require.Equal(t, tt.expectedValues, actualValues) @@ -3265,7 +3265,7 @@ func BenchmarkHeadLabelValuesWithMatchers(b *testing.B) { b.ReportAllocs() for benchIdx := 0; benchIdx < b.N; benchIdx++ { - actualValues, err := headIdxReader.LabelValues(ctx, "b_tens", matchers...) + actualValues, err := headIdxReader.LabelValues(ctx, "b_tens", &storage.LabelHints{}, matchers...) require.NoError(b, err) require.Len(b, actualValues, 9) } diff --git a/tsdb/index/index.go b/tsdb/index/index.go index 42ecd7245d..5ee5835d3b 100644 --- a/tsdb/index/index.go +++ b/tsdb/index/index.go @@ -1493,8 +1493,8 @@ func (r *Reader) SymbolTableSize() uint64 { // SortedLabelValues returns value tuples that exist for the given label name. // It is not safe to use the return value beyond the lifetime of the byte slice // passed into the Reader. -func (r *Reader) SortedLabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, error) { - values, err := r.LabelValues(ctx, name, matchers...) +func (r *Reader) SortedLabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) { + values, err := r.LabelValues(ctx, name, hints, matchers...) if err == nil && r.version == FormatV1 { slices.Sort(values) } @@ -1505,11 +1505,15 @@ func (r *Reader) SortedLabelValues(ctx context.Context, name string, matchers .. // It is not safe to use the return value beyond the lifetime of the byte slice // passed into the Reader. // TODO(replay): Support filtering by matchers. -func (r *Reader) LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, error) { +func (r *Reader) LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) { if len(matchers) > 0 { return nil, fmt.Errorf("matchers parameter is not implemented: %+v", matchers) } + if hints == nil { + hints = &storage.LabelHints{} + } + if r.version == FormatV1 { e, ok := r.postingsV1[name] if !ok { @@ -1529,9 +1533,16 @@ func (r *Reader) LabelValues(ctx context.Context, name string, matchers ...*labe return nil, nil } - values := make([]string, 0, len(e)*symbolFactor) + valuesLength := len(e) * symbolFactor + if hints.Limit > 0 && valuesLength > (hints.Limit*symbolFactor) { + valuesLength = hints.Limit * symbolFactor + } + values := make([]string, 0, valuesLength) lastVal := e[len(e)-1].value err := r.traversePostingOffsets(ctx, e[0].off, func(val string, _ uint64) (bool, error) { + if len(e) >= valuesLength { + return false, nil + } values = append(values, val) return val != lastVal, nil }) diff --git a/tsdb/index/index_test.go b/tsdb/index/index_test.go index ee186c1d95..c0f1807f87 100644 --- a/tsdb/index/index_test.go +++ b/tsdb/index/index_test.go @@ -421,7 +421,7 @@ func TestPersistence_index_e2e(t *testing.T) { for k, v := range labelPairs { sort.Strings(v) - res, err := ir.SortedLabelValues(ctx, k) + res, err := ir.SortedLabelValues(ctx, k, &storage.LabelHints{}) require.NoError(t, err) require.Equal(t, len(v), len(res)) diff --git a/tsdb/index/postings.go b/tsdb/index/postings.go index e3ba5d64b4..3661356c18 100644 --- a/tsdb/index/postings.go +++ b/tsdb/index/postings.go @@ -168,11 +168,19 @@ func (p *MemPostings) LabelNames() []string { } // LabelValues returns label values for the given name. -func (p *MemPostings) LabelValues(_ context.Context, name string) []string { +func (p *MemPostings) LabelValues(_ context.Context, name string, hints *storage.LabelHints) []string { p.mtx.RLock() values := p.lvs[name] p.mtx.RUnlock() + if hints == nil { + hints = &storage.LabelHints{} + } + + if hints.Limit > 0 && len(values) > hints.Limit { + values = values[:hints.Limit] + } + // The slice from p.lvs[name] is shared between all readers, and it is append-only. // Since it's shared, we need to make a copy of it before returning it to make // sure that no caller modifies the original one by sorting it or filtering it. diff --git a/tsdb/ooo_head_read.go b/tsdb/ooo_head_read.go index 6ab99a96e6..015847bf55 100644 --- a/tsdb/ooo_head_read.go +++ b/tsdb/ooo_head_read.go @@ -176,16 +176,16 @@ type multiMeta struct { // LabelValues needs to be overridden from the headIndexReader implementation // so we can return labels within either in-order range or ooo range. -func (oh *HeadAndOOOIndexReader) LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, error) { +func (oh *HeadAndOOOIndexReader) LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) { if oh.maxt < oh.head.MinTime() && oh.maxt < oh.head.MinOOOTime() || oh.mint > oh.head.MaxTime() && oh.mint > oh.head.MaxOOOTime() { return []string{}, nil } if len(matchers) == 0 { - return oh.head.postings.LabelValues(ctx, name), nil + return oh.head.postings.LabelValues(ctx, name, hints), nil } - return labelValuesWithMatchers(ctx, oh, name, matchers...) + return labelValuesWithMatchers(ctx, oh, name, hints, matchers...) } func lessByMinTimeAndMinRef(a, b chunks.Meta) int { @@ -484,11 +484,11 @@ func (ir *OOOCompactionHeadIndexReader) Series(ref storage.SeriesRef, builder *l return getOOOSeriesChunks(s, ir.ch.mint, ir.ch.maxt, 0, ir.ch.lastMmapRef, false, 0, chks) } -func (ir *OOOCompactionHeadIndexReader) SortedLabelValues(_ context.Context, _ string, _ ...*labels.Matcher) ([]string, error) { +func (ir *OOOCompactionHeadIndexReader) SortedLabelValues(_ context.Context, _ string, _ *storage.LabelHints, _ ...*labels.Matcher) ([]string, error) { return nil, errors.New("not implemented") } -func (ir *OOOCompactionHeadIndexReader) LabelValues(_ context.Context, _ string, _ ...*labels.Matcher) ([]string, error) { +func (ir *OOOCompactionHeadIndexReader) LabelValues(_ context.Context, _ string, _ *storage.LabelHints, _ ...*labels.Matcher) ([]string, error) { return nil, errors.New("not implemented") } diff --git a/tsdb/ooo_head_read_test.go b/tsdb/ooo_head_read_test.go index adbd3278ba..f1a7466214 100644 --- a/tsdb/ooo_head_read_test.go +++ b/tsdb/ooo_head_read_test.go @@ -453,24 +453,24 @@ func testOOOHeadChunkReader_LabelValues(t *testing.T, scenario sampleTypeScenari // We first want to test using a head index reader that covers the biggest query interval oh := NewHeadAndOOOIndexReader(head, tc.queryMinT, tc.queryMinT, tc.queryMaxT, 0) matchers := []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "foo", "bar1")} - values, err := oh.LabelValues(ctx, "foo", matchers...) + values, err := oh.LabelValues(ctx, "foo", &storage.LabelHints{}, matchers...) sort.Strings(values) require.NoError(t, err) require.Equal(t, tc.expValues1, values) matchers = []*labels.Matcher{labels.MustNewMatcher(labels.MatchNotRegexp, "foo", "^bar.")} - values, err = oh.LabelValues(ctx, "foo", matchers...) + values, err = oh.LabelValues(ctx, "foo", &storage.LabelHints{}, matchers...) sort.Strings(values) require.NoError(t, err) require.Equal(t, tc.expValues2, values) matchers = []*labels.Matcher{labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar.")} - values, err = oh.LabelValues(ctx, "foo", matchers...) + values, err = oh.LabelValues(ctx, "foo", &storage.LabelHints{}, matchers...) sort.Strings(values) require.NoError(t, err) require.Equal(t, tc.expValues3, values) - values, err = oh.LabelValues(ctx, "foo") + values, err = oh.LabelValues(ctx, "foo", &storage.LabelHints{}) sort.Strings(values) require.NoError(t, err) require.Equal(t, tc.expValues4, values) diff --git a/tsdb/querier.go b/tsdb/querier.go index 676bcf4627..56a09ad176 100644 --- a/tsdb/querier.go +++ b/tsdb/querier.go @@ -77,8 +77,8 @@ func newBlockBaseQuerier(b BlockReader, mint, maxt int64) (*blockBaseQuerier, er }, nil } -func (q *blockBaseQuerier) LabelValues(ctx context.Context, name string, _ *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { - res, err := q.index.SortedLabelValues(ctx, name, matchers...) +func (q *blockBaseQuerier) LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { + res, err := q.index.SortedLabelValues(ctx, name, hints, matchers...) return res, nil, err } @@ -390,8 +390,13 @@ func inversePostingsForMatcher(ctx context.Context, ix IndexReader, m *labels.Ma return it, it.Err() } -func labelValuesWithMatchers(ctx context.Context, r IndexReader, name string, matchers ...*labels.Matcher) ([]string, error) { - allValues, err := r.LabelValues(ctx, name) +func labelValuesWithMatchers(ctx context.Context, r IndexReader, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) { + if hints == nil { + hints = &storage.LabelHints{} + } + + // Do not apply limits here. We need all values. + allValues, err := r.LabelValues(ctx, name, &storage.LabelHints{}) if err != nil { return nil, fmt.Errorf("fetching values of label %s: %w", name, err) } @@ -428,6 +433,9 @@ func labelValuesWithMatchers(ctx context.Context, r IndexReader, name string, ma // If we don't have any matchers for other labels, then we're done. if !hasMatchersForOtherLabels { + if hints.Limit > 0 && len(allValues) > hints.Limit { + allValues = allValues[:hints.Limit] + } return allValues, nil } @@ -451,6 +459,9 @@ func labelValuesWithMatchers(ctx context.Context, r IndexReader, name string, ma values := make([]string, 0, len(indexes)) for _, idx := range indexes { values = append(values, allValues[idx]) + if hints.Limit > 0 && len(values) >= hints.Limit { + break + } } return values, nil diff --git a/tsdb/querier_bench_test.go b/tsdb/querier_bench_test.go index 038052e340..763a20813c 100644 --- a/tsdb/querier_bench_test.go +++ b/tsdb/querier_bench_test.go @@ -228,7 +228,7 @@ func benchmarkLabelValuesWithMatchers(b *testing.B, ir IndexReader) { for _, c := range cases { b.Run(c.name, func(b *testing.B) { for i := 0; i < b.N; i++ { - _, err := labelValuesWithMatchers(ctx, ir, c.labelName, c.matchers...) + _, err := labelValuesWithMatchers(ctx, ir, c.labelName, &storage.LabelHints{}, c.matchers...) require.NoError(b, err) } }) diff --git a/tsdb/querier_test.go b/tsdb/querier_test.go index 56249d3bd6..c28304b8c1 100644 --- a/tsdb/querier_test.go +++ b/tsdb/querier_test.go @@ -2258,19 +2258,26 @@ func (m mockIndex) Close() error { return nil } -func (m mockIndex) SortedLabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, error) { - values, _ := m.LabelValues(ctx, name, matchers...) +func (m mockIndex) SortedLabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) { + values, _ := m.LabelValues(ctx, name, hints, matchers...) sort.Strings(values) return values, nil } -func (m mockIndex) LabelValues(_ context.Context, name string, matchers ...*labels.Matcher) ([]string, error) { +func (m mockIndex) LabelValues(_ context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) { var values []string + if hints == nil { + hints = &storage.LabelHints{} + } + if len(matchers) == 0 { for l := range m.postings { if l.Name == name { values = append(values, l.Value) + if hints.Limit > 0 && len(values) >= hints.Limit { + break + } } } return values, nil @@ -2281,6 +2288,9 @@ func (m mockIndex) LabelValues(_ context.Context, name string, matchers ...*labe if matcher.Matches(series.l.Get(matcher.Name)) { // TODO(colega): shouldn't we check all the matchers before adding this to the values? values = append(values, series.l.Get(name)) + if hints.Limit > 0 && len(values) >= hints.Limit { + break + } } } } @@ -3305,12 +3315,12 @@ func (m mockMatcherIndex) Symbols() index.StringIter { return nil } func (m mockMatcherIndex) Close() error { return nil } // SortedLabelValues will return error if it is called. -func (m mockMatcherIndex) SortedLabelValues(context.Context, string, ...*labels.Matcher) ([]string, error) { +func (m mockMatcherIndex) SortedLabelValues(context.Context, string, *storage.LabelHints, ...*labels.Matcher) ([]string, error) { return []string{}, errors.New("sorted label values called") } // LabelValues will return error if it is called. -func (m mockMatcherIndex) LabelValues(context.Context, string, ...*labels.Matcher) ([]string, error) { +func (m mockMatcherIndex) LabelValues(context.Context, string, *storage.LabelHints, ...*labels.Matcher) ([]string, error) { return []string{}, errors.New("label values called") } @@ -3742,7 +3752,7 @@ func TestReader_PostingsForLabelMatchingHonorsContextCancel(t *testing.T) { failAfter := uint64(mockReaderOfLabelsSeriesCount / 2 / checkContextEveryNIterations) ctx := &testutil.MockContextErrAfter{FailAfter: failAfter} - _, err := labelValuesWithMatchers(ctx, ir, "__name__", labels.MustNewMatcher(labels.MatchRegexp, "__name__", ".+")) + _, err := labelValuesWithMatchers(ctx, ir, "__name__", &storage.LabelHints{}, labels.MustNewMatcher(labels.MatchRegexp, "__name__", ".+")) require.Error(t, err) require.Equal(t, failAfter+1, ctx.Count()) // Plus one for the Err() call that puts the error in the result. @@ -3752,7 +3762,7 @@ type mockReaderOfLabels struct{} const mockReaderOfLabelsSeriesCount = checkContextEveryNIterations * 10 -func (m mockReaderOfLabels) LabelValues(context.Context, string, ...*labels.Matcher) ([]string, error) { +func (m mockReaderOfLabels) LabelValues(context.Context, string, *storage.LabelHints, ...*labels.Matcher) ([]string, error) { return make([]string, mockReaderOfLabelsSeriesCount), nil } @@ -3760,7 +3770,7 @@ func (m mockReaderOfLabels) LabelValueFor(context.Context, storage.SeriesRef, st panic("LabelValueFor called") } -func (m mockReaderOfLabels) SortedLabelValues(context.Context, string, ...*labels.Matcher) ([]string, error) { +func (m mockReaderOfLabels) SortedLabelValues(context.Context, string, *storage.LabelHints, ...*labels.Matcher) ([]string, error) { panic("SortedLabelValues called") }