mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2025-03-05 20:59:13 -08:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

allow limiting label values calls

Browse source 
Signed-off-by: Andre Branchizio <andrejbranch@gmail.com>
This commit is contained in:
Andre Branchizio 2025-02-28 09:00:24 -07:00
parent 2ef8706c27
commit 68cd106a11
13 changed files with 92 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
cmd/promtool/tsdb.go
View file

@ -553,7 +553,7 @@ func analyzeBlock(ctx context.Context, path, blockID string, limit int, runExten
postingInfos = postingInfos[:0]
for _, n := range allLabelNames {
values, err := ir.SortedLabelValues(ctx, n, selectors...)
values, err := ir.SortedLabelValues(ctx, n, &storage.LabelHints{}, selectors...)
if err != nil {
return err
}
@ -569,7 +569,7 @@ func analyzeBlock(ctx context.Context, path, blockID string, limit int, runExten
postingInfos = postingInfos[:0]
for _, n := range allLabelNames {
lv, err := ir.SortedLabelValues(ctx, n, selectors...)
lv, err := ir.SortedLabelValues(ctx, n, &storage.LabelHints{}, selectors...)
if err != nil {
return err
}
@ -579,7 +579,7 @@ func analyzeBlock(ctx context.Context, path, blockID string, limit int, runExten
printInfo(postingInfos)
postingInfos = postingInfos[:0]
lv, err := ir.SortedLabelValues(ctx, "__name__", selectors...)
lv, err := ir.SortedLabelValues(ctx, "__name__", &storage.LabelHints{}, selectors...)
if err != nil {
return err
}

16
tsdb/block.go
View file

@ -67,10 +67,10 @@ type IndexReader interface {
Symbols() index.StringIter
// SortedLabelValues returns sorted possible label values.
SortedLabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, error)
SortedLabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error)
// LabelValues returns possible label values which may not be sorted.
LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, error)
LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error)
// Postings returns the postings list iterator for the label pairs.
// The Postings here contain the offsets to the series inside the index.
@ -476,14 +476,14 @@ func (r blockIndexReader) Symbols() index.StringIter {
return r.ir.Symbols()
}
func (r blockIndexReader) SortedLabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, error) {
func (r blockIndexReader) SortedLabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) {
var st []string
var err error
if len(matchers) == 0 {
st, err = r.ir.SortedLabelValues(ctx, name)
st, err = r.ir.SortedLabelValues(ctx, name, hints)
} else {
st, err = r.LabelValues(ctx, name, matchers...)
st, err = r.LabelValues(ctx, name, hints, matchers...)
if err == nil {
slices.Sort(st)
}
@ -494,16 +494,16 @@ func (r blockIndexReader) SortedLabelValues(ctx context.Context, name string, ma
return st, nil
}
func (r blockIndexReader) LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, error) {
func (r blockIndexReader) LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) {
if len(matchers) == 0 {
st, err := r.ir.LabelValues(ctx, name)
st, err := r.ir.LabelValues(ctx, name, hints)
if err != nil {
return st, fmt.Errorf("block: %s: %w", r.b.Meta().ULID, err)
}
return st, nil
}
return labelValuesWithMatchers(ctx, r.ir, name, matchers...)
return labelValuesWithMatchers(ctx, r.ir, name, hints, matchers...)
}
func (r blockIndexReader) LabelNames(ctx context.Context, matchers ...*labels.Matcher) ([]string, error) {

6
tsdb/block_test.go
View file

@ -299,11 +299,11 @@ func TestLabelValuesWithMatchers(t *testing.T) {
for _, tt := range testCases {
t.Run(tt.name, func(t *testing.T) {
actualValues, err := indexReader.SortedLabelValues(ctx, tt.labelName, tt.matchers...)
actualValues, err := indexReader.SortedLabelValues(ctx, tt.labelName, &storage.LabelHints{}, tt.matchers...)
require.NoError(t, err)
require.Equal(t, tt.expectedValues, actualValues)
actualValues, err = indexReader.LabelValues(ctx, tt.labelName, tt.matchers...)
actualValues, err = indexReader.LabelValues(ctx, tt.labelName, &storage.LabelHints{}, tt.matchers...)
sort.Strings(actualValues)
require.NoError(t, err)
require.Equal(t, tt.expectedValues, actualValues)
@ -459,7 +459,7 @@ func BenchmarkLabelValuesWithMatchers(b *testing.B) {
b.ReportAllocs()
for benchIdx := 0; benchIdx < b.N; benchIdx++ {
actualValues, err := indexReader.LabelValues(ctx, "b_tens", matchers...)
actualValues, err := indexReader.LabelValues(ctx, "b_tens", &storage.LabelHints{}, matchers...)
require.NoError(b, err)
require.Len(b, actualValues, 9)
}

10
tsdb/head_read.go
View file

@ -61,8 +61,8 @@ func (h *headIndexReader) Symbols() index.StringIter {
// specific label name that are within the time range mint to maxt.
// If matchers are specified the returned result set is reduced
// to label values of metrics matching the matchers.
func (h *headIndexReader) SortedLabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, error) {
values, err := h.LabelValues(ctx, name, matchers...)
func (h *headIndexReader) SortedLabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) {
values, err := h.LabelValues(ctx, name, hints, matchers...)
if err == nil {
slices.Sort(values)
}
@ -73,16 +73,16 @@ func (h *headIndexReader) SortedLabelValues(ctx context.Context, name string, ma
// specific label name that are within the time range mint to maxt.
// If matchers are specified the returned result set is reduced
// to label values of metrics matching the matchers.
func (h *headIndexReader) LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, error) {
func (h *headIndexReader) LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) {
if h.maxt < h.head.MinTime() || h.mint > h.head.MaxTime() {
return []string{}, nil
}
if len(matchers) == 0 {
return h.head.postings.LabelValues(ctx, name), nil
return h.head.postings.LabelValues(ctx, name, hints), nil
}
return labelValuesWithMatchers(ctx, h, name, matchers...)
return labelValuesWithMatchers(ctx, h, name, hints, matchers...)
}
// LabelNames returns all the unique label names present in the head

10
tsdb/head_test.go
View file

@ -1007,7 +1007,7 @@ func TestHead_Truncate(t *testing.T) {
ss = map[string]struct{}{}
values[name] = ss
}
for _, value := range h.postings.LabelValues(ctx, name) {
for _, value := range h.postings.LabelValues(ctx, name, &storage.LabelHints{}) {
ss[value] = struct{}{}
}
}
@ -2929,7 +2929,7 @@ func TestHeadLabelNamesValuesWithMinMaxRange(t *testing.T) {
require.Equal(t, tt.expectedNames, actualLabelNames)
if len(tt.expectedValues) > 0 {
for i, name := range expectedLabelNames {
actualLabelValue, err := headIdxReader.SortedLabelValues(ctx, name)
actualLabelValue, err := headIdxReader.SortedLabelValues(ctx, name, &storage.LabelHints{})
require.NoError(t, err)
require.Equal(t, []string{tt.expectedValues[i]}, actualLabelValue)
}
@ -3002,11 +3002,11 @@ func TestHeadLabelValuesWithMatchers(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
headIdxReader := head.indexRange(0, 200)
actualValues, err := headIdxReader.SortedLabelValues(ctx, tt.labelName, tt.matchers...)
actualValues, err := headIdxReader.SortedLabelValues(ctx, tt.labelName, &storage.LabelHints{}, tt.matchers...)
require.NoError(t, err)
require.Equal(t, tt.expectedValues, actualValues)
actualValues, err = headIdxReader.LabelValues(ctx, tt.labelName, tt.matchers...)
actualValues, err = headIdxReader.LabelValues(ctx, tt.labelName, &storage.LabelHints{}, tt.matchers...)
sort.Strings(actualValues)
require.NoError(t, err)
require.Equal(t, tt.expectedValues, actualValues)
@ -3265,7 +3265,7 @@ func BenchmarkHeadLabelValuesWithMatchers(b *testing.B) {
b.ReportAllocs()
for benchIdx := 0; benchIdx < b.N; benchIdx++ {
actualValues, err := headIdxReader.LabelValues(ctx, "b_tens", matchers...)
actualValues, err := headIdxReader.LabelValues(ctx, "b_tens", &storage.LabelHints{}, matchers...)
require.NoError(b, err)
require.Len(b, actualValues, 9)
}

19
tsdb/index/index.go
View file

@ -1493,8 +1493,8 @@ func (r *Reader) SymbolTableSize() uint64 {
// SortedLabelValues returns value tuples that exist for the given label name.
// It is not safe to use the return value beyond the lifetime of the byte slice
// passed into the Reader.
func (r *Reader) SortedLabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, error) {
values, err := r.LabelValues(ctx, name, matchers...)
func (r *Reader) SortedLabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) {
values, err := r.LabelValues(ctx, name, hints, matchers...)
if err == nil && r.version == FormatV1 {
slices.Sort(values)
}
@ -1505,11 +1505,15 @@ func (r *Reader) SortedLabelValues(ctx context.Context, name string, matchers ..
// It is not safe to use the return value beyond the lifetime of the byte slice
// passed into the Reader.
// TODO(replay): Support filtering by matchers.
func (r *Reader) LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, error) {
func (r *Reader) LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) {
if len(matchers) > 0 {
return nil, fmt.Errorf("matchers parameter is not implemented: %+v", matchers)
}
if hints == nil {
hints = &storage.LabelHints{}
}
if r.version == FormatV1 {
e, ok := r.postingsV1[name]
if !ok {
@ -1529,9 +1533,16 @@ func (r *Reader) LabelValues(ctx context.Context, name string, matchers ...*labe
return nil, nil
}
values := make([]string, 0, len(e)*symbolFactor)
valuesLength := len(e) * symbolFactor
if hints.Limit > 0 && valuesLength > (hints.Limit*symbolFactor) {
valuesLength = hints.Limit * symbolFactor
}
values := make([]string, 0, valuesLength)
lastVal := e[len(e)-1].value
err := r.traversePostingOffsets(ctx, e[0].off, func(val string, _ uint64) (bool, error) {
if len(e) >= valuesLength {
return false, nil
}
values = append(values, val)
return val != lastVal, nil
})

2
tsdb/index/index_test.go
View file

@ -421,7 +421,7 @@ func TestPersistence_index_e2e(t *testing.T) {
for k, v := range labelPairs {
sort.Strings(v)
res, err := ir.SortedLabelValues(ctx, k)
res, err := ir.SortedLabelValues(ctx, k, &storage.LabelHints{})
require.NoError(t, err)
require.Equal(t, len(v), len(res))

10
tsdb/index/postings.go
View file

@ -168,11 +168,19 @@ func (p *MemPostings) LabelNames() []string {
}
// LabelValues returns label values for the given name.
func (p *MemPostings) LabelValues(_ context.Context, name string) []string {
func (p *MemPostings) LabelValues(_ context.Context, name string, hints *storage.LabelHints) []string {
p.mtx.RLock()
values := p.lvs[name]
p.mtx.RUnlock()
if hints == nil {
hints = &storage.LabelHints{}
}
if hints.Limit > 0 && len(values) > hints.Limit {
values = values[:hints.Limit]
}
// The slice from p.lvs[name] is shared between all readers, and it is append-only.
// Since it's shared, we need to make a copy of it before returning it to make
// sure that no caller modifies the original one by sorting it or filtering it.

10
tsdb/ooo_head_read.go
View file

@ -176,16 +176,16 @@ type multiMeta struct {
// LabelValues needs to be overridden from the headIndexReader implementation
// so we can return labels within either in-order range or ooo range.
func (oh *HeadAndOOOIndexReader) LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, error) {
func (oh *HeadAndOOOIndexReader) LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) {
if oh.maxt < oh.head.MinTime() && oh.maxt < oh.head.MinOOOTime() || oh.mint > oh.head.MaxTime() && oh.mint > oh.head.MaxOOOTime() {
return []string{}, nil
}
if len(matchers) == 0 {
return oh.head.postings.LabelValues(ctx, name), nil
return oh.head.postings.LabelValues(ctx, name, hints), nil
}
return labelValuesWithMatchers(ctx, oh, name, matchers...)
return labelValuesWithMatchers(ctx, oh, name, hints, matchers...)
}
func lessByMinTimeAndMinRef(a, b chunks.Meta) int {
@ -484,11 +484,11 @@ func (ir *OOOCompactionHeadIndexReader) Series(ref storage.SeriesRef, builder *l
return getOOOSeriesChunks(s, ir.ch.mint, ir.ch.maxt, 0, ir.ch.lastMmapRef, false, 0, chks)
}
func (ir *OOOCompactionHeadIndexReader) SortedLabelValues(_ context.Context, _ string, _ ...*labels.Matcher) ([]string, error) {
func (ir *OOOCompactionHeadIndexReader) SortedLabelValues(_ context.Context, _ string, _ *storage.LabelHints, _ ...*labels.Matcher) ([]string, error) {
return nil, errors.New("not implemented")
}
func (ir *OOOCompactionHeadIndexReader) LabelValues(_ context.Context, _ string, _ ...*labels.Matcher) ([]string, error) {
func (ir *OOOCompactionHeadIndexReader) LabelValues(_ context.Context, _ string, _ *storage.LabelHints, _ ...*labels.Matcher) ([]string, error) {
return nil, errors.New("not implemented")
}

8
tsdb/ooo_head_read_test.go
View file

@ -453,24 +453,24 @@ func testOOOHeadChunkReader_LabelValues(t *testing.T, scenario sampleTypeScenari
// We first want to test using a head index reader that covers the biggest query interval
oh := NewHeadAndOOOIndexReader(head, tc.queryMinT, tc.queryMinT, tc.queryMaxT, 0)
matchers := []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "foo", "bar1")}
values, err := oh.LabelValues(ctx, "foo", matchers...)
values, err := oh.LabelValues(ctx, "foo", &storage.LabelHints{}, matchers...)
sort.Strings(values)
require.NoError(t, err)
require.Equal(t, tc.expValues1, values)
matchers = []*labels.Matcher{labels.MustNewMatcher(labels.MatchNotRegexp, "foo", "^bar.")}
values, err = oh.LabelValues(ctx, "foo", matchers...)
values, err = oh.LabelValues(ctx, "foo", &storage.LabelHints{}, matchers...)
sort.Strings(values)
require.NoError(t, err)
require.Equal(t, tc.expValues2, values)
matchers = []*labels.Matcher{labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar.")}
values, err = oh.LabelValues(ctx, "foo", matchers...)
values, err = oh.LabelValues(ctx, "foo", &storage.LabelHints{}, matchers...)
sort.Strings(values)
require.NoError(t, err)
require.Equal(t, tc.expValues3, values)
values, err = oh.LabelValues(ctx, "foo")
values, err = oh.LabelValues(ctx, "foo", &storage.LabelHints{})
sort.Strings(values)
require.NoError(t, err)
require.Equal(t, tc.expValues4, values)

19
tsdb/querier.go
View file

@ -77,8 +77,8 @@ func newBlockBaseQuerier(b BlockReader, mint, maxt int64) (*blockBaseQuerier, er
}, nil
}
func (q *blockBaseQuerier) LabelValues(ctx context.Context, name string, _ *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
res, err := q.index.SortedLabelValues(ctx, name, matchers...)
func (q *blockBaseQuerier) LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
res, err := q.index.SortedLabelValues(ctx, name, hints, matchers...)
return res, nil, err
}
@ -390,8 +390,13 @@ func inversePostingsForMatcher(ctx context.Context, ix IndexReader, m *labels.Ma
return it, it.Err()
}
func labelValuesWithMatchers(ctx context.Context, r IndexReader, name string, matchers ...*labels.Matcher) ([]string, error) {
allValues, err := r.LabelValues(ctx, name)
func labelValuesWithMatchers(ctx context.Context, r IndexReader, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) {
if hints == nil {
hints = &storage.LabelHints{}
}
// Do not apply limits here. We need all values.
allValues, err := r.LabelValues(ctx, name, &storage.LabelHints{})
if err != nil {
return nil, fmt.Errorf("fetching values of label %s: %w", name, err)
}
@ -428,6 +433,9 @@ func labelValuesWithMatchers(ctx context.Context, r IndexReader, name string, ma
// If we don't have any matchers for other labels, then we're done.
if !hasMatchersForOtherLabels {
if hints.Limit > 0 && len(allValues) > hints.Limit {
allValues = allValues[:hints.Limit]
}
return allValues, nil
}
@ -451,6 +459,9 @@ func labelValuesWithMatchers(ctx context.Context, r IndexReader, name string, ma
values := make([]string, 0, len(indexes))
for _, idx := range indexes {
values = append(values, allValues[idx])
if hints.Limit > 0 && len(values) >= hints.Limit {
break
}
}
return values, nil

2
tsdb/querier_bench_test.go
View file

@ -228,7 +228,7 @@ func benchmarkLabelValuesWithMatchers(b *testing.B, ir IndexReader) {
for _, c := range cases {
b.Run(c.name, func(b *testing.B) {
for i := 0; i < b.N; i++ {
_, err := labelValuesWithMatchers(ctx, ir, c.labelName, c.matchers...)
_, err := labelValuesWithMatchers(ctx, ir, c.labelName, &storage.LabelHints{}, c.matchers...)
require.NoError(b, err)
}
})

26
tsdb/querier_test.go
View file

@ -2258,19 +2258,26 @@ func (m mockIndex) Close() error {
return nil
}
func (m mockIndex) SortedLabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, error) {
values, _ := m.LabelValues(ctx, name, matchers...)
func (m mockIndex) SortedLabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) {
values, _ := m.LabelValues(ctx, name, hints, matchers...)
sort.Strings(values)
return values, nil
}
func (m mockIndex) LabelValues(_ context.Context, name string, matchers ...*labels.Matcher) ([]string, error) {
func (m mockIndex) LabelValues(_ context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) {
var values []string
if hints == nil {
hints = &storage.LabelHints{}
}
if len(matchers) == 0 {
for l := range m.postings {
if l.Name == name {
values = append(values, l.Value)
if hints.Limit > 0 && len(values) >= hints.Limit {
break
}
}
}
return values, nil
@ -2281,6 +2288,9 @@ func (m mockIndex) LabelValues(_ context.Context, name string, matchers ...*labe
if matcher.Matches(series.l.Get(matcher.Name)) {
// TODO(colega): shouldn't we check all the matchers before adding this to the values?
values = append(values, series.l.Get(name))
if hints.Limit > 0 && len(values) >= hints.Limit {
break
}
}
}
}
@ -3305,12 +3315,12 @@ func (m mockMatcherIndex) Symbols() index.StringIter { return nil }
func (m mockMatcherIndex) Close() error { return nil }
// SortedLabelValues will return error if it is called.
func (m mockMatcherIndex) SortedLabelValues(context.Context, string, ...*labels.Matcher) ([]string, error) {
func (m mockMatcherIndex) SortedLabelValues(context.Context, string, *storage.LabelHints, ...*labels.Matcher) ([]string, error) {
return []string{}, errors.New("sorted label values called")
}
// LabelValues will return error if it is called.
func (m mockMatcherIndex) LabelValues(context.Context, string, ...*labels.Matcher) ([]string, error) {
func (m mockMatcherIndex) LabelValues(context.Context, string, *storage.LabelHints, ...*labels.Matcher) ([]string, error) {
return []string{}, errors.New("label values called")
}
@ -3742,7 +3752,7 @@ func TestReader_PostingsForLabelMatchingHonorsContextCancel(t *testing.T) {
failAfter := uint64(mockReaderOfLabelsSeriesCount / 2 / checkContextEveryNIterations)
ctx := &testutil.MockContextErrAfter{FailAfter: failAfter}
_, err := labelValuesWithMatchers(ctx, ir, "__name__", labels.MustNewMatcher(labels.MatchRegexp, "__name__", ".+"))
_, err := labelValuesWithMatchers(ctx, ir, "__name__", &storage.LabelHints{}, labels.MustNewMatcher(labels.MatchRegexp, "__name__", ".+"))
require.Error(t, err)
require.Equal(t, failAfter+1, ctx.Count()) // Plus one for the Err() call that puts the error in the result.
@ -3752,7 +3762,7 @@ type mockReaderOfLabels struct{}
const mockReaderOfLabelsSeriesCount = checkContextEveryNIterations * 10
func (m mockReaderOfLabels) LabelValues(context.Context, string, ...*labels.Matcher) ([]string, error) {
func (m mockReaderOfLabels) LabelValues(context.Context, string, *storage.LabelHints, ...*labels.Matcher) ([]string, error) {
return make([]string, mockReaderOfLabelsSeriesCount), nil
}
@ -3760,7 +3770,7 @@ func (m mockReaderOfLabels) LabelValueFor(context.Context, storage.SeriesRef, st
panic("LabelValueFor called")
}
func (m mockReaderOfLabels) SortedLabelValues(context.Context, string, ...*labels.Matcher) ([]string, error) {
func (m mockReaderOfLabels) SortedLabelValues(context.Context, string, *storage.LabelHints, ...*labels.Matcher) ([]string, error) {
panic("SortedLabelValues called")
}