From b1ed4a0a663d0c62526312311c7529471abbc565 Mon Sep 17 00:00:00 2001 From: Oleg Zaytsev Date: Tue, 20 Jul 2021 14:38:08 +0200 Subject: [PATCH] LabelNames API with matchers (#9083) * Push the matchers for LabelNames all the way into the index. NB This doesn't actually implement it in the index, just plumbs it through for now... Signed-off-by: Tom Wilkie * Hack it up. Does not work. Signed-off-by: Tom Wilkie * Revert changes I don't understand Can't see why do we need to hold a mutex on symbols, and the purpose of the LabelNamesFor method. Maybe I'll need to re-add this later. Signed-off-by: Oleg Zaytsev * Implement LabelNamesFor This method provides the label names that appear in the postings provided. We do that deeper than the label values because we know beforehand that most of the label names we'll be the same across different postings, and we don't want to go down an up looking up the same symbols for all different series. Signed-off-by: Oleg Zaytsev * Mutex on symbols should be unlocked However, I still don't understand why do we need a mutex here. Signed-off-by: Oleg Zaytsev * Fix head.LabelNamesFor Signed-off-by: Oleg Zaytsev * Implement mockIndex LabelNames with matchers Signed-off-by: Oleg Zaytsev * Nitpick on slice initialisation Signed-off-by: Oleg Zaytsev * Add tests for LabelNamesWithMatchers Signed-off-by: Oleg Zaytsev * Fix the mutex mess on head.LabelValues/LabelNames I still don't see why we need to grab that unrelated mutex, but at least now we're grabbing it consistently Signed-off-by: Oleg Zaytsev * Check error after iterating postings Signed-off-by: Oleg Zaytsev * Use the error from posting when there was en error in postings Co-authored-by: Ganesh Vernekar <15064823+codesome@users.noreply.github.com> Signed-off-by: Oleg Zaytsev * Update storage/interface.go comment Signed-off-by: Oleg Zaytsev Co-authored-by: Ganesh Vernekar <15064823+codesome@users.noreply.github.com> * Update tsdb/index/index.go comment Signed-off-by: Oleg Zaytsev Co-authored-by: Ganesh Vernekar <15064823+codesome@users.noreply.github.com> * Update tsdb/index/index.go wrapped error msg Signed-off-by: Oleg Zaytsev Co-authored-by: Ganesh Vernekar <15064823+codesome@users.noreply.github.com> * Update tsdb/index/index.go wrapped error msg Signed-off-by: Oleg Zaytsev Co-authored-by: Ganesh Vernekar <15064823+codesome@users.noreply.github.com> * Update tsdb/index/index.go warpped error msg Signed-off-by: Oleg Zaytsev Co-authored-by: Ganesh Vernekar <15064823+codesome@users.noreply.github.com> * Remove unneeded comment Signed-off-by: Oleg Zaytsev * Add testcases for LabelNames w/matchers in api.go Signed-off-by: Oleg Zaytsev * Use t.Cleanup() instead of defer in tests Signed-off-by: Oleg Zaytsev Co-authored-by: Tom Wilkie Co-authored-by: Ganesh Vernekar <15064823+codesome@users.noreply.github.com> --- promql/engine_test.go | 6 ++-- storage/fanout_test.go | 2 +- storage/interface.go | 5 +-- storage/merge.go | 4 +-- storage/merge_test.go | 2 +- storage/noop.go | 4 +-- storage/remote/read.go | 2 +- storage/secondary.go | 4 +-- tsdb/block.go | 26 +++++++++++---- tsdb/block_test.go | 76 ++++++++++++++++++++++++++++++++++++++++++ tsdb/head.go | 38 +++++++++++++++++---- tsdb/head_test.go | 72 +++++++++++++++++++++++++++++++++++++-- tsdb/index/index.go | 69 +++++++++++++++++++++++++++++++++++++- tsdb/querier.go | 21 ++++++++++-- tsdb/querier_test.go | 48 +++++++++++++++++++++++--- web/api/v1/api.go | 26 +++++---------- web/api/v1/api_test.go | 65 ++++++++++++++++++++++++++++-------- 17 files changed, 402 insertions(+), 68 deletions(-) diff --git a/promql/engine_test.go b/promql/engine_test.go index e6b00c84e..d5734439d 100644 --- a/promql/engine_test.go +++ b/promql/engine_test.go @@ -184,8 +184,10 @@ func (q *errQuerier) Select(bool, *storage.SelectHints, ...*labels.Matcher) stor func (*errQuerier) LabelValues(string, ...*labels.Matcher) ([]string, storage.Warnings, error) { return nil, nil, nil } -func (*errQuerier) LabelNames() ([]string, storage.Warnings, error) { return nil, nil, nil } -func (*errQuerier) Close() error { return nil } +func (*errQuerier) LabelNames(...*labels.Matcher) ([]string, storage.Warnings, error) { + return nil, nil, nil +} +func (*errQuerier) Close() error { return nil } // errSeriesSet implements storage.SeriesSet which always returns error. type errSeriesSet struct { diff --git a/storage/fanout_test.go b/storage/fanout_test.go index 486f60f2f..725ffec49 100644 --- a/storage/fanout_test.go +++ b/storage/fanout_test.go @@ -234,7 +234,7 @@ func (errQuerier) LabelValues(name string, matchers ...*labels.Matcher) ([]strin return nil, nil, errors.New("label values error") } -func (errQuerier) LabelNames() ([]string, storage.Warnings, error) { +func (errQuerier) LabelNames(...*labels.Matcher) ([]string, storage.Warnings, error) { return nil, nil, errors.New("label names error") } diff --git a/storage/interface.go b/storage/interface.go index 92ad15b8c..d3dab0e21 100644 --- a/storage/interface.go +++ b/storage/interface.go @@ -113,8 +113,9 @@ type LabelQuerier interface { LabelValues(name string, matchers ...*labels.Matcher) ([]string, Warnings, error) // LabelNames returns all the unique label names present in the block in sorted order. - // TODO(yeya24): support matchers or hints. - LabelNames() ([]string, Warnings, error) + // If matchers are specified the returned result set is reduced + // to label names of metrics matching the matchers. + LabelNames(matchers ...*labels.Matcher) ([]string, Warnings, error) // Close releases the resources of the Querier. Close() error diff --git a/storage/merge.go b/storage/merge.go index 81e45d55d..1c08a537f 100644 --- a/storage/merge.go +++ b/storage/merge.go @@ -218,13 +218,13 @@ func mergeStrings(a, b []string) []string { } // LabelNames returns all the unique label names present in all queriers in sorted order. -func (q *mergeGenericQuerier) LabelNames() ([]string, Warnings, error) { +func (q *mergeGenericQuerier) LabelNames(matchers ...*labels.Matcher) ([]string, Warnings, error) { var ( labelNamesMap = make(map[string]struct{}) warnings Warnings ) for _, querier := range q.queriers { - names, wrn, err := querier.LabelNames() + names, wrn, err := querier.LabelNames(matchers...) if wrn != nil { // TODO(bwplotka): We could potentially wrap warnings. warnings = append(warnings, wrn...) diff --git a/storage/merge_test.go b/storage/merge_test.go index 90f5725e1..d44ffce7c 100644 --- a/storage/merge_test.go +++ b/storage/merge_test.go @@ -778,7 +778,7 @@ func (m *mockGenericQuerier) LabelValues(name string, matchers ...*labels.Matche return m.resp, m.warnings, m.err } -func (m *mockGenericQuerier) LabelNames() ([]string, Warnings, error) { +func (m *mockGenericQuerier) LabelNames(...*labels.Matcher) ([]string, Warnings, error) { m.mtx.Lock() m.labelNamesCalls++ m.mtx.Unlock() diff --git a/storage/noop.go b/storage/noop.go index 3f800e76c..c63353b92 100644 --- a/storage/noop.go +++ b/storage/noop.go @@ -32,7 +32,7 @@ func (noopQuerier) LabelValues(string, ...*labels.Matcher) ([]string, Warnings, return nil, nil, nil } -func (noopQuerier) LabelNames() ([]string, Warnings, error) { +func (noopQuerier) LabelNames(...*labels.Matcher) ([]string, Warnings, error) { return nil, nil, nil } @@ -55,7 +55,7 @@ func (noopChunkQuerier) LabelValues(string, ...*labels.Matcher) ([]string, Warni return nil, nil, nil } -func (noopChunkQuerier) LabelNames() ([]string, Warnings, error) { +func (noopChunkQuerier) LabelNames(...*labels.Matcher) ([]string, Warnings, error) { return nil, nil, nil } diff --git a/storage/remote/read.go b/storage/remote/read.go index 94eab01cf..7f1d749e6 100644 --- a/storage/remote/read.go +++ b/storage/remote/read.go @@ -212,7 +212,7 @@ func (q *querier) LabelValues(string, ...*labels.Matcher) ([]string, storage.War } // LabelNames implements storage.Querier and is a noop. -func (q *querier) LabelNames() ([]string, storage.Warnings, error) { +func (q *querier) LabelNames(...*labels.Matcher) ([]string, storage.Warnings, error) { // TODO: Implement: https://github.com/prometheus/prometheus/issues/3351 return nil, nil, errors.New("not implemented") } diff --git a/storage/secondary.go b/storage/secondary.go index 2586a7744..64a83b5e7 100644 --- a/storage/secondary.go +++ b/storage/secondary.go @@ -55,8 +55,8 @@ func (s *secondaryQuerier) LabelValues(name string, matchers ...*labels.Matcher) return vals, w, nil } -func (s *secondaryQuerier) LabelNames() ([]string, Warnings, error) { - names, w, err := s.genericQuerier.LabelNames() +func (s *secondaryQuerier) LabelNames(matchers ...*labels.Matcher) ([]string, Warnings, error) { + names, w, err := s.genericQuerier.LabelNames(matchers...) if err != nil { return nil, append([]error{err}, w...), nil } diff --git a/tsdb/block.go b/tsdb/block.go index 23a075c4f..42a91ff59 100644 --- a/tsdb/block.go +++ b/tsdb/block.go @@ -85,13 +85,17 @@ type IndexReader interface { Series(ref uint64, lset *labels.Labels, chks *[]chunks.Meta) error // LabelNames returns all the unique label names present in the index in sorted order. - LabelNames() ([]string, error) + LabelNames(matchers ...*labels.Matcher) ([]string, error) // LabelValueFor returns label value for the given label name in the series referred to by ID. // If the series couldn't be found or the series doesn't have the requested label a // storage.ErrNotFound is returned as error. LabelValueFor(id uint64, label string) (string, error) + // LabelNamesFor returns all the label names for the series referred to by IDs. + // The names returned are sorted. + LabelNamesFor(ids ...uint64) ([]string, error) + // Close releases the underlying resources of the reader. Close() error } @@ -443,7 +447,15 @@ func (r blockIndexReader) LabelValues(name string, matchers ...*labels.Matcher) return st, errors.Wrapf(err, "block: %s", r.b.Meta().ULID) } - return labelValuesWithMatchers(r, name, matchers...) + return labelValuesWithMatchers(r.ir, name, matchers...) +} + +func (r blockIndexReader) LabelNames(matchers ...*labels.Matcher) ([]string, error) { + if len(matchers) == 0 { + return r.b.LabelNames() + } + + return labelNamesWithMatchers(r.ir, matchers...) } func (r blockIndexReader) Postings(name string, values ...string) (index.Postings, error) { @@ -465,10 +477,6 @@ func (r blockIndexReader) Series(ref uint64, lset *labels.Labels, chks *[]chunks return nil } -func (r blockIndexReader) LabelNames() ([]string, error) { - return r.b.LabelNames() -} - func (r blockIndexReader) Close() error { r.b.pendingReaders.Done() return nil @@ -479,6 +487,12 @@ func (r blockIndexReader) LabelValueFor(id uint64, label string) (string, error) return r.ir.LabelValueFor(id, label) } +// LabelNamesFor returns all the label names for the series referred to by IDs. +// The names returned are sorted. +func (r blockIndexReader) LabelNamesFor(ids ...uint64) ([]string, error) { + return r.ir.LabelNamesFor(ids...) +} + type blockTombstoneReader struct { tombstones.Reader b *Block diff --git a/tsdb/block_test.go b/tsdb/block_test.go index fc53d02dc..54cfbc2c4 100644 --- a/tsdb/block_test.go +++ b/tsdb/block_test.go @@ -418,6 +418,82 @@ func BenchmarkLabelValuesWithMatchers(b *testing.B) { } } +func TestLabelNamesWithMatchers(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "test_block_label_names_with_matchers") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(tmpdir)) }) + + var seriesEntries []storage.Series + for i := 0; i < 100; i++ { + seriesEntries = append(seriesEntries, storage.NewListSeries(labels.Labels{ + {Name: "unique", Value: fmt.Sprintf("value%d", i)}, + }, []tsdbutil.Sample{sample{100, 0}})) + + if i%10 == 0 { + seriesEntries = append(seriesEntries, storage.NewListSeries(labels.Labels{ + {Name: "unique", Value: fmt.Sprintf("value%d", i)}, + {Name: "tens", Value: fmt.Sprintf("value%d", i/10)}, + }, []tsdbutil.Sample{sample{100, 0}})) + } + + if i%20 == 0 { + seriesEntries = append(seriesEntries, storage.NewListSeries(labels.Labels{ + {Name: "unique", Value: fmt.Sprintf("value%d", i)}, + {Name: "tens", Value: fmt.Sprintf("value%d", i/10)}, + {Name: "twenties", Value: fmt.Sprintf("value%d", i/20)}, + }, []tsdbutil.Sample{sample{100, 0}})) + } + + } + + blockDir := createBlock(t, tmpdir, seriesEntries) + files, err := sequenceFiles(chunkDir(blockDir)) + require.NoError(t, err) + require.Greater(t, len(files), 0, "No chunk created.") + + // Check open err. + block, err := OpenBlock(nil, blockDir, nil) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, block.Close()) }) + + indexReader, err := block.Index() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, indexReader.Close()) }) + + testCases := []struct { + name string + labelName string + matchers []*labels.Matcher + expectedNames []string + }{ + { + name: "get with non-empty unique: all", + matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchNotEqual, "unique", "")}, + expectedNames: []string{"tens", "twenties", "unique"}, + }, { + name: "get with unique ending in 1: only unique", + matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchRegexp, "unique", "value.*1")}, + expectedNames: []string{"unique"}, + }, { + name: "get with unique = value20: all", + matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "unique", "value20")}, + expectedNames: []string{"tens", "twenties", "unique"}, + }, { + name: "get tens = 1: unique & tens", + matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "tens", "value1")}, + expectedNames: []string{"tens", "unique"}, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + actualNames, err := indexReader.LabelNames(tt.matchers...) + require.NoError(t, err) + require.Equal(t, tt.expectedNames, actualNames) + }) + } +} + // createBlock creates a block with given set of series and returns its dir. func createBlock(tb testing.TB, dir string, series []storage.Series) string { blockDir, err := CreateBlock(series, dir, 0, log.NewNopLogger()) diff --git a/tsdb/head.go b/tsdb/head.go index 2f1896dee..92ad7b53c 100644 --- a/tsdb/head.go +++ b/tsdb/head.go @@ -2019,18 +2019,21 @@ func (h *headIndexReader) LabelValues(name string, matchers ...*labels.Matcher) // LabelNames returns all the unique label names present in the head // that are within the time range mint to maxt. -func (h *headIndexReader) LabelNames() ([]string, error) { - h.head.symMtx.RLock() +func (h *headIndexReader) LabelNames(matchers ...*labels.Matcher) ([]string, error) { if h.maxt < h.head.MinTime() || h.mint > h.head.MaxTime() { - h.head.symMtx.RUnlock() return []string{}, nil } - labelNames := h.head.postings.LabelNames() - h.head.symMtx.RUnlock() + if len(matchers) == 0 { + h.head.symMtx.RLock() + labelNames := h.head.postings.LabelNames() + h.head.symMtx.RUnlock() - sort.Strings(labelNames) - return labelNames, nil + sort.Strings(labelNames) + return labelNames, nil + } + + return labelNamesWithMatchers(h, matchers...) } // Postings returns the postings list iterator for the label pairs. @@ -2122,6 +2125,27 @@ func (h *headIndexReader) LabelValueFor(id uint64, label string) (string, error) return value, nil } +// LabelNamesFor returns all the label names for the series referred to by IDs. +// The names returned are sorted. +func (h *headIndexReader) LabelNamesFor(ids ...uint64) ([]string, error) { + namesMap := make(map[string]struct{}) + for _, id := range ids { + memSeries := h.head.series.getByID(id) + if memSeries == nil { + return nil, storage.ErrNotFound + } + for _, lbl := range memSeries.lset { + namesMap[lbl.Name] = struct{}{} + } + } + names := make([]string, 0, len(namesMap)) + for name := range namesMap { + names = append(names, name) + } + sort.Strings(names) + return names, nil +} + func (h *Head) getOrCreate(hash uint64, lset labels.Labels) (*memSeries, bool, error) { // Just using `getOrCreateWithID` below would be semantically sufficient, but we'd create // a new series on every sample inserted via Add(), which causes allocations diff --git a/tsdb/head_test.go b/tsdb/head_test.go index d9a494e7f..b6aab0780 100644 --- a/tsdb/head_test.go +++ b/tsdb/head_test.go @@ -1934,9 +1934,7 @@ func TestHeadLabelNamesValuesWithMinMaxRange(t *testing.T) { func TestHeadLabelValuesWithMatchers(t *testing.T) { head, _ := newTestHead(t, 1000, false) - defer func() { - require.NoError(t, head.Close()) - }() + t.Cleanup(func() { require.NoError(t, head.Close()) }) app := head.Appender(context.Background()) for i := 0; i < 100; i++ { @@ -1993,6 +1991,74 @@ func TestHeadLabelValuesWithMatchers(t *testing.T) { } } +func TestHeadLabelNamesWithMatchers(t *testing.T) { + head, _ := newTestHead(t, 1000, false) + defer func() { + require.NoError(t, head.Close()) + }() + + app := head.Appender(context.Background()) + for i := 0; i < 100; i++ { + _, err := app.Append(0, labels.Labels{ + {Name: "unique", Value: fmt.Sprintf("value%d", i)}, + }, 100, 0) + require.NoError(t, err) + + if i%10 == 0 { + _, err := app.Append(0, labels.Labels{ + {Name: "unique", Value: fmt.Sprintf("value%d", i)}, + {Name: "tens", Value: fmt.Sprintf("value%d", i/10)}, + }, 100, 0) + require.NoError(t, err) + } + + if i%20 == 0 { + _, err := app.Append(0, labels.Labels{ + {Name: "unique", Value: fmt.Sprintf("value%d", i)}, + {Name: "tens", Value: fmt.Sprintf("value%d", i/10)}, + {Name: "twenties", Value: fmt.Sprintf("value%d", i/20)}, + }, 100, 0) + require.NoError(t, err) + } + } + require.NoError(t, app.Commit()) + + testCases := []struct { + name string + labelName string + matchers []*labels.Matcher + expectedNames []string + }{ + { + name: "get with non-empty unique: all", + matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchNotEqual, "unique", "")}, + expectedNames: []string{"tens", "twenties", "unique"}, + }, { + name: "get with unique ending in 1: only unique", + matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchRegexp, "unique", "value.*1")}, + expectedNames: []string{"unique"}, + }, { + name: "get with unique = value20: all", + matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "unique", "value20")}, + expectedNames: []string{"tens", "twenties", "unique"}, + }, { + name: "get tens = 1: unique & tens", + matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "tens", "value1")}, + expectedNames: []string{"tens", "unique"}, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + headIdxReader := head.indexRange(0, 200) + + actualNames, err := headIdxReader.LabelNames(tt.matchers...) + require.NoError(t, err) + require.Equal(t, tt.expectedNames, actualNames) + }) + } +} + func TestErrReuseAppender(t *testing.T) { head, _ := newTestHead(t, 1000, false) defer func() { diff --git a/tsdb/index/index.go b/tsdb/index/index.go index 3df9c36d4..a7b9e57ef 100644 --- a/tsdb/index/index.go +++ b/tsdb/index/index.go @@ -1517,6 +1517,49 @@ func (r *Reader) LabelValues(name string, matchers ...*labels.Matcher) ([]string return values, nil } +// LabelNamesFor returns all the label names for the series referred to by IDs. +// The names returned are sorted. +func (r *Reader) LabelNamesFor(ids ...uint64) ([]string, error) { + // Gather offsetsMap the name offsetsMap in the symbol table first + offsetsMap := make(map[uint32]struct{}) + for _, id := range ids { + offset := id + // In version 2 series IDs are no longer exact references but series are 16-byte padded + // and the ID is the multiple of 16 of the actual position. + if r.version == FormatV2 { + offset = id * 16 + } + + d := encoding.NewDecbufUvarintAt(r.b, int(offset), castagnoliTable) + buf := d.Get() + if d.Err() != nil { + return nil, errors.Wrap(d.Err(), "get buffer for series") + } + + offsets, err := r.dec.LabelNamesOffsetsFor(buf) + if err != nil { + return nil, errors.Wrap(err, "get label name offsets") + } + for _, off := range offsets { + offsetsMap[off] = struct{}{} + } + } + + // Lookup the unique symbols. + names := make([]string, 0, len(offsetsMap)) + for off := range offsetsMap { + name, err := r.lookupSymbol(off) + if err != nil { + return nil, errors.Wrap(err, "lookup symbol in LabelNamesFor") + } + names = append(names, name) + } + + sort.Strings(names) + + return names, nil +} + // LabelValueFor returns label value for the given label name in the series referred to by ID. func (r *Reader) LabelValueFor(id uint64, label string) (string, error) { offset := id @@ -1670,7 +1713,12 @@ func (r *Reader) Size() int64 { } // LabelNames returns all the unique label names present in the index. -func (r *Reader) LabelNames() ([]string, error) { +// TODO(twilkie) implement support for matchers +func (r *Reader) LabelNames(matchers ...*labels.Matcher) ([]string, error) { + if len(matchers) > 0 { + return nil, errors.Errorf("matchers parameter is not implemented: %+v", matchers) + } + labelNames := make([]string, 0, len(r.postings)) for name := range r.postings { if name == allPostingsKey.Name { @@ -1721,6 +1769,25 @@ func (dec *Decoder) Postings(b []byte) (int, Postings, error) { return n, newBigEndianPostings(l), d.Err() } +// LabelNamesOffsetsFor decodes the offsets of the name symbols for a given series. +// They are returned in the same order they're stored, which should be sorted lexicographically. +func (dec *Decoder) LabelNamesOffsetsFor(b []byte) ([]uint32, error) { + d := encoding.Decbuf{B: b} + k := d.Uvarint() + + offsets := make([]uint32, k) + for i := 0; i < k; i++ { + offsets[i] = uint32(d.Uvarint()) + _ = d.Uvarint() // skip the label value + + if d.Err() != nil { + return nil, errors.Wrap(d.Err(), "read series label offsets") + } + } + + return offsets, d.Err() +} + // LabelValueFor decodes a label for a given series. func (dec *Decoder) LabelValueFor(b []byte, label string) (string, error) { d := encoding.Decbuf{B: b} diff --git a/tsdb/querier.go b/tsdb/querier.go index af5007fc1..18a1fd20a 100644 --- a/tsdb/querier.go +++ b/tsdb/querier.go @@ -88,8 +88,8 @@ func (q *blockBaseQuerier) LabelValues(name string, matchers ...*labels.Matcher) return res, nil, err } -func (q *blockBaseQuerier) LabelNames() ([]string, storage.Warnings, error) { - res, err := q.index.LabelNames() +func (q *blockBaseQuerier) LabelNames(matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { + res, err := q.index.LabelNames(matchers...) return res, nil, err } @@ -407,6 +407,23 @@ func labelValuesWithMatchers(r IndexReader, name string, matchers ...*labels.Mat return values, nil } +func labelNamesWithMatchers(r IndexReader, matchers ...*labels.Matcher) ([]string, error) { + p, err := PostingsForMatchers(r, matchers...) + if err != nil { + return nil, err + } + + var postings []uint64 + for p.Next() { + postings = append(postings, p.At()) + } + if p.Err() != nil { + return nil, errors.Wrapf(p.Err(), "postings for label names with matchers") + } + + return r.LabelNamesFor(postings...) +} + // blockBaseSeriesSet allows to iterate over all series in the single block. // Iterated series are trimmed with given min and max time as well as tombstones. // See newBlockSeriesSet and newBlockChunkSeriesSet to use it for either sample or chunk iterating. diff --git a/tsdb/querier_test.go b/tsdb/querier_test.go index 8f82d1d55..514b1f5b9 100644 --- a/tsdb/querier_test.go +++ b/tsdb/querier_test.go @@ -1154,7 +1154,7 @@ func (m mockIndex) SortedLabelValues(name string, matchers ...*labels.Matcher) ( } func (m mockIndex) LabelValues(name string, matchers ...*labels.Matcher) ([]string, error) { - values := []string{} + var values []string if len(matchers) == 0 { for l := range m.postings { @@ -1168,6 +1168,7 @@ func (m mockIndex) LabelValues(name string, matchers ...*labels.Matcher) ([]stri for _, series := range m.series { for _, matcher := range matchers { if matcher.Matches(series.l.Get(matcher.Name)) { + // TODO(colega): shouldn't we check all the matchers before adding this to the values? values = append(values, series.l.Get(name)) } } @@ -1180,6 +1181,20 @@ func (m mockIndex) LabelValueFor(id uint64, label string) (string, error) { return m.series[id].l.Get(label), nil } +func (m mockIndex) LabelNamesFor(ids ...uint64) ([]string, error) { + namesMap := make(map[string]bool) + for _, id := range ids { + for _, lbl := range m.series[id].l { + namesMap[lbl.Name] = true + } + } + names := make([]string, 0, len(namesMap)) + for name := range namesMap { + names = append(names, name) + } + return names, nil +} + func (m mockIndex) Postings(name string, values ...string) (index.Postings, error) { res := make([]index.Postings, 0, len(values)) for _, value := range values { @@ -1212,10 +1227,27 @@ func (m mockIndex) Series(ref uint64, lset *labels.Labels, chks *[]chunks.Meta) return nil } -func (m mockIndex) LabelNames() ([]string, error) { +func (m mockIndex) LabelNames(matchers ...*labels.Matcher) ([]string, error) { names := map[string]struct{}{} - for l := range m.postings { - names[l.Name] = struct{}{} + if len(matchers) == 0 { + for l := range m.postings { + names[l.Name] = struct{}{} + } + } else { + for _, series := range m.series { + matches := true + for _, matcher := range matchers { + matches = matches || matcher.Matches(series.l.Get(matcher.Name)) + if !matches { + break + } + } + if matches { + for _, lbl := range series.l { + names[lbl.Name] = struct{}{} + } + } + } } l := make([]string, 0, len(names)) for name := range names { @@ -2007,6 +2039,10 @@ func (m mockMatcherIndex) LabelValueFor(id uint64, label string) (string, error) return "", errors.New("label value for called") } +func (m mockMatcherIndex) LabelNamesFor(ids ...uint64) ([]string, error) { + return nil, errors.New("label names for for called") +} + func (m mockMatcherIndex) Postings(name string, values ...string) (index.Postings, error) { return index.EmptyPostings(), nil } @@ -2019,7 +2055,9 @@ func (m mockMatcherIndex) Series(ref uint64, lset *labels.Labels, chks *[]chunks return nil } -func (m mockMatcherIndex) LabelNames() ([]string, error) { return []string{}, nil } +func (m mockMatcherIndex) LabelNames(...*labels.Matcher) ([]string, error) { + return []string{}, nil +} func TestPostingsForMatcher(t *testing.T) { cases := []struct { diff --git a/web/api/v1/api.go b/web/api/v1/api.go index 8cbc915b5..745a28c8d 100644 --- a/web/api/v1/api.go +++ b/web/api/v1/api.go @@ -559,26 +559,18 @@ func (api *API) labelNames(r *http.Request) apiFuncResult { warnings storage.Warnings ) if len(matcherSets) > 0 { - hints := &storage.SelectHints{ - Start: timestamp.FromTime(start), - End: timestamp.FromTime(end), - Func: "series", // There is no series function, this token is used for lookups that don't need samples. - } - labelNamesSet := make(map[string]struct{}) - // Get all series which match matchers. - for _, mset := range matcherSets { - s := q.Select(false, hints, mset...) - for s.Next() { - series := s.At() - for _, lb := range series.Labels() { - labelNamesSet[lb.Name] = struct{}{} - } - } - warnings = append(warnings, s.Warnings()...) - if err := s.Err(); err != nil { + + for _, matchers := range matcherSets { + vals, callWarnings, err := q.LabelNames(matchers...) + if err != nil { return apiFuncResult{nil, &apiError{errorExec, err}, warnings, nil} } + + warnings = append(warnings, callWarnings...) + for _, val := range vals { + labelNamesSet[val] = struct{}{} + } } // Convert the map to an array. diff --git a/web/api/v1/api_test.go b/web/api/v1/api_test.go index 4495832a5..420889778 100644 --- a/web/api/v1/api_test.go +++ b/web/api/v1/api_test.go @@ -464,6 +464,7 @@ func TestLabelNames(t *testing.T) { test_metric1{foo2="boo"} 1+0x100 test_metric2{foo="boo"} 1+0x100 test_metric2{foo="boo", xyz="qwerty"} 1+0x100 + test_metric2{foo="baz", abc="qwerty"} 1+0x100 `) require.NoError(t, err) defer suite.Close() @@ -472,21 +473,57 @@ func TestLabelNames(t *testing.T) { api := &API{ Queryable: suite.Storage(), } - request := func(m string) (*http.Request, error) { - if m == http.MethodPost { - r, err := http.NewRequest(m, "http://example.com", nil) - r.Header.Set("Content-Type", "application/x-www-form-urlencoded") - return r, err - } - return http.NewRequest(m, "http://example.com", nil) - } - for _, method := range []string{http.MethodGet, http.MethodPost} { - ctx := context.Background() - req, err := request(method) + request := func(method string, matchers ...string) (*http.Request, error) { + u, err := url.Parse("http://example.com") require.NoError(t, err) - res := api.labelNames(req.WithContext(ctx)) - assertAPIError(t, res.err, "") - assertAPIResponse(t, res.data, []string{"__name__", "baz", "foo", "foo1", "foo2", "xyz"}) + q := u.Query() + for _, matcher := range matchers { + q.Add("match[]", matcher) + } + u.RawQuery = q.Encode() + + r, err := http.NewRequest(method, u.String(), nil) + if method == http.MethodPost { + r.Header.Set("Content-Type", "application/x-www-form-urlencoded") + } + return r, err + } + + for _, tc := range []struct { + name string + matchers []string + expected []string + }{ + { + name: "no matchers", + expected: []string{"__name__", "abc", "baz", "foo", "foo1", "foo2", "xyz"}, + }, + { + name: "non empty label matcher", + matchers: []string{`{foo=~".+"}`}, + expected: []string{"__name__", "abc", "foo", "xyz"}, + }, + { + name: "exact label matcher", + matchers: []string{`{foo="boo"}`}, + expected: []string{"__name__", "foo", "xyz"}, + }, + { + name: "two matchers", + matchers: []string{`{foo="boo"}`, `{foo="baz"}`}, + expected: []string{"__name__", "abc", "foo", "xyz"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + for _, method := range []string{http.MethodGet, http.MethodPost} { + ctx := context.Background() + req, err := request(method, tc.matchers...) + require.NoError(t, err) + res := api.labelNames(req.WithContext(ctx)) + assertAPIError(t, res.err, "") + assertAPIResponse(t, res.data, tc.expected) + } + }) } }