Add remote read external label handling (#2555)

* Add remote read external label handling

This implements rule 1 and 2 from
https://docs.google.com/document/d/188YauRgfF0J4CYMigLsVNN34V_kUwKnApBs2dQMfBbs/edit

* Use more descriptive example labels in read test

* Add comment for querier.addExternalLabels()

* Make argument naming in removeLabels() more generic

storage/remote/read.go

@@ -27,8 +27,9 @@ import (
 
 // Reader allows reading from multiple remote sources.
 type Reader struct {
-	mtx     sync.Mutex
-	clients []*Client
+	mtx            sync.Mutex
+	clients        []*Client
+	externalLabels model.LabelSet
 }
 
 // ApplyConfig updates the state as the new config requires.
@@ -50,6 +51,7 @@ func (r *Reader) ApplyConfig(conf *config.Config) error {
 	defer r.mtx.Unlock()
 
 	r.clients = clients
+	r.externalLabels = conf.GlobalConfig.ExternalLabels
 
 	return nil
 }
@@ -62,22 +64,74 @@ func (r *Reader) Queriers() []local.Querier {
 
 	queriers := make([]local.Querier, 0, len(r.clients))
 	for _, c := range r.clients {
-		queriers = append(queriers, &querier{client: c})
+		queriers = append(queriers, &querier{
+			client:         c,
+			externalLabels: r.externalLabels,
+		})
 	}
 	return queriers
 }
 
 // querier is an adapter to make a Client usable as a promql.Querier.
 type querier struct {
-	client *Client
+	client         *Client
+	externalLabels model.LabelSet
 }
 
 func (q *querier) QueryRange(ctx context.Context, from, through model.Time, matchers ...*metric.LabelMatcher) ([]local.SeriesIterator, error) {
-	return MatrixToIterators(q.client.Read(ctx, from, through, matchers))
+	return MatrixToIterators(q.read(ctx, from, through, matchers))
 }
 
 func (q *querier) QueryInstant(ctx context.Context, ts model.Time, stalenessDelta time.Duration, matchers ...*metric.LabelMatcher) ([]local.SeriesIterator, error) {
-	return MatrixToIterators(q.client.Read(ctx, ts.Add(-stalenessDelta), ts, matchers))
+	return MatrixToIterators(q.read(ctx, ts.Add(-stalenessDelta), ts, matchers))
+}
+
+func (q *querier) read(ctx context.Context, from, through model.Time, matchers metric.LabelMatchers) (model.Matrix, error) {
+	m, added := q.addExternalLabels(matchers)
+
+	res, err := q.client.Read(ctx, from, through, m)
+	if err != nil {
+		return nil, err
+	}
+
+	removeLabels(res, added)
+	return res, err
+}
+
+// addExternalLabels adds matchers for each external label. External labels
+// that already have a corresponding user-supplied matcher are skipped, as we
+// assume that the user explicitly wants to select a different value for them.
+// We return the new set of matchers, along with a map of labels for which
+// matchers were added, so that these can later be removed from the result
+// time series again.
+func (q *querier) addExternalLabels(matchers metric.LabelMatchers) (metric.LabelMatchers, model.LabelSet) {
+	el := make(model.LabelSet, len(q.externalLabels))
+	for k, v := range q.externalLabels {
+		el[k] = v
+	}
+	for _, m := range matchers {
+		if _, ok := el[m.Name]; ok {
+			delete(el, m.Name)
+		}
+	}
+
+	for k, v := range el {
+		m, err := metric.NewLabelMatcher(metric.Equal, k, v)
+		if err != nil {
+			panic(err)
+		}
+		matchers = append(matchers, m)
+	}
+
+	return matchers, el
+}
+
+func removeLabels(m model.Matrix, labels model.LabelSet) {
+	for _, ss := range m {
+		for k := range labels {
+			delete(ss.Metric, k)
+		}
+	}
 }
 
 // MatrixToIterators returns series iterators for a given matrix.

storage/remote/read_test.go

@@ -0,0 +1,94 @@
+// Copyright 2017 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package remote
+
+import (
+	"reflect"
+	"sort"
+	"testing"
+
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/prometheus/storage/metric"
+)
+
+func mustNewLabelMatcher(mt metric.MatchType, name model.LabelName, val model.LabelValue) *metric.LabelMatcher {
+	m, err := metric.NewLabelMatcher(mt, name, val)
+	if err != nil {
+		panic(err)
+	}
+	return m
+}
+
+func TestAddExternalLabels(t *testing.T) {
+	tests := []struct {
+		el          model.LabelSet
+		inMatchers  metric.LabelMatchers
+		outMatchers metric.LabelMatchers
+		added       model.LabelSet
+	}{
+		{
+			el: model.LabelSet{},
+			inMatchers: metric.LabelMatchers{
+				mustNewLabelMatcher(metric.Equal, "job", "api-server"),
+			},
+			outMatchers: metric.LabelMatchers{
+				mustNewLabelMatcher(metric.Equal, "job", "api-server"),
+			},
+			added: model.LabelSet{},
+		},
+		{
+			el: model.LabelSet{"region": "europe", "dc": "berlin-01"},
+			inMatchers: metric.LabelMatchers{
+				mustNewLabelMatcher(metric.Equal, "job", "api-server"),
+			},
+			outMatchers: metric.LabelMatchers{
+				mustNewLabelMatcher(metric.Equal, "job", "api-server"),
+				mustNewLabelMatcher(metric.Equal, "region", "europe"),
+				mustNewLabelMatcher(metric.Equal, "dc", "berlin-01"),
+			},
+			added: model.LabelSet{"region": "europe", "dc": "berlin-01"},
+		},
+		{
+			el: model.LabelSet{"region": "europe", "dc": "berlin-01"},
+			inMatchers: metric.LabelMatchers{
+				mustNewLabelMatcher(metric.Equal, "job", "api-server"),
+				mustNewLabelMatcher(metric.Equal, "dc", "munich-02"),
+			},
+			outMatchers: metric.LabelMatchers{
+				mustNewLabelMatcher(metric.Equal, "job", "api-server"),
+				mustNewLabelMatcher(metric.Equal, "region", "europe"),
+				mustNewLabelMatcher(metric.Equal, "dc", "munich-02"),
+			},
+			added: model.LabelSet{"region": "europe"},
+		},
+	}
+
+	for i, test := range tests {
+		q := querier{
+			externalLabels: test.el,
+		}
+
+		matchers, added := q.addExternalLabels(test.inMatchers)
+
+		sort.Slice(test.outMatchers, func(i, j int) bool { return test.outMatchers[i].Name < test.outMatchers[j].Name })
+		sort.Slice(matchers, func(i, j int) bool { return matchers[i].Name < matchers[j].Name })
+
+		if !reflect.DeepEqual(matchers, test.outMatchers) {
+			t.Fatalf("%d. unexpected matchers; want %v, got %v", i, test.outMatchers, matchers)
+		}
+		if !reflect.DeepEqual(added, test.added) {
+			t.Fatalf("%d. unexpected added labels; want %v, got %v", i, test.added, added)
+		}
+	}
+}