Merge external labels in order.

storage/remote/codec.go

@@ -321,6 +321,9 @@ func MetricToLabelProtos(metric model.Metric) []*prompb.Label {
 			Value: string(v),
 		})
 	}
+	sort.Slice(labels, func(i int, j int) bool {
+		return labels[i].Name < labels[j].Name
+	})
 	return labels
 }
 

web/api/v1/api.go

@@ -21,6 +21,7 @@ import (
 	"math"
 	"net/http"
 	"net/url"
+	"sort"
 	"strconv"
 	"time"
 
@@ -503,22 +504,20 @@ func (api *API) remoteRead(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 
-		// Add external labels back in.
-		for _, ts := range resp.Results[i].Timeseries {
-			globalUsed := map[string]struct{}{}
-			for _, l := range ts.Labels {
-				if _, ok := externalLabels[model.LabelName(l.Name)]; ok {
-					globalUsed[l.Name] = struct{}{}
-				}
-			}
-			for ln, lv := range externalLabels {
-				if _, ok := globalUsed[string(ln)]; !ok {
-					ts.Labels = append(ts.Labels, &prompb.Label{
-						Name:  string(ln),
-						Value: string(lv),
+		// Add external labels back in, in sorted order.
+		sortedExternalLabels := make([]*prompb.Label, 0, len(externalLabels))
+		for name, value := range externalLabels {
+			sortedExternalLabels = append(sortedExternalLabels, &prompb.Label{
+				Name:  string(name),
+				Value: string(value),
 			})
 		}
-			}
+		sort.Slice(sortedExternalLabels, func(i, j int) bool {
+			return sortedExternalLabels[i].Name < sortedExternalLabels[j].Name
+		})
+
+		for _, ts := range resp.Results[i].Timeseries {
+			ts.Labels = mergeLabels(ts.Labels, sortedExternalLabels)
 		}
 	}
 
@@ -528,6 +527,33 @@ func (api *API) remoteRead(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// mergeLabels merges two sets of sorted proto labels, preferring those in
+// primary to those in secondary when there is an overlap.
+func mergeLabels(primary, secondary []*prompb.Label) []*prompb.Label {
+	result := make([]*prompb.Label, 0, len(primary)+len(secondary))
+	i, j := 0, 0
+	for i < len(primary) && j < len(secondary) {
+		if primary[i].Name < secondary[j].Name {
+			result = append(result, primary[i])
+			i++
+		} else if primary[i].Name > secondary[j].Name {
+			result = append(result, secondary[j])
+			j++
+		} else {
+			result = append(result, primary[i])
+			i++
+			j++
+		}
+	}
+	for ; i < len(primary); i++ {
+		result = append(result, primary[i])
+	}
+	for ; j < len(secondary); j++ {
+		result = append(result, secondary[j])
+	}
+	return result
+}
+
 func respond(w http.ResponseWriter, data interface{}) {
 	w.Header().Set("Content-Type", "application/json")
 	w.WriteHeader(http.StatusOK)

web/api/v1/api_test.go

@@ -563,10 +563,10 @@ func TestReadEndpoint(t *testing.T) {
 			{
 				Labels: []*prompb.Label{
 					{Name: "__name__", Value: "test_metric1"},
-					{Name: "baz", Value: "qux"},
-					{Name: "foo", Value: "bar"},
 					{Name: "b", Value: "c"},
+					{Name: "baz", Value: "qux"},
 					{Name: "d", Value: "e"},
+					{Name: "foo", Value: "bar"},
 				},
 				Samples: []*prompb.Sample{{Value: 1, Timestamp: 0}},
 			},