Merge pull request #4575 from Nexucis/bugfix/fix-unregistered-source

[ServiceDiscovery] Unregister source when the target is empty

documentation/examples/custom-sd/adapter-usage/main.go

@@ -94,6 +94,7 @@ type discovery struct {
 	clientDatacenter string
 	tagSeparator     string
 	logger           log.Logger
+	oldSourceList    map[string]bool
 }
 
 func (d *discovery) parseServiceNodes(resp *http.Response, name string) (*targetgroup.Group, error) {
@@ -180,6 +181,8 @@ func (d *discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 		// list of targets simply because there may have been a timeout. If the service is actually
 		// gone as far as consul is concerned, that will be picked up during the next iteration of
 		// the outer loop.
+
+		newSourceList := make(map[string]bool)
 		for name := range srvs {
 			if name == "consul" {
 				continue
@@ -195,7 +198,17 @@ func (d *discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 				break
 			}
 			tgs = append(tgs, tg)
+			newSourceList[tg.Source] = true
 		}
+		// When targetGroup disappear, send an update with empty targetList.
+		for key := range d.oldSourceList {
+			if !newSourceList[key] {
+				tgs = append(tgs, &targetgroup.Group{
+					Source: key,
+				})
+			}
+		}
+		d.oldSourceList = newSourceList
 		if err == nil {
 			// We're returning all Consul services as a single targetgroup.
 			ch <- tgs
@@ -216,6 +229,7 @@ func newDiscovery(conf sdConfig) (*discovery, error) {
 		refreshInterval: conf.RefreshInterval,
 		tagSeparator:    conf.TagSeparator,
 		logger:          logger,
+		oldSourceList:   make(map[string]bool),
 	}
 	return cd, nil
 }

documentation/examples/custom-sd/adapter/adapter.go

@@ -54,12 +54,16 @@ func mapToArray(m map[string]*customSD) []customSD {
 	return arr
 }
 
-// Parses incoming target groups updates. If the update contains changes to the target groups
-// Adapter already knows about, or new target groups, we Marshal to JSON and write to file.
-func (a *Adapter) generateTargetGroups(allTargetGroups map[string][]*targetgroup.Group) {
-	tempGroups := make(map[string]*customSD)
+func generateTargetGroups(allTargetGroups map[string][]*targetgroup.Group) map[string]*customSD {
+	groups := make(map[string]*customSD)
 	for k, sdTargetGroups := range allTargetGroups {
 		for i, group := range sdTargetGroups {
+
+			// There is no target, so no need to keep it.
+			if len(group.Targets) <= 0 {
+				continue
+			}
+
 			newTargets := make([]string, 0)
 			newLabels := make(map[string]string)
 
@@ -74,12 +78,21 @@ func (a *Adapter) generateTargetGroups(allTargetGroups map[string][]*targetgroup
 			}
 			// Make a unique key, including the current index, in case the sd_type (map key) and group.Source is not unique.
 			key := fmt.Sprintf("%s:%s:%d", k, group.Source, i)
-			tempGroups[key] = &customSD{
+			groups[key] = &customSD{
 				Targets: newTargets,
 				Labels:  newLabels,
 			}
 		}
 	}
+
+	return groups
+}
+
+// Parses incoming target groups updates. If the update contains changes to the target groups
+// Adapter already knows about, or new target groups, we Marshal to JSON and write to file.
+func (a *Adapter) refreshTargetGroups(allTargetGroups map[string][]*targetgroup.Group) {
+	tempGroups := generateTargetGroups(allTargetGroups)
+
 	if !reflect.DeepEqual(a.groups, tempGroups) {
 		a.groups = tempGroups
 		err := a.writeOutput()
@@ -87,7 +100,6 @@ func (a *Adapter) generateTargetGroups(allTargetGroups map[string][]*targetgroup
 			level.Error(log.With(a.logger, "component", "sd-adapter")).Log("err", err)
 		}
 	}
-
 }
 
 // Writes JSON formatted targets to output file.
@@ -125,7 +137,7 @@ func (a *Adapter) runCustomSD(ctx context.Context) {
 			if !ok {
 				return
 			}
-			a.generateTargetGroups(allTargetGroups)
+			a.refreshTargetGroups(allTargetGroups)
 		}
 	}
 }

documentation/examples/custom-sd/adapter/adapter_test.go

@@ -0,0 +1,152 @@
+// Copyright 2018 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package adapter
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/prometheus/discovery/targetgroup"
+)
+
+// TestGenerateTargetGroups checks that the target is correctly generated.
+// It covers the case when the target is empty.
+func TestGenerateTargetGroups(t *testing.T) {
+	testCases := []struct {
+		title            string
+		targetGroup      map[string][]*targetgroup.Group
+		expectedCustomSD map[string]*customSD
+	}{
+		{
+			title: "Empty targetGroup",
+			targetGroup: map[string][]*targetgroup.Group{
+				"customSD": {
+					{
+						Source: "Consul",
+					},
+					{
+						Source: "Kubernetes",
+					},
+				},
+			},
+			expectedCustomSD: map[string]*customSD{},
+		},
+		{
+			title: "targetGroup filled",
+			targetGroup: map[string][]*targetgroup.Group{
+				"customSD": {
+					{
+						Source: "Azure",
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: "host1",
+							},
+							{
+								model.AddressLabel: "host2",
+							},
+						},
+						Labels: model.LabelSet{
+							model.LabelName("__meta_test_label"): model.LabelValue("label_test_1"),
+						},
+					},
+					{
+						Source: "Openshift",
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: "host3",
+							},
+							{
+								model.AddressLabel: "host4",
+							},
+						},
+						Labels: model.LabelSet{
+							model.LabelName("__meta_test_label"): model.LabelValue("label_test_2"),
+						},
+					},
+				},
+			},
+			expectedCustomSD: map[string]*customSD{
+				"customSD:Azure:0": {
+					Targets: []string{
+						"host1",
+						"host2",
+					},
+					Labels: map[string]string{
+						"__meta_test_label": "label_test_1",
+					},
+				},
+				"customSD:Openshift:1": {
+					Targets: []string{
+						"host3",
+						"host4",
+					},
+					Labels: map[string]string{
+						"__meta_test_label": "label_test_2",
+					},
+				},
+			},
+		},
+		{
+			title: "Mixed between empty targetGroup and targetGroup filled",
+			targetGroup: map[string][]*targetgroup.Group{
+				"customSD": {
+					{
+						Source: "GCE",
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: "host1",
+							},
+							{
+								model.AddressLabel: "host2",
+							},
+						},
+						Labels: model.LabelSet{
+							model.LabelName("__meta_test_label"): model.LabelValue("label_test_1"),
+						},
+					},
+					{
+						Source: "Kubernetes",
+						Labels: model.LabelSet{
+							model.LabelName("__meta_test_label"): model.LabelValue("label_test_2"),
+						},
+					},
+				},
+			},
+			expectedCustomSD: map[string]*customSD{
+				"customSD:GCE:0": {
+					Targets: []string{
+						"host1",
+						"host2",
+					},
+					Labels: map[string]string{
+						"__meta_test_label": "label_test_1",
+					},
+				},
+			},
+		},
+	}
+
+	for _, testCase := range testCases {
+		result := generateTargetGroups(testCase.targetGroup)
+
+		if !reflect.DeepEqual(result, testCase.expectedCustomSD) {
+			t.Errorf("%q failed\ngot: %#v\nexpected: %v",
+				testCase.title,
+				result,
+				testCase.expectedCustomSD)
+		}
+
+	}
+}