mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2024-11-09 23:24:05 -08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

discovery/kubernetes: Fix incorrect premature break of reading results

Browse source 
Previously `max` results stopped reading from results in tests
prematurely, as it stopped when `max` number of items were received from
the channel instead of `max` number of unique target groups received.
This caused flaky tests where the same target group was received
multiple times, as Kubernetes informers may emit the same event multiple
times.

Before this patch, running this test repeatedly failed eventually. After
this patch I have run the test many thousand times without failure.

```bash
go test -run TestEndpointsDiscoveryNamespaces -count 1000 -test.v
```

Signed-off-by: Frederic Branczyk <fbranczyk@gmail.com>
This commit is contained in:
Frederic Branczyk 2020-06-11 16:02:09 +02:00
parent b71c00e13c
commit 7b1c0d6b66
No known key found for this signature in database
GPG key ID: 7741A52782A90069
1 changed files with 9 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
discovery/kubernetes/kubernetes_test.go
View file

@ -107,14 +107,18 @@ func (d k8sDiscoveryTest) Run(t *testing.T) {
// readResultWithTimeout reads all targegroups from channel with timeout. // readResultWithTimeout reads all targegroups from channel with timeout.
// It merges targegroups by source and sends the result to result channel. // It merges targegroups by source and sends the result to result channel.
func readResultWithTimeout(t *testing.T, ch <-chan []*targetgroup.Group, max int, timeout time.Duration, resChan chan<- map[string]*targetgroup.Group) { func readResultWithTimeout(t *testing.T, ch <-chan []*targetgroup.Group, max int, timeout time.Duration, resChan chan<- map[string]*targetgroup.Group) {
allTgs := make([][]*targetgroup.Group, 0) res := make(map[string]*targetgroup.Group)
Loop: Loop:
for { for {
select { select {
case tgs := <-ch: case tgs := <-ch:
allTgs = append(allTgs, tgs) for _, tg := range tgs {
if len(allTgs) == max { if tg == nil {
continue
}
res[tg.Source] = tg
}
if len(res) == max {
// Reached max target groups we may get, break fast. // Reached max target groups we may get, break fast.
break Loop break Loop
} }
@ -122,21 +126,11 @@ Loop:
// Because we use queue, an object that is created then // Because we use queue, an object that is created then
// deleted or updated may be processed only once. // deleted or updated may be processed only once.
// So possibly we may skip events, timed out here. // So possibly we may skip events, timed out here.
t.Logf("timed out, got %d (max: %d) items, some events are skipped", len(allTgs), max) t.Logf("timed out, got %d (max: %d) items, some events are skipped", len(res), max)
break Loop break Loop
} }
} }
// Merge by source and sent it to channel.
res := make(map[string]*targetgroup.Group)
for _, tgs := range allTgs {
for _, tg := range tgs {
if tg == nil {
continue
}
res[tg.Source] = tg
}
}
resChan <- res resChan <- res
} }