diff --git a/CHANGELOG.md b/CHANGELOG.md index 56a5583f8b..572872fc7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 2.11.1 / 2019-07-10 + +* [BUGFIX] Fix potential panic when prometheus is watching multiple zookeeper paths. #5749 + ## 2.11.0 / 2019-07-09 * [CHANGE] Remove `max_retries` from queue_config (it has been unused since rewriting remote-write to utilize the write-ahead-log). #5649 diff --git a/VERSION b/VERSION index 46b81d815a..6ceb272eec 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.11.0 +2.11.1 diff --git a/discovery/zookeeper/zookeeper.go b/discovery/zookeeper/zookeeper.go index ee27857219..019af105f3 100644 --- a/discovery/zookeeper/zookeeper.go +++ b/discovery/zookeeper/zookeeper.go @@ -108,8 +108,9 @@ type Discovery struct { sources map[string]*targetgroup.Group - updates chan treecache.ZookeeperTreeCacheEvent - treeCaches []*treecache.ZookeeperTreeCache + updates chan treecache.ZookeeperTreeCacheEvent + pathUpdates []chan treecache.ZookeeperTreeCacheEvent + treeCaches []*treecache.ZookeeperTreeCache parse func(data []byte, path string) (model.LabelSet, error) logger log.Logger @@ -155,7 +156,9 @@ func NewDiscovery( logger: logger, } for _, path := range paths { - sd.treeCaches = append(sd.treeCaches, treecache.NewZookeeperTreeCache(conn, path, updates, logger)) + pathUpdate := make(chan treecache.ZookeeperTreeCacheEvent) + sd.pathUpdates = append(sd.pathUpdates, pathUpdate) + sd.treeCaches = append(sd.treeCaches, treecache.NewZookeeperTreeCache(conn, path, pathUpdate, logger)) } return sd, nil } @@ -166,12 +169,26 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { for _, tc := range d.treeCaches { tc.Stop() } - // Drain event channel in case the treecache leaks goroutines otherwise. - for range d.updates { + for _, pathUpdate := range d.pathUpdates { + // Drain event channel in case the treecache leaks goroutines otherwise. + for range pathUpdate { + } } d.conn.Close() }() + for _, pathUpdate := range d.pathUpdates { + go func(update chan treecache.ZookeeperTreeCacheEvent) { + for event := range update { + select { + case d.updates <- event: + case <-ctx.Done(): + return + } + } + }(pathUpdate) + } + for { select { case <-ctx.Done():