ruler: stop all rule groups asynchronously on shutdown (#15804)

* ruler: stop all rule groups asynchronously on shutdown

During shutdown of the rules manager some rule groups have already stopped and are missing evaluations while we're waiting for other groups to finish their evaluation.

When there are many groups (in the thousands), the whole shutdown process can take up to 10 minutes, during which we get miss evaluations.

Signed-off-by: Dimitar Dimitrov <dimitar.dimitrov@grafana.com>

* Use wrappers in stop(); rename awaitStopped()

Signed-off-by: Dimitar Dimitrov <dimitar.dimitrov@grafana.com>

* Add comment

Signed-off-by: Dimitar Dimitrov <dimitar.dimitrov@grafana.com>

---------

Signed-off-by: Dimitar Dimitrov <dimitar.dimitrov@grafana.com>

rules/group.go

@@ -302,11 +302,19 @@ func (g *Group) run(ctx context.Context) {
 	}
 }
 
-func (g *Group) stop() {
+func (g *Group) stopAsync() {
 	close(g.done)
+}
+
+func (g *Group) waitStopped() {
 	<-g.terminated
 }
 
+func (g *Group) stop() {
+	g.stopAsync()
+	g.waitStopped()
+}
+
 func (g *Group) hash() uint64 {
 	l := labels.New(
 		labels.Label{Name: "name", Value: g.name},

rules/manager.go

@@ -188,8 +188,14 @@ func (m *Manager) Stop() {
 
 	m.logger.Info("Stopping rule manager...")
 
+	// Stop all groups asynchronously, then wait for them to finish.
+	// This is faster than stopping and waiting for each group in sequence.
 	for _, eg := range m.groups {
-		eg.stop()
+		eg.stopAsync()
+	}
+
+	for _, eg := range m.groups {
+		eg.waitStopped()
 	}
 
 	// Shut down the groups waiting multiple evaluation intervals to write