Fix startup/teardown order, add documentation

cmd/prometheus/main.go

@@ -132,7 +132,8 @@ func Main() int {
 		}
 	}()
 
-	// Start all components.
+	// Start all components. The order is NOT arbitrary.
+
 	if err := memStorage.Start(); err != nil {
 		log.Errorln("Error opening memory series storage:", err)
 		return 1
@@ -155,15 +156,19 @@ func Main() int {
 	prometheus.MustRegister(configSuccess)
 	prometheus.MustRegister(configSuccessTime)
 
-	go ruleManager.Run()
-	defer ruleManager.Stop()
-
+	// The notification is a dependency of the rule manager. It has to be
+	// started before and torn down afterwards.
 	go notificationHandler.Run()
 	defer notificationHandler.Stop()
 
+	go ruleManager.Run()
+	defer ruleManager.Stop()
+
 	go targetManager.Run()
 	defer targetManager.Stop()
 
+	// Shutting down the query engine before the rule manager will cause pending queries
+	// to be canceled and ensures a quick shutdown of the rule manager.
 	defer queryEngine.Stop()
 
 	go webHandler.Run()

notification/notification.go

@@ -200,6 +200,7 @@ func (n *Handler) Run() {
 }
 
 // SubmitReqs queues the given notification requests for processing.
+// Panics if called on a handler that is not running.
 func (n *Handler) Send(alerts ...*model.Alert) {
 	n.mtx.Lock()
 	defer n.mtx.Unlock()