diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go
index 30e8869d3c..53d820c83a 100644
--- a/cmd/prometheus/main.go
+++ b/cmd/prometheus/main.go
@@ -132,7 +132,8 @@ func Main() int {
 		}
 	}()
 
-	// Start all components.
+	// Start all components. The order is NOT arbitrary.
+
 	if err := memStorage.Start(); err != nil {
 		log.Errorln("Error opening memory series storage:", err)
 		return 1
@@ -155,15 +156,19 @@ func Main() int {
 	prometheus.MustRegister(configSuccess)
 	prometheus.MustRegister(configSuccessTime)
 
-	go ruleManager.Run()
-	defer ruleManager.Stop()
-
+	// The notification is a dependency of the rule manager. It has to be
+	// started before and torn down afterwards.
 	go notificationHandler.Run()
 	defer notificationHandler.Stop()
 
+	go ruleManager.Run()
+	defer ruleManager.Stop()
+
 	go targetManager.Run()
 	defer targetManager.Stop()
 
+	// Shutting down the query engine before the rule manager will cause pending queries
+	// to be canceled and ensures a quick shutdown of the rule manager.
 	defer queryEngine.Stop()
 
 	go webHandler.Run()
diff --git a/notification/notification.go b/notification/notification.go
index acf3b49d70..0060047e48 100644
--- a/notification/notification.go
+++ b/notification/notification.go
@@ -200,6 +200,7 @@ func (n *Handler) Run() {
 }
 
 // SubmitReqs queues the given notification requests for processing.
+// Panics if called on a handler that is not running.
 func (n *Handler) Send(alerts ...*model.Alert) {
 	n.mtx.Lock()
 	defer n.mtx.Unlock()
diff --git a/rules/manager.go b/rules/manager.go
index 212c9f9e5c..d779979b73 100644
--- a/rules/manager.go
+++ b/rules/manager.go
@@ -220,8 +220,12 @@ func (g *Group) eval() {
 
 			vector, err := rule.eval(now, g.opts.QueryEngine)
 			if err != nil {
+				// Canceled queries are intentional termination of queries. This normally
+				// happens on shutdown and thus we skip logging of any errors here.
+				if _, ok := err.(promql.ErrQueryCanceled); !ok {
+					log.Warnf("Error while evaluating rule %q: %s", rule, err)
+				}
 				evalFailures.Inc()
-				log.Warnf("Error while evaluating rule %q: %s", rule, err)
 			}
 			var rtyp ruleType