Merge pull request #1266 from prometheus/ruels

rules/ refactoring

cmd/prometheus/main.go

@@ -155,7 +155,6 @@ func Main() int {
 	prometheus.MustRegister(configSuccess)
 	prometheus.MustRegister(configSuccessTime)
 
-	go ruleManager.Run()
 	defer ruleManager.Stop()
 
 	go notificationHandler.Run()

notification/notification.go

@@ -159,6 +159,11 @@ func (n *Handler) nextBatch() []*model.Alert {
 
 // Run dispatches notifications continuously.
 func (n *Handler) Run() {
+	// Just warn once in the beginning to prevent noisy logs.
+	if n.opts.AlertmanagerURL == "" {
+		log.Warnf("No AlertManager configured, not dispatching any alerts")
+	}
+
 	for {
 		select {
 		case <-n.ctx.Done():
@@ -168,8 +173,10 @@ func (n *Handler) Run() {
 
 		alerts := n.nextBatch()
 
+		if len(alerts) == 0 {
+			continue
+		}
 		if n.opts.AlertmanagerURL == "" {
-			log.Warn("No AlertManager configured, not dispatching %d alerts", len(alerts))
 			n.dropped.Add(float64(len(alerts)))
 			continue
 		}

rules/alerting.go

@@ -38,19 +38,6 @@ const (
 // AlertState denotes the state of an active alert.
 type AlertState int
 
-func (s AlertState) String() string {
-	switch s {
-	case StateInactive:
-		return "inactive"
-	case StatePending:
-		return "pending"
-	case StateFiring:
-		return "firing"
-	default:
-		panic("undefined")
-	}
-}
-
 const (
 	// StateInactive is the state of an alert that is either firing nor pending.
 	StateInactive AlertState = iota
@@ -62,36 +49,27 @@ const (
 	StateFiring
 )
 
-// Alert is used to track active (pending/firing) alerts over time.
+func (s AlertState) String() string {
+	switch s {
+	case StateInactive:
+		return "inactive"
+	case StatePending:
+		return "pending"
+	case StateFiring:
+		return "firing"
+	}
+	panic(fmt.Errorf("unknown alert state: %v", s))
+}
+
+// Alert is the user-level representation of a single instance of an alerting rule.
 type Alert struct {
-	// The name of the alert.
-	Name string
-	// The vector element labelset triggering this alert.
-	Labels model.LabelSet
-	// The state of the alert (Pending or Firing).
 	State  AlertState
-	// The time when the alert first transitioned into Pending state.
+	Labels model.LabelSet
-	ActiveSince model.Time
+	// The value at the last evaluation of the alerting expression.
-	// The value of the alert expression for this vector element.
 	Value model.SampleValue
-}
+	// The interval during which the condition of this alert held true.
-
+	// ResolvedAt will be 0 to indicate a still active alert.
-// sample returns a Sample suitable for recording the alert.
+	ActiveAt, ResolvedAt model.Time
-func (a Alert) sample(timestamp model.Time, value model.SampleValue) *model.Sample {
-	recordedMetric := make(model.Metric, len(a.Labels)+3)
-	for label, value := range a.Labels {
-		recordedMetric[label] = value
-	}
-
-	recordedMetric[model.MetricNameLabel] = alertMetricName
-	recordedMetric[alertNameLabel] = model.LabelValue(a.Name)
-	recordedMetric[alertStateLabel] = model.LabelValue(a.State.String())
-
-	return &model.Sample{
-		Metric:    recordedMetric,
-		Value:     value,
-		Timestamp: timestamp,
-	}
 }
 
 // An AlertingRule generates alerts from its vector expression.
@@ -109,10 +87,10 @@ type AlertingRule struct {
 	annotations model.LabelSet
 
 	// Protects the below.
-	mutex sync.Mutex
+	mtx sync.Mutex
 	// A map of alerts which are currently active (Pending or Firing), keyed by
 	// the fingerprint of the labelset they correspond to.
-	activeAlerts map[model.Fingerprint]*Alert
+	active map[model.Fingerprint]*Alert
 }
 
 // NewAlertingRule constructs a new AlertingRule.
@@ -123,8 +101,7 @@ func NewAlertingRule(name string, vec promql.Expr, hold time.Duration, lbls, ann
 		holdDuration: hold,
 		labels:       lbls,
 		annotations:  anns,
-
+		active:       map[model.Fingerprint]*Alert{},
-		activeAlerts: map[model.Fingerprint]*Alert{},
 	}
 }
 
@@ -133,65 +110,145 @@ func (rule *AlertingRule) Name() string {
 	return rule.name
 }
 
+func (r *AlertingRule) sample(alert *Alert, ts model.Time, set bool) *model.Sample {
+	metric := model.Metric(r.labels.Clone())
+
+	for ln, lv := range alert.Labels {
+		metric[ln] = lv
+	}
+
+	metric[model.MetricNameLabel] = alertMetricName
+	metric[model.AlertNameLabel] = model.LabelValue(r.name)
+	metric[alertStateLabel] = model.LabelValue(alert.State.String())
+
+	s := &model.Sample{
+		Metric:    metric,
+		Timestamp: ts,
+		Value:     0,
+	}
+	if set {
+		s.Value = 1
+	}
+	return s
+}
+
+// resolvedRetention is the duration for which a resolved alert instance
+// is kept in memory state and consequentally repeatedly sent to the AlertManager.
+const resolvedRetention = 15 * time.Minute
+
 // eval evaluates the rule expression and then creates pending alerts and fires
 // or removes previously pending alerts accordingly.
-func (rule *AlertingRule) eval(timestamp model.Time, engine *promql.Engine) (model.Vector, error) {
+func (r *AlertingRule) eval(ts model.Time, engine *promql.Engine) (model.Vector, error) {
-	query, err := engine.NewInstantQuery(rule.vector.String(), timestamp)
+	query, err := engine.NewInstantQuery(r.vector.String(), ts)
 	if err != nil {
 		return nil, err
 	}
-	exprResult, err := query.Exec().Vector()
+	res, err := query.Exec().Vector()
 	if err != nil {
 		return nil, err
 	}
 
-	rule.mutex.Lock()
+	r.mtx.Lock()
-	defer rule.mutex.Unlock()
+	defer r.mtx.Unlock()
 
 	// Create pending alerts for any new vector elements in the alert expression
 	// or update the expression value for existing elements.
 	resultFPs := map[model.Fingerprint]struct{}{}
-	for _, sample := range exprResult {
+
-		fp := sample.Metric.Fingerprint()
+	for _, smpl := range res {
+		fp := smpl.Metric.Fingerprint()
 		resultFPs[fp] = struct{}{}
 
-		if alert, ok := rule.activeAlerts[fp]; !ok {
+		if alert, ok := r.active[fp]; ok {
-			labels := model.LabelSet(sample.Metric.Clone())
+			alert.Value = smpl.Value
-			labels = labels.Merge(rule.labels)
-			if _, ok := labels[model.MetricNameLabel]; ok {
-				delete(labels, model.MetricNameLabel)
-			}
-			rule.activeAlerts[fp] = &Alert{
-				Name:        rule.name,
-				Labels:      labels,
-				State:       StatePending,
-				ActiveSince: timestamp,
-				Value:       sample.Value,
-			}
-		} else {
-			alert.Value = sample.Value
-		}
-	}
-
-	var vector model.Vector
-
-	// Check if any pending alerts should be removed or fire now. Write out alert timeseries.
-	for fp, activeAlert := range rule.activeAlerts {
-		if _, ok := resultFPs[fp]; !ok {
-			vector = append(vector, activeAlert.sample(timestamp, 0))
-			delete(rule.activeAlerts, fp)
 			continue
 		}
 
-		if activeAlert.State == StatePending && timestamp.Sub(activeAlert.ActiveSince) >= rule.holdDuration {
+		delete(smpl.Metric, model.MetricNameLabel)
-			vector = append(vector, activeAlert.sample(timestamp, 0))
+
-			activeAlert.State = StateFiring
+		r.active[fp] = &Alert{
+			Labels:   model.LabelSet(smpl.Metric),
+			ActiveAt: ts,
+			State:    StatePending,
+			Value:    smpl.Value,
+		}
 	}
 
-		vector = append(vector, activeAlert.sample(timestamp, 1))
+	var vec model.Vector
+	// Check if any pending alerts should be removed or fire now. Write out alert timeseries.
+	for fp, a := range r.active {
+		if _, ok := resultFPs[fp]; !ok {
+			if a.State != StateInactive {
+				vec = append(vec, r.sample(a, ts, false))
+			}
+			// If the alert was previously firing, keep it around for a given
+			// retention time so it is reported as resolved to the AlertManager.
+			if a.State == StatePending || (a.ResolvedAt != 0 && ts.Sub(a.ResolvedAt) > resolvedRetention) {
+				delete(r.active, fp)
+			}
+			if a.State != StateInactive {
+				a.State = StateInactive
+				a.ResolvedAt = ts
+			}
+			continue
 		}
 
-	return vector, nil
+		if a.State == StatePending && ts.Sub(a.ActiveAt) >= r.holdDuration {
+			vec = append(vec, r.sample(a, ts, false))
+			a.State = StateFiring
+		}
+
+		vec = append(vec, r.sample(a, ts, true))
+	}
+
+	return vec, nil
+}
+
+// State returns the maximum state of alert instances for this rule.
+// StateFiring > StatePending > StateInactive
+func (r *AlertingRule) State() AlertState {
+	r.mtx.Lock()
+	defer r.mtx.Unlock()
+
+	maxState := StateInactive
+	for _, a := range r.active {
+		if a.State > maxState {
+			maxState = a.State
+		}
+	}
+	return maxState
+}
+
+// ActiveAlerts returns a slice of active alerts.
+func (r *AlertingRule) ActiveAlerts() []*Alert {
+	var res []*Alert
+	for _, a := range r.currentAlerts() {
+		if a.ResolvedAt == 0 {
+			res = append(res, a)
+		}
+	}
+	return res
+}
+
+// currentAlerts returns all instances of alerts for this rule. This may include
+// inactive alerts that were previously firing.
+func (r *AlertingRule) currentAlerts() []*Alert {
+	r.mtx.Lock()
+	defer r.mtx.Unlock()
+
+	alerts := make([]*Alert, 0, len(r.active))
+
+	for _, a := range r.active {
+		labels := r.labels.Clone()
+		for ln, lv := range a.Labels {
+			labels[ln] = lv
+		}
+		anew := *a
+		anew.Labels = labels
+
+		alerts = append(alerts, &anew)
+	}
+	return alerts
 }
 
 func (rule *AlertingRule) String() string {
@@ -230,29 +287,3 @@ func (rule *AlertingRule) HTMLSnippet(pathPrefix string) template.HTML {
 	}
 	return template.HTML(s)
 }
-
-// State returns the "maximum" state: firing > pending > inactive.
-func (rule *AlertingRule) State() AlertState {
-	rule.mutex.Lock()
-	defer rule.mutex.Unlock()
-
-	maxState := StateInactive
-	for _, activeAlert := range rule.activeAlerts {
-		if activeAlert.State > maxState {
-			maxState = activeAlert.State
-		}
-	}
-	return maxState
-}
-
-// ActiveAlerts returns a slice of active alerts.
-func (rule *AlertingRule) ActiveAlerts() []Alert {
-	rule.mutex.Lock()
-	defer rule.mutex.Unlock()
-
-	alerts := make([]Alert, 0, len(rule.activeAlerts))
-	for _, alert := range rule.activeAlerts {
-		alerts = append(alerts, *alert)
-	}
-	return alerts
-}

rules/manager.go

@@ -40,15 +40,13 @@ const (
 	namespace = "prometheus"
 
 	ruleTypeLabel = "rule_type"
-	ruleTypeAlerting  = "alerting"
-	ruleTypeRecording = "recording"
 )
 
 var (
 	evalDuration = prometheus.NewSummaryVec(
 		prometheus.SummaryOpts{
 			Namespace: namespace,
-			Name:      "rule_evaluation_duration_milliseconds",
+			Name:      "rule_evaluation_duration_seconds",
 			Help:      "The duration for a rule to execute.",
 		},
 		[]string{ruleTypeLabel},
@@ -60,9 +58,16 @@ var (
 			Help:      "The total number of rule evaluation failures.",
 		},
 	)
+	evalTotal = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: namespace,
+			Name:      "rule_evaluations_total",
+			Help:      "The total number of rule evaluations.",
+		},
+	)
 	iterationDuration = prometheus.NewSummary(prometheus.SummaryOpts{
 		Namespace:  namespace,
-		Name:       "evaluator_duration_milliseconds",
+		Name:       "evaluator_duration_seconds",
 		Help:       "The duration for all evaluations to execute.",
 		Objectives: map[float64]float64{0.01: 0.001, 0.05: 0.005, 0.5: 0.05, 0.90: 0.01, 0.99: 0.001},
 	})
@@ -74,12 +79,18 @@ func init() {
 	prometheus.MustRegister(evalDuration)
 }
 
+type ruleType string
+
+const (
+	ruleTypeAlert     = "alerting"
+	ruleTypeRecording = "recording"
+)
+
 // A Rule encapsulates a vector expression which is evaluated at a specified
 // interval and acted upon (currently either recorded or used for alerting).
 type Rule interface {
-	// Name returns the name of the rule.
 	Name() string
-	// Eval evaluates the rule, including any associated recording or alerting actions.
+	// eval evaluates the rule, including any associated recording or alerting actions.
 	eval(model.Time, *promql.Engine) (model.Vector, error)
 	// String returns a human-readable string representation of the rule.
 	String() string
@@ -88,306 +99,382 @@ type Rule interface {
 	HTMLSnippet(pathPrefix string) html_template.HTML
 }
 
-// The Manager manages recording and alerting rules.
+// Group is a set of rules that have a logical relation.
-type Manager struct {
+type Group struct {
-	// Protects the rules list.
+	name     string
-	sync.Mutex
-	rules []Rule
-
-	done chan bool
-
 	interval time.Duration
-	queryEngine *promql.Engine
+	rules    []Rule
+	opts     *ManagerOptions
 
-	sampleAppender      storage.SampleAppender
+	done       chan struct{}
-	notificationHandler *notification.Handler
+	terminated chan struct{}
-
-	externalURL *url.URL
 }
 
-// ManagerOptions bundles options for the Manager.
+func newGroup(name string, opts *ManagerOptions) *Group {
-type ManagerOptions struct {
+	return &Group{
-	EvaluationInterval time.Duration
+		name:       name,
-	QueryEngine        *promql.Engine
+		opts:       opts,
-
+		done:       make(chan struct{}),
-	NotificationHandler *notification.Handler
+		terminated: make(chan struct{}),
-	SampleAppender      storage.SampleAppender
+	}
-
-	ExternalURL *url.URL
 }
 
-// NewManager returns an implementation of Manager, ready to be started
+func (g *Group) run() {
-// by calling the Run method.
+	defer close(g.terminated)
-func NewManager(o *ManagerOptions) *Manager {
-	manager := &Manager{
-		rules: []Rule{},
-		done:  make(chan bool),
 
-		interval:            o.EvaluationInterval,
+	// Wait an initial amount to have consistently slotted intervals.
-		sampleAppender:      o.SampleAppender,
+	time.Sleep(g.offset())
-		queryEngine:         o.QueryEngine,
+
-		notificationHandler: o.NotificationHandler,
+	iter := func() {
-		externalURL:         o.ExternalURL,
+		start := time.Now()
-	}
+		g.eval()
-	return manager
+
+		iterationDuration.Observe(float64(time.Since(start)) / float64(time.Millisecond))
 	}
+	iter()
 
-// Run the rule manager's periodic rule evaluation.
+	tick := time.NewTicker(g.interval)
-func (m *Manager) Run() {
+	defer tick.Stop()
-	defer log.Info("Rule manager stopped.")
-
-	m.Lock()
-	lastInterval := m.interval
-	m.Unlock()
-
-	ticker := time.NewTicker(lastInterval)
-	defer ticker.Stop()
 
 	for {
-		// The outer select clause makes sure that m.done is looked at
-		// first. Otherwise, if m.runIteration takes longer than
-		// m.interval, there is only a 50% chance that m.done will be
-		// looked at before the next m.runIteration call happens.
 		select {
-		case <-m.done:
+		case <-g.done:
 			return
 		default:
 			select {
-			case <-ticker.C:
+			case <-g.done:
-				start := time.Now()
-				m.runIteration()
-				iterationDuration.Observe(float64(time.Since(start) / time.Millisecond))
-
-				m.Lock()
-				if lastInterval != m.interval {
-					ticker.Stop()
-					ticker = time.NewTicker(m.interval)
-					lastInterval = m.interval
-				}
-				m.Unlock()
-			case <-m.done:
 				return
+			case <-tick.C:
+				iter()
 			}
 		}
 	}
 }
 
-// Stop the rule manager's rule evaluation cycles.
+func (g *Group) stop() {
-func (m *Manager) Stop() {
+	close(g.done)
-	log.Info("Stopping rule manager...")
+	<-g.terminated
-	m.done <- true
 }
 
-func (m *Manager) sendAlertNotifications(rule *AlertingRule, timestamp model.Time) {
+func (g *Group) fingerprint() model.Fingerprint {
-	activeAlerts := rule.ActiveAlerts()
+	l := model.LabelSet{"name": model.LabelValue(g.name)}
-	if len(activeAlerts) == 0 {
+	return l.Fingerprint()
-		return
 }
 
-	alerts := make(model.Alerts, 0, len(activeAlerts))
+// offset returns until the next consistently slotted evaluation interval.
+func (g *Group) offset() time.Duration {
+	now := time.Now().UnixNano()
 
-	for _, aa := range activeAlerts {
+	var (
-		if aa.State != StateFiring {
+		base   = now - (now % int64(g.interval))
-			// BUG: In the future, make AlertManager support pending alerts?
+		offset = uint64(g.fingerprint()) % uint64(g.interval)
+		next   = base + int64(offset)
+	)
+
+	if next < now {
+		next += int64(g.interval)
+	}
+	return time.Duration(next - now)
+}
+
+// copyState copies the alerting rule state from the given group.
+func (g *Group) copyState(from *Group) {
+	for _, fromRule := range from.rules {
+		far, ok := fromRule.(*AlertingRule)
+		if !ok {
 			continue
 		}
-
+		for _, rule := range g.rules {
-		// Provide the alert information to the template.
+			ar, ok := rule.(*AlertingRule)
-		l := map[string]string{}
+			if !ok {
-		for k, v := range aa.Labels {
+				continue
-			l[string(k)] = string(v)
+			}
+			if far.Name() == ar.Name() {
+				ar.active = far.active
 			}
-		tmplData := struct {
-			Labels map[string]string
-			Value  float64
-		}{
-			Labels: l,
-			Value:  float64(aa.Value),
 		}
-		// Inject some convenience variables that are easier to remember for users
-		// who are not used to Go's templating system.
-		defs := "{{$labels := .Labels}}{{$value := .Value}}"
-
-		expand := func(text model.LabelValue) model.LabelValue {
-			tmpl := template.NewTemplateExpander(defs+string(text), "__alert_"+rule.Name(), tmplData, timestamp, m.queryEngine, m.externalURL.Path)
-			result, err := tmpl.Expand()
-			if err != nil {
-				result = err.Error()
-				log.Warnf("Error expanding alert template %v with data '%v': %v", rule.Name(), tmplData, err)
 	}
-			return model.LabelValue(result)
 }
 
-		labels := aa.Labels.Clone()
+// eval runs a single evaluation cycle in which all rules are evaluated in parallel.
-		labels[model.AlertNameLabel] = model.LabelValue(rule.Name())
+// In the future a single group will be evaluated sequentially to properly handle
+// rule dependency.
+func (g *Group) eval() {
+	var (
+		now = model.Now()
+		wg  sync.WaitGroup
+	)
 
-		annotations := rule.annotations.Clone()
+	for _, rule := range g.rules {
-		for an, av := range rule.annotations {
-			annotations[an] = expand(av)
-		}
-
-		alerts = append(alerts, &model.Alert{
-			StartsAt:     aa.ActiveSince.Time(),
-			Labels:       labels,
-			Annotations:  annotations,
-			GeneratorURL: m.externalURL.String() + strutil.GraphLinkForExpression(rule.vector.String()),
-		})
-	}
-	m.notificationHandler.Send(alerts...)
-}
-
-func (m *Manager) runIteration() {
-	now := model.Now()
-	wg := sync.WaitGroup{}
-
-	m.Lock()
-	rulesSnapshot := make([]Rule, len(m.rules))
-	copy(rulesSnapshot, m.rules)
-	m.Unlock()
-
-	for _, rule := range rulesSnapshot {
 		wg.Add(1)
 		// BUG(julius): Look at fixing thundering herd.
 		go func(rule Rule) {
 			defer wg.Done()
 
 			start := time.Now()
-			vector, err := rule.eval(now, m.queryEngine)
+			evalTotal.Inc()
-			duration := time.Since(start)
 
+			vector, err := rule.eval(now, g.opts.QueryEngine)
 			if err != nil {
 				evalFailures.Inc()
 				log.Warnf("Error while evaluating rule %q: %s", rule, err)
-				return
 			}
+			var rtyp ruleType
 
 			switch r := rule.(type) {
 			case *AlertingRule:
-				m.sendAlertNotifications(r, now)
+				rtyp = ruleTypeRecording
-				evalDuration.WithLabelValues(ruleTypeAlerting).Observe(
+				g.sendAlerts(r, now)
-					float64(duration / time.Millisecond),
+
-				)
 			case *RecordingRule:
-				evalDuration.WithLabelValues(ruleTypeRecording).Observe(
+				rtyp = ruleTypeAlert
-					float64(duration / time.Millisecond),
+
-				)
 			default:
 				panic(fmt.Errorf("unknown rule type: %T", rule))
 			}
 
+			evalDuration.WithLabelValues(string(rtyp)).Observe(
+				float64(time.Since(start)) / float64(time.Second),
+			)
+
 			for _, s := range vector {
-				m.sampleAppender.Append(s)
+				g.opts.SampleAppender.Append(s)
 			}
 		}(rule)
 	}
 	wg.Wait()
 }
 
-// transferAlertState makes a copy of the state of alerting rules and returns a function
+// sendAlerts sends alert notifications for the given rule.
-// that restores them in the current state.
+func (g *Group) sendAlerts(rule *AlertingRule, timestamp model.Time) error {
-func (m *Manager) transferAlertState() func() {
+	var alerts model.Alerts
 
-	alertingRules := map[string]*AlertingRule{}
+	for _, alert := range rule.currentAlerts() {
-	for _, r := range m.rules {
+		// Only send actually firing alerts.
-		if ar, ok := r.(*AlertingRule); ok {
+		if alert.State == StatePending {
-			alertingRules[ar.name] = ar
-		}
-	}
-
-	return func() {
-		// Restore alerting rule state.
-		for _, r := range m.rules {
-			ar, ok := r.(*AlertingRule)
-			if !ok {
 			continue
 		}
-			if old, ok := alertingRules[ar.name]; ok {
+
-				ar.activeAlerts = old.activeAlerts
+		// Provide the alert information to the template.
+		l := make(map[string]string, len(alert.Labels))
+		for k, v := range alert.Labels {
+			l[string(k)] = string(v)
 		}
+
+		tmplData := struct {
+			Labels map[string]string
+			Value  float64
+		}{
+			Labels: l,
+			Value:  float64(alert.Value),
 		}
+		// Inject some convenience variables that are easier to remember for users
+		// who are not used to Go's templating system.
+		defs := "{{$labels := .Labels}}{{$value := .Value}}"
+
+		expand := func(text model.LabelValue) model.LabelValue {
+			tmpl := template.NewTemplateExpander(
+				defs+string(text),
+				"__alert_"+rule.Name(),
+				tmplData,
+				timestamp,
+				g.opts.QueryEngine,
+				g.opts.ExternalURL.Path,
+			)
+			result, err := tmpl.Expand()
+			if err != nil {
+				result = fmt.Sprintf("<error expanding template: %s>", err)
+				log.Warnf("Error expanding alert template %v with data '%v': %s", rule.Name(), tmplData, err)
 			}
+			return model.LabelValue(result)
+		}
+
+		labels := make(model.LabelSet, len(alert.Labels)+1)
+		for ln, lv := range alert.Labels {
+			labels[ln] = expand(lv)
+		}
+		labels[model.AlertNameLabel] = model.LabelValue(rule.Name())
+
+		annotations := make(model.LabelSet, len(rule.annotations))
+		for an, av := range rule.annotations {
+			annotations[an] = expand(av)
+		}
+
+		a := &model.Alert{
+			StartsAt:     alert.ActiveAt.Add(rule.holdDuration).Time(),
+			Labels:       labels,
+			Annotations:  annotations,
+			GeneratorURL: g.opts.ExternalURL.String() + strutil.GraphLinkForExpression(rule.vector.String()),
+		}
+		if alert.ResolvedAt != 0 {
+			a.EndsAt = alert.ResolvedAt.Time()
+		}
+
+		alerts = append(alerts, a)
+	}
+
+	if len(alerts) > 0 {
+		g.opts.NotificationHandler.Send(alerts...)
+	}
+
+	return nil
+}
+
+// The Manager manages recording and alerting rules.
+type Manager struct {
+	opts   *ManagerOptions
+	groups map[string]*Group
+	mtx    sync.RWMutex
+}
+
+// ManagerOptions bundles options for the Manager.
+type ManagerOptions struct {
+	ExternalURL         *url.URL
+	QueryEngine         *promql.Engine
+	NotificationHandler *notification.Handler
+	SampleAppender      storage.SampleAppender
+}
+
+// NewManager returns an implementation of Manager, ready to be started
+// by calling the Run method.
+func NewManager(o *ManagerOptions) *Manager {
+	manager := &Manager{
+		groups: map[string]*Group{},
+		opts:   o,
+	}
+	return manager
+}
+
+// Stop the rule manager's rule evaluation cycles.
+func (m *Manager) Stop() {
+	log.Info("Stopping rule manager...")
+
+	for _, eg := range m.groups {
+		eg.stop()
+	}
+
+	log.Info("Rule manager stopped.")
 }
 
 // ApplyConfig updates the rule manager's state as the config requires. If
 // loading the new rules failed the old rule set is restored. Returns true on success.
 func (m *Manager) ApplyConfig(conf *config.Config) bool {
-	m.Lock()
+	m.mtx.Lock()
-	defer m.Unlock()
+	defer m.mtx.Unlock()
-
-	defer m.transferAlertState()()
-
-	success := true
-	m.interval = time.Duration(conf.GlobalConfig.EvaluationInterval)
-
-	rulesSnapshot := make([]Rule, len(m.rules))
-	copy(rulesSnapshot, m.rules)
-	m.rules = m.rules[:0]
 
+	// Get all rule files and load the groups they define.
 	var files []string
 	for _, pat := range conf.RuleFiles {
 		fs, err := filepath.Glob(pat)
 		if err != nil {
 			// The only error can be a bad pattern.
 			log.Errorf("Error retrieving rule files for %s: %s", pat, err)
-			success = false
+			return false
 		}
 		files = append(files, fs...)
 	}
-	if err := m.loadRuleFiles(files...); err != nil {
+
-		// If loading the new rules failed, restore the old rule set.
+	groups, err := m.loadGroups(files...)
-		m.rules = rulesSnapshot
+	if err != nil {
 		log.Errorf("Error loading rules, previous rule set restored: %s", err)
-		success = false
+		return false
 	}
 
-	return success
+	var wg sync.WaitGroup
+
+	for _, newg := range groups {
+		// To be replaced with a configurable per-group interval.
+		newg.interval = time.Duration(conf.GlobalConfig.EvaluationInterval)
+
+		wg.Add(1)
+
+		// If there is an old group with the same identifier, stop it and wait for
+		// it to finish the current iteration. Then copy its into the new group.
+		oldg, ok := m.groups[newg.name]
+		delete(m.groups, newg.name)
+
+		go func(newg *Group) {
+			if ok {
+				oldg.stop()
+				newg.copyState(oldg)
+			}
+			go newg.run()
+			wg.Done()
+		}(newg)
 	}
 
-// loadRuleFiles loads alerting and recording rules from the given files.
+	// Stop remaining old groups.
-func (m *Manager) loadRuleFiles(filenames ...string) error {
+	for _, oldg := range m.groups {
+		oldg.stop()
+	}
+
+	wg.Wait()
+	m.groups = groups
+
+	return true
+}
+
+// loadGroups reads groups from a list of files.
+// As there's currently no group syntax a single group named "default" containing
+// all rules will be returned.
+func (m *Manager) loadGroups(filenames ...string) (map[string]*Group, error) {
+	groups := map[string]*Group{}
+
+	// Currently there is no group syntax implemented. Thus all rules
+	// are read into a single default group.
+	g := newGroup("default", m.opts)
+	groups[g.name] = g
+
 	for _, fn := range filenames {
 		content, err := ioutil.ReadFile(fn)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		stmts, err := promql.ParseStmts(string(content))
 		if err != nil {
-			return fmt.Errorf("error parsing %s: %s", fn, err)
+			return nil, fmt.Errorf("error parsing %s: %s", fn, err)
 		}
 
 		for _, stmt := range stmts {
+			var rule Rule
+
 			switch r := stmt.(type) {
 			case *promql.AlertStmt:
-				rule := NewAlertingRule(r.Name, r.Expr, r.Duration, r.Labels, r.Annotations)
+				rule = NewAlertingRule(r.Name, r.Expr, r.Duration, r.Labels, r.Annotations)
-				m.rules = append(m.rules, rule)
+
 			case *promql.RecordStmt:
-				rule := NewRecordingRule(r.Name, r.Expr, r.Labels)
+				rule = NewRecordingRule(r.Name, r.Expr, r.Labels)
-				m.rules = append(m.rules, rule)
+
 			default:
 				panic("retrieval.Manager.LoadRuleFiles: unknown statement type")
 			}
+			g.rules = append(g.rules, rule)
 		}
 	}
-	return nil
+
+	return groups, nil
 }
 
 // Rules returns the list of the manager's rules.
 func (m *Manager) Rules() []Rule {
-	m.Lock()
+	m.mtx.RLock()
-	defer m.Unlock()
+	defer m.mtx.RUnlock()
+
+	var rules []Rule
+	for _, g := range m.groups {
+		rules = append(rules, g.rules...)
+	}
 
-	rules := make([]Rule, len(m.rules))
-	copy(rules, m.rules)
 	return rules
 }
 
 // AlertingRules returns the list of the manager's alerting rules.
 func (m *Manager) AlertingRules() []*AlertingRule {
-	m.Lock()
+	m.mtx.RLock()
-	defer m.Unlock()
+	defer m.mtx.RUnlock()
 
 	alerts := []*AlertingRule{}
-	for _, rule := range m.rules {
+	for _, rule := range m.Rules() {
 		if alertingRule, ok := rule.(*AlertingRule); ok {
 			alerts = append(alerts, alertingRule)
 		}

rules/manager_test.go

@@ -15,7 +15,6 @@ package rules
 
 import (
 	"fmt"
-	"reflect"
 	"strings"
 	"testing"
 	"time"
@@ -138,46 +137,3 @@ func annotateWithTime(lines []string, timestamp model.Time) []string {
 	}
 	return annotatedLines
 }
-
-func TestTransferAlertState(t *testing.T) {
-	m := NewManager(&ManagerOptions{})
-
-	alert := &Alert{
-		Name:  "testalert",
-		State: StateFiring,
-	}
-
-	arule := AlertingRule{
-		name:         "test",
-		activeAlerts: map[model.Fingerprint]*Alert{},
-	}
-	aruleCopy := arule
-
-	m.rules = append(m.rules, &arule)
-
-	// Set an alert.
-	arule.activeAlerts[0] = alert
-
-	// Save state and get the restore function.
-	restore := m.transferAlertState()
-
-	// Remove arule from the rule list and add an unrelated rule and the
-	// stateless copy of arule.
-	m.rules = []Rule{
-		&AlertingRule{
-			name:         "test_other",
-			activeAlerts: map[model.Fingerprint]*Alert{},
-		},
-		&aruleCopy,
-	}
-
-	// Apply the restore function.
-	restore()
-
-	if ar := m.rules[0].(*AlertingRule); len(ar.activeAlerts) != 0 {
-		t.Fatalf("unexpected alert for unrelated alerting rule")
-	}
-	if ar := m.rules[1].(*AlertingRule); !reflect.DeepEqual(ar.activeAlerts[0], alert) {
-		t.Fatalf("alert state was not restored")
-	}
-}

rules/recording.go

@@ -40,7 +40,9 @@ func NewRecordingRule(name string, vector promql.Expr, labels model.LabelSet) *R
 }
 
 // Name returns the rule name.
-func (rule RecordingRule) Name() string { return rule.name }
+func (rule RecordingRule) Name() string {
+	return rule.name
+}
 
 // eval evaluates the rule and then overrides the metric names and labels accordingly.
 func (rule RecordingRule) eval(timestamp model.Time, engine *promql.Engine) (model.Vector, error) {

web/ui/bindata.go

@@ -119,7 +119,7 @@ func webUiTemplates_baseHtml() (*asset, error) {
 	return a, nil
 }
 
-var _webUiTemplatesAlertsHtml = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\x8c\x54\x4d\x8f\x9b\x30\x10\xbd\xef\xaf\xb0\xd0\x1e\x5a\xa9\x80\xd4\x63\x45\x90\x56\x7b\xe9\x61\x5b\x55\x9b\x74\xaf\x91\xb1\x27\x8b\xb7\x8e\x41\xb6\x93\x4d\xe4\xf2\xdf\x3b\xb6\x21\x4b\x08\x48\xbd\x24\x78\x66\xde\xf3\xf3\x7c\x39\xc7\x61\x27\x14\x90\xa4\x06\xca\x93\xae\xbb\x23\xa4\x90\x42\xfd\x21\xf6\xdc\xc2\x2a\xb1\x70\xb2\x39\x33\x26\x21\x1a\xe4\x2a\x31\xf6\x2c\xc1\xd4\x00\x36\x21\xb5\x86\xdd\x2a\x71\x8e\xb4\xd4\xd6\xbf\xf0\x20\x4e\xa4\xeb\x72\x63\xa9\x15\xcc\x63\x72\x2a\x41\x5b\x93\x79\x78\xe9\x79\x0d\xd3\xa2\xb5\xc4\x68\xb6\x8c\x7b\xbb\xc0\xde\x10\x55\xe4\x11\x53\xde\x39\x07\x8a\xa3\xbc\xbb\x0f\xc5\xac\x51\x16\x94\xf5\xa2\x0b\x2e\x8e\x84\x49\x6a\xcc\x2a\x98\x29\x06\xe8\x74\x27\x0f\x82\xc7\xab\xeb\xaf\xe5\x43\xa0\x2d\x72\xfc\xf4\x16\x4b\x2b\x09\x03\x26\x1e\xc2\x6f\x5a\x35\x9a\x83\x06\xde\x1f\x59\x23\x25\x6d\x0d\x44\x22\x0f\xac\x1a\x7e\x8e\xdf\xce\xdd\x07\xb1\x6b\xd4\x0e\x9b\xe6\xb9\x79\x7f\xf4\x7c\xe4\xdb\x8a\x64\x0f\x33\x8e\x90\x5e\x0f\xd3\x54\xbd\x42\x1f\x23\xd4\xeb\xf3\x01\xb3\xda\x3b\x23\x2b\xb3\xe2\x08\x51\x71\x64\x1b\x19\x2e\x81\x85\xd5\xc3\x03\x9c\x13\x8a\xc3\x89\xcc\xeb\xc9\x82\xa1\xeb\x48\xf0\x6e\x7d\xa9\x41\xf7\xef\x89\x44\xbc\x2c\xc4\xc0\x25\x30\x83\x29\xab\xe1\xa8\xf1\x9f\x37\xef\xca\xd7\x41\x94\xa4\xa8\x4a\xe7\xb2\x9f\x74\x8f\x4c\x45\x5e\x95\xe4\x93\x73\x12\x14\xb9\x52\xeb\x2f\x09\xc7\xcf\x45\x8e\xac\x83\xd2\xdc\xea\xf2\x56\x75\x94\xc3\x01\xeb\x25\xcd\x44\xcf\xe5\x80\x47\xac\xee\xf8\x8c\x96\x56\x43\x59\xb0\x86\x83\x97\xf4\x7d\xf3\xe3\x69\xad\x44\xdb\x82\x1d\x35\x95\x17\x19\x22\x8a\xdc\x47\x8f\xf9\xf2\x09\x21\x66\x6f\x37\x7d\xc6\x38\xfe\x7f\x7b\xa5\x6e\x8e\xa0\x2f\x7d\x83\x05\x51\xd8\x37\x7d\xd2\x41\xc2\x1e\xbb\xd5\x6c\x83\x3b\x99\xbc\xe7\x23\x27\x13\x8f\xf7\xd5\xe5\x13\xad\x40\x62\xef\xe2\xe7\x8c\x37\x54\x77\xc9\x19\x3b\x87\xac\x85\x62\x8b\x31\x2f\x54\x1e\x16\x9d\x6b\x81\x45\x9e\xc3\x8e\x8b\x3a\xe4\x31\x36\xf6\x72\x2a\xc3\x53\x6f\x6f\xe1\x53\xd3\x88\x4b\xfa\xb7\x7f\x21\xf7\x47\x2f\x32\x0c\x43\xcc\xc6\x84\xb7\xa7\x32\x2d\x55\x43\x2a\x03\x92\x84\xdf\xb4\xd5\x62\x4f\xf5\x39\xc1\x7e\x89\x8c\x5d\xe7\xa7\x26\xb2\x76\x5d\x82\x9b\x06\x91\x73\x32\xe2\xde\x99\x5c\x93\xdf\x4a\x0e\x43\x34\xbe\x3e\xd4\x3d\x56\x3f\xc5\x75\x17\x87\x90\xfc\x25\xe3\x11\x8d\xf3\x89\x43\xe3\xd7\x1f\x6c\x71\x88\x05\xa3\xb6\xc1\x26\xc2\xc5\x9b\x1e\xb0\xa5\x35\xa3\x06\xbc\xec\x61\x88\x7b\xa5\x4b\x12\x30\x30\x96\x3c\x54\x3c\xdb\x88\x3d\x64\xbf\x37\x8f\x1e\xb7\x08\x78\x89\x49\xb8\x8d\x98\x2b\xf1\x34\x1f\x18\xe3\x3b\xfa\x7a\x9e\xae\x83\xe6\x57\xc1\x38\x0a\xad\xc3\x5a\xbd\xf0\xf5\x63\x3a\x84\xfd\x0b\x00\x00\xff\xff\xdb\x8e\x0c\x63\xab\x06\x00\x00")
+var _webUiTemplatesAlertsHtml = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\x8c\x54\x4d\x8f\x9b\x30\x10\xbd\xef\xaf\xb0\xd0\x1e\x5a\xa9\x80\xd4\x63\x45\x90\x56\x7b\xe9\x61\x5b\x55\x9b\x74\xaf\x91\xb1\x27\x8b\xb7\x8e\x41\xb6\x93\x4d\xe4\xf2\xdf\x3b\xb6\x21\x4b\x08\x48\xbd\x24\x78\xe6\xcd\xf3\xf3\x7c\x39\xc7\x61\x27\x14\x90\xa4\x06\xca\x93\xae\xbb\x23\xa4\x90\x42\xfd\x21\xf6\xdc\xc2\x2a\xb1\x70\xb2\x39\x33\x26\x21\x1a\xe4\x2a\x31\xf6\x2c\xc1\xd4\x00\x36\x21\xb5\x86\xdd\x2a\x71\x8e\xb4\xd4\xd6\xbf\xf0\x20\x4e\xa4\xeb\x72\x63\xa9\x15\xcc\xc7\xe4\x54\x82\xb6\x26\xf3\xe1\xa5\xe7\x35\x4c\x8b\xd6\x12\xa3\xd9\x72\xdc\xdb\x25\xec\x0d\xa3\x8a\x3c\xc6\x94\x77\xce\x81\xe2\x28\xef\xee\x43\x31\x6b\x94\x05\x65\xbd\xe8\x82\x8b\x23\x61\x92\x1a\xb3\x0a\x66\x8a\x00\x9d\xee\xe4\x41\xf0\x78\x75\xfd\xb5\x7c\x08\xb4\x45\x8e\x9f\xde\x62\x69\x25\x61\x88\x89\x87\xf0\x9b\x56\x8d\xe6\xa0\x81\xf7\x47\xd6\x48\x49\x5b\x03\x91\xc8\x07\x56\x0d\x3f\xc7\x6f\xe7\xee\x83\xd8\x35\x6a\x87\x4d\xf3\xdc\xbc\x3f\x7a\x3e\xf2\x6d\x45\xb2\x87\x19\x47\x48\xaf\x0f\xd3\x54\xbd\x42\x8f\x11\xea\xf5\xf9\x80\x59\xed\x9d\x91\x95\x59\x71\x84\xa8\x38\xb2\x8d\x0c\x17\x60\x61\xf5\xf0\x00\xe7\x84\xe2\x70\x22\xf3\x7a\xb2\x60\xe8\x3a\x12\xbc\x5b\x5f\x6a\xd0\xfd\x7b\x22\x11\x2f\x0b\x31\x70\x09\xcc\x60\xca\x6a\x38\x6a\xfc\xe7\xcd\xbb\xf2\x75\x10\x25\x29\xaa\xd2\xb9\xec\x27\xdd\x23\x53\x91\x57\x25\xf9\xe4\x9c\x04\x45\xae\xd4\xfa\x4b\xc2\xf1\x73\x91\x23\xeb\xa0\x34\xb7\xba\xbc\x55\x1d\xe5\x70\xc0\x7a\x49\x33\xd1\x73\x39\xe0\x11\xab\x3b\x3e\xa3\xa5\xd5\x50\x16\xac\xe1\xe0\x25\x7d\xdf\xfc\x78\x5a\x2b\xd1\xb6\x60\x47\x4d\xe5\x45\x06\x44\x91\x7b\xf4\x98\x2f\x9f\x10\x62\xf6\x76\xd3\x67\x8c\xf1\xff\xdb\x2b\x75\x73\x04\x7d\xe9\x1b\x2c\x88\xc2\xbe\xe9\x93\x0e\x12\xf6\xd8\xad\x66\x1b\xdc\xc9\xe4\x3d\x1f\x39\x99\x78\xbc\xaf\x2e\x9f\x68\x05\x12\x7b\x17\x3f\x67\xbc\xa1\xba\x4b\xce\xd8\x39\x64\x2d\x14\x5b\xc4\xbc\x50\x79\x58\x74\xae\x05\x16\x79\x2e\x76\x5c\xd4\x21\x8f\xb1\xb1\x97\x53\x19\x9e\x7a\x7b\x0b\x9f\x9a\x46\x5c\xd2\xbf\xfd\x0b\xb9\x3f\x7a\x91\x61\x18\x62\x36\x26\xbc\x3d\x95\x69\xa9\x1a\x52\x19\x22\x49\xf8\x4d\x5b\x2d\xf6\x54\x9f\x13\xec\x97\xc8\xd8\x75\x7e\x6a\x22\x6b\xd7\x25\xb8\x69\x30\x72\x4e\x46\xdc\x3b\x93\x6b\xf2\x5b\xc9\x61\x88\xc6\xd7\x87\xba\xc7\xea\xa7\xb8\xee\xe2\x10\x92\xbf\x64\x3c\xa2\x71\x3e\x71\x68\xfc\xfa\x83\x2d\x0e\xb1\x60\xd4\x36\xd8\x44\xb8\x78\xd3\x03\xb6\xb4\x66\xd4\x80\x97\x3d\x0c\x71\xaf\x74\x49\x02\x02\xfb\x65\x61\xb3\x8d\xd8\x43\xf6\x7b\xf3\xe8\x83\x16\xd1\x2f\x31\x03\xb7\x88\xb9\xfa\x4e\x93\x81\x18\xdf\xce\xd7\xc3\x74\x0d\x9a\xdf\x03\x63\x14\x5a\x87\x9d\x7a\xe1\xeb\x67\x74\x80\xfd\x0b\x00\x00\xff\xff\xb3\xf6\xaa\xca\xa8\x06\x00\x00")
 
 func webUiTemplatesAlertsHtmlBytes() ([]byte, error) {
 	return bindataRead(
@@ -134,7 +134,7 @@ func webUiTemplatesAlertsHtml() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/templates/alerts.html", size: 1707, mode: os.FileMode(420), modTime: time.Unix(1450269200, 0)}
+	info := bindataFileInfo{name: "web/ui/templates/alerts.html", size: 1704, mode: os.FileMode(420), modTime: time.Unix(1450348695, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }

web/ui/templates/alerts.html

@@ -36,7 +36,7 @@
                 {{end}}
               </td>
               <td><span class="alert alert-{{ .State | alertStateToClass }} state_indicator text-uppercase">{{.State}}</span></td>
-              <td>{{.ActiveSince.Time.UTC}}</td>
+              <td>{{.ActiveAt.Time.UTC}}</td>
               <td>{{.Value}}</td>
             </tr>
             {{end}}