Rework #5009 after comments

Signed-off-by: Bjoern Rabenstein <bjoern@rabenste.in>

cmd/prometheus/main.go

@@ -443,7 +443,11 @@ func main() {
 				}
 				files = append(files, fs...)
 			}
-			return ruleManager.Update(time.Duration(cfg.GlobalConfig.EvaluationInterval), files)
+			return ruleManager.Update(
+				time.Duration(cfg.GlobalConfig.EvaluationInterval),
+				files,
+				cfg.GlobalConfig.ExternalLabels,
+			)
 		},
 	}
 
@@ -747,13 +751,8 @@ func reloadConfig(filename string, logger log.Logger, rls ...func(*config.Config
 	if failed {
 		return errors.Errorf("one or more errors occurred while applying the new configuration (--config.file=%q)", filename)
 	}
+
 	promql.SetDefaultEvaluationInterval(time.Duration(conf.GlobalConfig.EvaluationInterval))
-
-	// Register conf as current config.
-	config.CurrentConfigMutex.Lock()
-	config.CurrentConfig = conf
-	config.CurrentConfigMutex.Unlock()
-
 	level.Info(logger).Log("msg", "Completed loading of configuration file", "filename", filename)
 	return nil
 }

cmd/promtool/unittest.go

@@ -163,7 +163,8 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou
 		Logger:     &dummyLogger{},
 	}
 	m := rules.NewManager(opts)
-	groupsMap, ers := m.LoadGroups(tg.Interval, ruleFiles...)
+	// TODO(beorn7): Provide a way to pass in external labels.
+	groupsMap, ers := m.LoadGroups(tg.Interval, nil, ruleFiles...)
 	if ers != nil {
 		return ers
 	}

config/config.go

@@ -20,7 +20,6 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/pkg/errors"
@@ -69,10 +68,6 @@ func LoadFile(filename string) (*Config, error) {
 
 // The defaults applied before parsing the respective config sections.
 var (
-	// CurrentConfig is a pointer to the current configuration.
-	CurrentConfig      *Config
-	CurrentConfigMutex sync.RWMutex
-
 	// DefaultConfig is the default top-level configuration.
 	DefaultConfig = Config{
 		GlobalConfig: DefaultGlobalConfig,

docs/configuration/alerting_rules.md

@@ -42,9 +42,11 @@ The `annotations` clause specifies a set of informational labels that can be use
 
 #### Templating
 
-Label and annotation values can be templated using [console templates](https://prometheus.io/docs/visualization/consoles).
+Label and annotation values can be templated using [console
-The `$labels` & `$externalLabels` variables hold the label key/value pairs of an alert instance
+templates](https://prometheus.io/docs/visualization/consoles).  The `$labels`
-and `$value` holds the evaluated value of an alert instance.
+variable holds the label key/value pairs of an alert instance. The configured
+external labels can be accessed via the `$externalLabels` variable. The
+`$value` variable holds the evaluated value of an alert instance.
 
     # To insert a firing element's label values:
     {{ $labels.<labelname> }}

docs/configuration/template_reference.md

@@ -89,8 +89,10 @@ parameterize templates, and have a few other differences.
 
 ### Alert field templates
 
-`.Value` and `.Labels` contain the alert value and labels. They are also exposed
+`.Value`, `.Labels`, and `ExternalLabels` contain the alert value, the alert
-as the `$value` and `$labels` variables for convenience.
+labels, and the globally configured external labels, respectively. They are
+also exposed as the `$value`, `$labels`, and `$externalLabels` variables for
+convenience.
 
 ### Console templates
 
@@ -104,7 +106,9 @@ auto-escaping. To bypass the auto-escaping use the `safe*` functions.,
 URL parameters are available as a map in `.Params`. To access multiple URL
 parameters by the same name, `.RawParams` is a map of the list values for each
 parameter. The URL path is available in `.Path`, excluding the `/consoles/`
-prefix.
+prefix. The globally configured external labels are available as
+`.ExternalLabels`. There are also convenience variables for all four:
+`$rawParams`, `$params`, `$path`, and `$externalLabels`.
 
 Consoles also have access to all the templates defined with `{{define
 "templateName"}}...{{end}}` found in `*.lib` files in the directory pointed to

pkg/rulefmt/rulefmt.go

@@ -16,6 +16,7 @@ package rulefmt
 import (
 	"context"
 	"io/ioutil"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -155,12 +156,16 @@ func testTemplateParsing(rl *Rule) (errs []error) {
 	}
 
 	// Trying to parse templates.
-	tmplData := template.AlertTemplateData(make(map[string]string), make(map[string]string), 0)
+	tmplData := template.AlertTemplateData(map[string]string{}, map[string]string{}, 0)
-	defs := "{{$labels := .Labels}}{{$value := .Value}}"
+	defs := []string{
+		"{{$labels := .Labels}}",
+		"{{$externalLabels := .ExternalLabels}}",
+		"{{$value := .Value}}",
+	}
 	parseTest := func(text string) error {
 		tmpl := template.NewTemplateExpander(
 			context.TODO(),
-			defs+text,
+			strings.Join(append(defs, text), ""),
 			"__alert_"+rl.Alert,
 			tmplData,
 			model.Time(timestamp.FromTime(time.Now())),

rules/alerting.go

@@ -17,6 +17,7 @@ import (
 	"context"
 	"fmt"
 	"net/url"
+	"strings"
 	"sync"
 	"time"
 
@@ -28,7 +29,6 @@ import (
 	"github.com/go-kit/kit/log/level"
 	"github.com/pkg/errors"
 	"github.com/prometheus/common/model"
-	"github.com/prometheus/prometheus/config"
 
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/prometheus/prometheus/pkg/rulefmt"
@@ -80,9 +80,8 @@ func (s AlertState) String() string {
 type Alert struct {
 	State AlertState
 
-	Labels         labels.Labels
+	Labels      labels.Labels
-	ExternalLabels labels.Labels
+	Annotations labels.Labels
-	Annotations    labels.Labels
 
 	// The value at the last evaluation of the alerting expression.
 	Value float64
@@ -121,6 +120,8 @@ type AlertingRule struct {
 	labels labels.Labels
 	// Non-identifying key/value pairs.
 	annotations labels.Labels
+	// External labels from the global config.
+	externalLabels map[string]string
 	// true if old state has been restored. We start persisting samples for ALERT_FOR_STATE
 	// only after the restoration.
 	restored bool
@@ -142,17 +143,27 @@ type AlertingRule struct {
 }
 
 // NewAlertingRule constructs a new AlertingRule.
-func NewAlertingRule(name string, vec promql.Expr, hold time.Duration, lbls, anns labels.Labels, restored bool, logger log.Logger) *AlertingRule {
+func NewAlertingRule(
+	name string, vec promql.Expr, hold time.Duration,
+	labels, annotations, externalLabels labels.Labels,
+	restored bool, logger log.Logger,
+) *AlertingRule {
+	el := make(map[string]string, len(externalLabels))
+	for _, lbl := range externalLabels {
+		el[lbl.Name] = lbl.Value
+	}
+
 	return &AlertingRule{
-		name:         name,
+		name:           name,
-		vector:       vec,
+		vector:         vec,
-		holdDuration: hold,
+		holdDuration:   hold,
-		labels:       lbls,
+		labels:         labels,
-		annotations:  anns,
+		annotations:    annotations,
-		health:       HealthUnknown,
+		externalLabels: el,
-		active:       map[uint64]*Alert{},
+		health:         HealthUnknown,
-		logger:       logger,
+		active:         map[uint64]*Alert{},
-		restored:     restored,
+		logger:         logger,
+		restored:       restored,
 	}
 }
 
@@ -302,32 +313,24 @@ func (r *AlertingRule) Eval(ctx context.Context, ts time.Time, query QueryFunc,
 	var vec promql.Vector
 	for _, smpl := range res {
 		// Provide the alert information to the template.
-		l := make(map[string]string)
+		l := make(map[string]string, len(smpl.Metric))
 		for _, lbl := range smpl.Metric {
 			l[lbl.Name] = lbl.Value
 		}
 
-		// Add external labels.
+		tmplData := template.AlertTemplateData(l, r.externalLabels, smpl.V)
-		el := make(map[string]string)
-		if config.CurrentConfig != nil {
-			config.CurrentConfigMutex.RLock()
-			for eln, elv := range (*config.CurrentConfig).GlobalConfig.ExternalLabels {
-				el[string(eln)] = string(elv)
-			}
-			config.CurrentConfigMutex.RUnlock()
-		}
-
-		tmplData := template.AlertTemplateData(l, el, smpl.V)
 		// Inject some convenience variables that are easier to remember for users
 		// who are not used to Go's templating system.
-		defs := "{{$labels := .Labels}}"
+		defs := []string{
-		defs += "{{$externalLabels := .ExternalLabels}}"
+			"{{$labels := .Labels}}",
-		defs += "{{$value := .Value}}"
+			"{{$externalLabels := .ExternalLabels}}",
+			"{{$value := .Value}}",
+		}
 
 		expand := func(text string) string {
 			tmpl := template.NewTemplateExpander(
 				ctx,
-				defs+text,
+				strings.Join(append(defs, text), ""),
 				"__alert_"+r.Name(),
 				tmplData,
 				model.Time(timestamp.FromTime(ts)),
@@ -463,7 +466,7 @@ func (r *AlertingRule) ForEachActiveAlert(f func(*Alert)) {
 }
 
 func (r *AlertingRule) sendAlerts(ctx context.Context, ts time.Time, resendDelay time.Duration, interval time.Duration, notifyFunc NotifyFunc) {
-	alerts := make([]*Alert, 0)
+	alerts := []*Alert{}
 	r.ForEachActiveAlert(func(alert *Alert) {
 		if alert.needsSending(ts, resendDelay) {
 			alert.LastSentAt = ts

rules/alerting_test.go

@@ -26,7 +26,7 @@ import (
 func TestAlertingRuleHTMLSnippet(t *testing.T) {
 	expr, err := promql.ParseExpr(`foo{html="<b>BOLD<b>"}`)
 	testutil.Ok(t, err)
-	rule := NewAlertingRule("testrule", expr, 0, labels.FromStrings("html", "<b>BOLD</b>"), labels.FromStrings("html", "<b>BOLD</b>"), false, nil)
+	rule := NewAlertingRule("testrule", expr, 0, labels.FromStrings("html", "<b>BOLD</b>"), labels.FromStrings("html", "<b>BOLD</b>"), nil, false, nil)
 
 	const want = `alert: <a href="/test/prefix/graph?g0.expr=ALERTS%7Balertname%3D%22testrule%22%7D&g0.tab=1">testrule</a>
 expr: <a href="/test/prefix/graph?g0.expr=foo%7Bhtml%3D%22%3Cb%3EBOLD%3Cb%3E%22%7D&g0.tab=1">foo{html=&#34;&lt;b&gt;BOLD&lt;b&gt;&#34;}</a>
@@ -62,7 +62,7 @@ func TestAlertingRuleLabelsUpdate(t *testing.T) {
 		// If an alert is going back and forth between two label values it will never fire.
 		// Instead, you should write two alerts with constant labels.
 		labels.FromStrings("severity", "{{ if lt $value 80.0 }}critical{{ else }}warning{{ end }}"),
-		nil, true, nil,
+		nil, nil, true, nil,
 	)
 
 	results := []promql.Vector{

rules/manager.go

@@ -751,11 +751,11 @@ func (m *Manager) Stop() {
 
 // Update the rule manager's state as the config requires. If
 // loading the new rules failed the old rule set is restored.
-func (m *Manager) Update(interval time.Duration, files []string) error {
+func (m *Manager) Update(interval time.Duration, files []string, externalLabels labels.Labels) error {
 	m.mtx.Lock()
 	defer m.mtx.Unlock()
 
-	groups, errs := m.LoadGroups(interval, files...)
+	groups, errs := m.LoadGroups(interval, externalLabels, files...)
 	if errs != nil {
 		for _, e := range errs {
 			level.Error(m.logger).Log("msg", "loading groups failed", "err", e)
@@ -803,7 +803,9 @@ func (m *Manager) Update(interval time.Duration, files []string) error {
 }
 
 // LoadGroups reads groups from a list of files.
-func (m *Manager) LoadGroups(interval time.Duration, filenames ...string) (map[string]*Group, []error) {
+func (m *Manager) LoadGroups(
+	interval time.Duration, externalLabels labels.Labels, filenames ...string,
+) (map[string]*Group, []error) {
 	groups := make(map[string]*Group)
 
 	shouldRestore := !m.restored
@@ -834,6 +836,7 @@ func (m *Manager) LoadGroups(interval time.Duration, filenames ...string) (map[s
 						time.Duration(r.For),
 						labels.FromMap(r.Labels),
 						labels.FromMap(r.Annotations),
+						externalLabels,
 						m.restored,
 						log.With(m.logger, "alert", r.Alert),
 					))

rules/manager_test.go

@@ -51,7 +51,7 @@ func TestAlertingRule(t *testing.T) {
 		expr,
 		time.Minute,
 		labels.FromStrings("severity", "{{\"c\"}}ritical"),
-		nil, true, nil,
+		nil, nil, true, nil,
 	)
 	result := promql.Vector{
 		{
@@ -192,7 +192,7 @@ func TestForStateAddSamples(t *testing.T) {
 		expr,
 		time.Minute,
 		labels.FromStrings("severity", "{{\"c\"}}ritical"),
-		nil, true, nil,
+		nil, nil, true, nil,
 	)
 	result := promql.Vector{
 		{
@@ -366,7 +366,7 @@ func TestForStateRestore(t *testing.T) {
 		expr,
 		alertForDuration,
 		labels.FromStrings("severity", "critical"),
-		nil, true, nil,
+		nil, nil, true, nil,
 	)
 
 	group := NewGroup("default", "", time.Second, []Rule{rule}, true, opts)
@@ -426,7 +426,7 @@ func TestForStateRestore(t *testing.T) {
 			expr,
 			alertForDuration,
 			labels.FromStrings("severity", "critical"),
-			nil, false, nil,
+			nil, nil, false, nil,
 		)
 		newGroup := NewGroup("default", "", time.Second, []Rule{newRule}, true, opts)
 
@@ -581,12 +581,12 @@ func readSeriesSet(ss storage.SeriesSet) (map[string][]promql.Point, error) {
 func TestCopyState(t *testing.T) {
 	oldGroup := &Group{
 		rules: []Rule{
-			NewAlertingRule("alert", nil, 0, nil, nil, true, nil),
+			NewAlertingRule("alert", nil, 0, nil, nil, nil, true, nil),
 			NewRecordingRule("rule1", nil, nil),
 			NewRecordingRule("rule2", nil, nil),
 			NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v1"}}),
 			NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v2"}}),
-			NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, true, nil),
+			NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, nil, true, nil),
 		},
 		seriesInPreviousEval: []map[string]labels.Labels{
 			{"a": nil},
@@ -604,10 +604,10 @@ func TestCopyState(t *testing.T) {
 			NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v0"}}),
 			NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v1"}}),
 			NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v2"}}),
-			NewAlertingRule("alert", nil, 0, nil, nil, true, nil),
+			NewAlertingRule("alert", nil, 0, nil, nil, nil, true, nil),
 			NewRecordingRule("rule1", nil, nil),
-			NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v0"}}, nil, true, nil),
+			NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v0"}}, nil, nil, true, nil),
-			NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, true, nil),
+			NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, nil, true, nil),
 			NewRecordingRule("rule4", nil, nil),
 		},
 		seriesInPreviousEval: make([]map[string]labels.Labels, 8),
@@ -654,7 +654,7 @@ func TestUpdate(t *testing.T) {
 	ruleManager.Run()
 	defer ruleManager.Stop()
 
-	err := ruleManager.Update(10*time.Second, files)
+	err := ruleManager.Update(10*time.Second, files, nil)
 	testutil.Ok(t, err)
 	testutil.Assert(t, len(ruleManager.groups) > 0, "expected non-empty rule groups")
 	for _, g := range ruleManager.groups {
@@ -663,7 +663,7 @@ func TestUpdate(t *testing.T) {
 		}
 	}
 
-	err = ruleManager.Update(10*time.Second, files)
+	err = ruleManager.Update(10*time.Second, files, nil)
 	testutil.Ok(t, err)
 	for _, g := range ruleManager.groups {
 		for _, actual := range g.seriesInPreviousEval {
@@ -699,7 +699,7 @@ func TestNotify(t *testing.T) {
 
 	expr, err := promql.ParseExpr("a > 1")
 	testutil.Ok(t, err)
-	rule := NewAlertingRule("aTooHigh", expr, 0, labels.Labels{}, labels.Labels{}, true, log.NewNopLogger())
+	rule := NewAlertingRule("aTooHigh", expr, 0, labels.Labels{}, labels.Labels{}, nil, true, log.NewNopLogger())
 	group := NewGroup("alert", "", time.Second, []Rule{rule}, true, opts)
 
 	app, _ := storage.Appender()

template/template.go

@@ -31,7 +31,6 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
-	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/util/strutil"
 )
@@ -134,19 +133,6 @@ func NewTemplateExpander(
 			"label": func(label string, s *sample) string {
 				return s.Labels[label]
 			},
-			"externalLabel": func(label string) string {
-				if config.CurrentConfig == nil {
-					return ""
-				}
-				config.CurrentConfigMutex.RLock()
-				defer config.CurrentConfigMutex.RUnlock()
-				for eln, elv := range (*config.CurrentConfig).GlobalConfig.ExternalLabels {
-					if label == string(eln) {
-						return string(elv)
-					}
-				}
-				return ""
-			},
 			"value": func(s *sample) float64 {
 				return s.Value
 			},

web/api/v1/api_test.go

@@ -142,6 +142,7 @@ func (m rulesRetrieverMock) AlertingRules() []*rules.AlertingRule {
 		time.Second,
 		labels.Labels{},
 		labels.Labels{},
+		labels.Labels{},
 		true,
 		log.NewNopLogger(),
 	)
@@ -151,6 +152,7 @@ func (m rulesRetrieverMock) AlertingRules() []*rules.AlertingRule {
 		time.Second,
 		labels.Labels{},
 		labels.Labels{},
+		labels.Labels{},
 		true,
 		log.NewNopLogger(),
 	)

web/web.go

@@ -542,22 +542,22 @@ func (h *Handler) consoles(w http.ResponseWriter, r *http.Request) {
 		params[k] = v[0]
 	}
 
-	// External labels
+	externalLabels := map[string]string{}
-	el := make(map[string]string)
+	h.mtx.RLock()
-	if config.CurrentConfig != nil {
+	els := h.config.GlobalConfig.ExternalLabels
-		config.CurrentConfigMutex.RLock()
+	h.mtx.RUnlock()
-		for eln, elv := range (*config.CurrentConfig).GlobalConfig.ExternalLabels {
+	for _, el := range els {
-			el[string(eln)] = string(elv)
+		externalLabels[el.Name] = el.Value
-		}
-		config.CurrentConfigMutex.RUnlock()
 	}
 
 	// Inject some convenience variables that are easier to remember for users
 	// who are not used to Go's templating system.
-	defs := "{{$rawParams := .RawParams }}"
+	defs := []string{
-	defs += "{{$params := .Params}}"
+		"{{$rawParams := .RawParams }}",
-	defs += "{{$path := .Path}}"
+		"{{$params := .Params}}",
-	defs += "{{$externalLabels := .ExternalLabels}}"
+		"{{$path := .Path}}",
+		"{{$externalLabels := .ExternalLabels}}",
+	}
 
 	data := struct {
 		RawParams      url.Values
@@ -568,12 +568,12 @@ func (h *Handler) consoles(w http.ResponseWriter, r *http.Request) {
 		RawParams:      rawParams,
 		Params:         params,
 		Path:           strings.TrimLeft(name, "/"),
-		ExternalLabels: el,
+		ExternalLabels: externalLabels,
 	}
 
 	tmpl := template.NewTemplateExpander(
 		h.context,
-		defs+string(text),
+		strings.Join(append(defs, string(text)), ""),
 		"__console_"+name,
 		data,
 		h.now(),