diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index 27484e899..108888d65 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -443,7 +443,11 @@ func main() { } files = append(files, fs...) } - return ruleManager.Update(time.Duration(cfg.GlobalConfig.EvaluationInterval), files) + return ruleManager.Update( + time.Duration(cfg.GlobalConfig.EvaluationInterval), + files, + cfg.GlobalConfig.ExternalLabels, + ) }, } @@ -747,13 +751,8 @@ func reloadConfig(filename string, logger log.Logger, rls ...func(*config.Config if failed { return errors.Errorf("one or more errors occurred while applying the new configuration (--config.file=%q)", filename) } + promql.SetDefaultEvaluationInterval(time.Duration(conf.GlobalConfig.EvaluationInterval)) - - // Register conf as current config. - config.CurrentConfigMutex.Lock() - config.CurrentConfig = conf - config.CurrentConfigMutex.Unlock() - level.Info(logger).Log("msg", "Completed loading of configuration file", "filename", filename) return nil } diff --git a/cmd/promtool/unittest.go b/cmd/promtool/unittest.go index 7a67aed64..c73895b93 100644 --- a/cmd/promtool/unittest.go +++ b/cmd/promtool/unittest.go @@ -163,7 +163,8 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou Logger: &dummyLogger{}, } m := rules.NewManager(opts) - groupsMap, ers := m.LoadGroups(tg.Interval, ruleFiles...) + // TODO(beorn7): Provide a way to pass in external labels. + groupsMap, ers := m.LoadGroups(tg.Interval, nil, ruleFiles...) if ers != nil { return ers } diff --git a/config/config.go b/config/config.go index 7acd54ad4..f4d8d470d 100644 --- a/config/config.go +++ b/config/config.go @@ -20,7 +20,6 @@ import ( "path/filepath" "regexp" "strings" - "sync" "time" "github.com/pkg/errors" @@ -69,10 +68,6 @@ func LoadFile(filename string) (*Config, error) { // The defaults applied before parsing the respective config sections. var ( - // CurrentConfig is a pointer to the current configuration. - CurrentConfig *Config - CurrentConfigMutex sync.RWMutex - // DefaultConfig is the default top-level configuration. DefaultConfig = Config{ GlobalConfig: DefaultGlobalConfig, diff --git a/docs/configuration/alerting_rules.md b/docs/configuration/alerting_rules.md index e8110a1a8..1bc435beb 100644 --- a/docs/configuration/alerting_rules.md +++ b/docs/configuration/alerting_rules.md @@ -42,9 +42,11 @@ The `annotations` clause specifies a set of informational labels that can be use #### Templating -Label and annotation values can be templated using [console templates](https://prometheus.io/docs/visualization/consoles). -The `$labels` & `$externalLabels` variables hold the label key/value pairs of an alert instance -and `$value` holds the evaluated value of an alert instance. +Label and annotation values can be templated using [console +templates](https://prometheus.io/docs/visualization/consoles). The `$labels` +variable holds the label key/value pairs of an alert instance. The configured +external labels can be accessed via the `$externalLabels` variable. The +`$value` variable holds the evaluated value of an alert instance. # To insert a firing element's label values: {{ $labels. }} diff --git a/docs/configuration/template_reference.md b/docs/configuration/template_reference.md index 2e10e4247..8791d68b3 100644 --- a/docs/configuration/template_reference.md +++ b/docs/configuration/template_reference.md @@ -89,8 +89,10 @@ parameterize templates, and have a few other differences. ### Alert field templates -`.Value` and `.Labels` contain the alert value and labels. They are also exposed -as the `$value` and `$labels` variables for convenience. +`.Value`, `.Labels`, and `ExternalLabels` contain the alert value, the alert +labels, and the globally configured external labels, respectively. They are +also exposed as the `$value`, `$labels`, and `$externalLabels` variables for +convenience. ### Console templates @@ -104,7 +106,9 @@ auto-escaping. To bypass the auto-escaping use the `safe*` functions., URL parameters are available as a map in `.Params`. To access multiple URL parameters by the same name, `.RawParams` is a map of the list values for each parameter. The URL path is available in `.Path`, excluding the `/consoles/` -prefix. +prefix. The globally configured external labels are available as +`.ExternalLabels`. There are also convenience variables for all four: +`$rawParams`, `$params`, `$path`, and `$externalLabels`. Consoles also have access to all the templates defined with `{{define "templateName"}}...{{end}}` found in `*.lib` files in the directory pointed to diff --git a/pkg/rulefmt/rulefmt.go b/pkg/rulefmt/rulefmt.go index 19a2d1836..b89470d38 100644 --- a/pkg/rulefmt/rulefmt.go +++ b/pkg/rulefmt/rulefmt.go @@ -16,6 +16,7 @@ package rulefmt import ( "context" "io/ioutil" + "strings" "time" "github.com/pkg/errors" @@ -155,12 +156,16 @@ func testTemplateParsing(rl *Rule) (errs []error) { } // Trying to parse templates. - tmplData := template.AlertTemplateData(make(map[string]string), make(map[string]string), 0) - defs := "{{$labels := .Labels}}{{$value := .Value}}" + tmplData := template.AlertTemplateData(map[string]string{}, map[string]string{}, 0) + defs := []string{ + "{{$labels := .Labels}}", + "{{$externalLabels := .ExternalLabels}}", + "{{$value := .Value}}", + } parseTest := func(text string) error { tmpl := template.NewTemplateExpander( context.TODO(), - defs+text, + strings.Join(append(defs, text), ""), "__alert_"+rl.Alert, tmplData, model.Time(timestamp.FromTime(time.Now())), diff --git a/rules/alerting.go b/rules/alerting.go index c215fc014..b66d6db88 100644 --- a/rules/alerting.go +++ b/rules/alerting.go @@ -17,6 +17,7 @@ import ( "context" "fmt" "net/url" + "strings" "sync" "time" @@ -28,7 +29,6 @@ import ( "github.com/go-kit/kit/log/level" "github.com/pkg/errors" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/rulefmt" @@ -80,9 +80,8 @@ func (s AlertState) String() string { type Alert struct { State AlertState - Labels labels.Labels - ExternalLabels labels.Labels - Annotations labels.Labels + Labels labels.Labels + Annotations labels.Labels // The value at the last evaluation of the alerting expression. Value float64 @@ -121,6 +120,8 @@ type AlertingRule struct { labels labels.Labels // Non-identifying key/value pairs. annotations labels.Labels + // External labels from the global config. + externalLabels map[string]string // true if old state has been restored. We start persisting samples for ALERT_FOR_STATE // only after the restoration. restored bool @@ -142,17 +143,27 @@ type AlertingRule struct { } // NewAlertingRule constructs a new AlertingRule. -func NewAlertingRule(name string, vec promql.Expr, hold time.Duration, lbls, anns labels.Labels, restored bool, logger log.Logger) *AlertingRule { +func NewAlertingRule( + name string, vec promql.Expr, hold time.Duration, + labels, annotations, externalLabels labels.Labels, + restored bool, logger log.Logger, +) *AlertingRule { + el := make(map[string]string, len(externalLabels)) + for _, lbl := range externalLabels { + el[lbl.Name] = lbl.Value + } + return &AlertingRule{ - name: name, - vector: vec, - holdDuration: hold, - labels: lbls, - annotations: anns, - health: HealthUnknown, - active: map[uint64]*Alert{}, - logger: logger, - restored: restored, + name: name, + vector: vec, + holdDuration: hold, + labels: labels, + annotations: annotations, + externalLabels: el, + health: HealthUnknown, + active: map[uint64]*Alert{}, + logger: logger, + restored: restored, } } @@ -302,32 +313,24 @@ func (r *AlertingRule) Eval(ctx context.Context, ts time.Time, query QueryFunc, var vec promql.Vector for _, smpl := range res { // Provide the alert information to the template. - l := make(map[string]string) + l := make(map[string]string, len(smpl.Metric)) for _, lbl := range smpl.Metric { l[lbl.Name] = lbl.Value } - // Add external labels. - el := make(map[string]string) - if config.CurrentConfig != nil { - config.CurrentConfigMutex.RLock() - for eln, elv := range (*config.CurrentConfig).GlobalConfig.ExternalLabels { - el[string(eln)] = string(elv) - } - config.CurrentConfigMutex.RUnlock() - } - - tmplData := template.AlertTemplateData(l, el, smpl.V) + tmplData := template.AlertTemplateData(l, r.externalLabels, smpl.V) // Inject some convenience variables that are easier to remember for users // who are not used to Go's templating system. - defs := "{{$labels := .Labels}}" - defs += "{{$externalLabels := .ExternalLabels}}" - defs += "{{$value := .Value}}" + defs := []string{ + "{{$labels := .Labels}}", + "{{$externalLabels := .ExternalLabels}}", + "{{$value := .Value}}", + } expand := func(text string) string { tmpl := template.NewTemplateExpander( ctx, - defs+text, + strings.Join(append(defs, text), ""), "__alert_"+r.Name(), tmplData, model.Time(timestamp.FromTime(ts)), @@ -463,7 +466,7 @@ func (r *AlertingRule) ForEachActiveAlert(f func(*Alert)) { } func (r *AlertingRule) sendAlerts(ctx context.Context, ts time.Time, resendDelay time.Duration, interval time.Duration, notifyFunc NotifyFunc) { - alerts := make([]*Alert, 0) + alerts := []*Alert{} r.ForEachActiveAlert(func(alert *Alert) { if alert.needsSending(ts, resendDelay) { alert.LastSentAt = ts diff --git a/rules/alerting_test.go b/rules/alerting_test.go index 884da30bf..a862e0923 100644 --- a/rules/alerting_test.go +++ b/rules/alerting_test.go @@ -26,7 +26,7 @@ import ( func TestAlertingRuleHTMLSnippet(t *testing.T) { expr, err := promql.ParseExpr(`foo{html="BOLD"}`) testutil.Ok(t, err) - rule := NewAlertingRule("testrule", expr, 0, labels.FromStrings("html", "BOLD"), labels.FromStrings("html", "BOLD"), false, nil) + rule := NewAlertingRule("testrule", expr, 0, labels.FromStrings("html", "BOLD"), labels.FromStrings("html", "BOLD"), nil, false, nil) const want = `alert: testrule expr: foo{html="<b>BOLD<b>"} @@ -62,7 +62,7 @@ func TestAlertingRuleLabelsUpdate(t *testing.T) { // If an alert is going back and forth between two label values it will never fire. // Instead, you should write two alerts with constant labels. labels.FromStrings("severity", "{{ if lt $value 80.0 }}critical{{ else }}warning{{ end }}"), - nil, true, nil, + nil, nil, true, nil, ) results := []promql.Vector{ diff --git a/rules/manager.go b/rules/manager.go index 96766b2ab..77130489f 100644 --- a/rules/manager.go +++ b/rules/manager.go @@ -751,11 +751,11 @@ func (m *Manager) Stop() { // Update the rule manager's state as the config requires. If // loading the new rules failed the old rule set is restored. -func (m *Manager) Update(interval time.Duration, files []string) error { +func (m *Manager) Update(interval time.Duration, files []string, externalLabels labels.Labels) error { m.mtx.Lock() defer m.mtx.Unlock() - groups, errs := m.LoadGroups(interval, files...) + groups, errs := m.LoadGroups(interval, externalLabels, files...) if errs != nil { for _, e := range errs { level.Error(m.logger).Log("msg", "loading groups failed", "err", e) @@ -803,7 +803,9 @@ func (m *Manager) Update(interval time.Duration, files []string) error { } // LoadGroups reads groups from a list of files. -func (m *Manager) LoadGroups(interval time.Duration, filenames ...string) (map[string]*Group, []error) { +func (m *Manager) LoadGroups( + interval time.Duration, externalLabels labels.Labels, filenames ...string, +) (map[string]*Group, []error) { groups := make(map[string]*Group) shouldRestore := !m.restored @@ -834,6 +836,7 @@ func (m *Manager) LoadGroups(interval time.Duration, filenames ...string) (map[s time.Duration(r.For), labels.FromMap(r.Labels), labels.FromMap(r.Annotations), + externalLabels, m.restored, log.With(m.logger, "alert", r.Alert), )) diff --git a/rules/manager_test.go b/rules/manager_test.go index 6dbff9c38..6298c0866 100644 --- a/rules/manager_test.go +++ b/rules/manager_test.go @@ -51,7 +51,7 @@ func TestAlertingRule(t *testing.T) { expr, time.Minute, labels.FromStrings("severity", "{{\"c\"}}ritical"), - nil, true, nil, + nil, nil, true, nil, ) result := promql.Vector{ { @@ -192,7 +192,7 @@ func TestForStateAddSamples(t *testing.T) { expr, time.Minute, labels.FromStrings("severity", "{{\"c\"}}ritical"), - nil, true, nil, + nil, nil, true, nil, ) result := promql.Vector{ { @@ -366,7 +366,7 @@ func TestForStateRestore(t *testing.T) { expr, alertForDuration, labels.FromStrings("severity", "critical"), - nil, true, nil, + nil, nil, true, nil, ) group := NewGroup("default", "", time.Second, []Rule{rule}, true, opts) @@ -426,7 +426,7 @@ func TestForStateRestore(t *testing.T) { expr, alertForDuration, labels.FromStrings("severity", "critical"), - nil, false, nil, + nil, nil, false, nil, ) newGroup := NewGroup("default", "", time.Second, []Rule{newRule}, true, opts) @@ -581,12 +581,12 @@ func readSeriesSet(ss storage.SeriesSet) (map[string][]promql.Point, error) { func TestCopyState(t *testing.T) { oldGroup := &Group{ rules: []Rule{ - NewAlertingRule("alert", nil, 0, nil, nil, true, nil), + NewAlertingRule("alert", nil, 0, nil, nil, nil, true, nil), NewRecordingRule("rule1", nil, nil), NewRecordingRule("rule2", nil, nil), NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v1"}}), NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v2"}}), - NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, true, nil), + NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, nil, true, nil), }, seriesInPreviousEval: []map[string]labels.Labels{ {"a": nil}, @@ -604,10 +604,10 @@ func TestCopyState(t *testing.T) { NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v0"}}), NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v1"}}), NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v2"}}), - NewAlertingRule("alert", nil, 0, nil, nil, true, nil), + NewAlertingRule("alert", nil, 0, nil, nil, nil, true, nil), NewRecordingRule("rule1", nil, nil), - NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v0"}}, nil, true, nil), - NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, true, nil), + NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v0"}}, nil, nil, true, nil), + NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, nil, true, nil), NewRecordingRule("rule4", nil, nil), }, seriesInPreviousEval: make([]map[string]labels.Labels, 8), @@ -654,7 +654,7 @@ func TestUpdate(t *testing.T) { ruleManager.Run() defer ruleManager.Stop() - err := ruleManager.Update(10*time.Second, files) + err := ruleManager.Update(10*time.Second, files, nil) testutil.Ok(t, err) testutil.Assert(t, len(ruleManager.groups) > 0, "expected non-empty rule groups") for _, g := range ruleManager.groups { @@ -663,7 +663,7 @@ func TestUpdate(t *testing.T) { } } - err = ruleManager.Update(10*time.Second, files) + err = ruleManager.Update(10*time.Second, files, nil) testutil.Ok(t, err) for _, g := range ruleManager.groups { for _, actual := range g.seriesInPreviousEval { @@ -699,7 +699,7 @@ func TestNotify(t *testing.T) { expr, err := promql.ParseExpr("a > 1") testutil.Ok(t, err) - rule := NewAlertingRule("aTooHigh", expr, 0, labels.Labels{}, labels.Labels{}, true, log.NewNopLogger()) + rule := NewAlertingRule("aTooHigh", expr, 0, labels.Labels{}, labels.Labels{}, nil, true, log.NewNopLogger()) group := NewGroup("alert", "", time.Second, []Rule{rule}, true, opts) app, _ := storage.Appender() diff --git a/template/template.go b/template/template.go index 88b0e841d..de6ee5453 100644 --- a/template/template.go +++ b/template/template.go @@ -31,7 +31,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/util/strutil" ) @@ -134,19 +133,6 @@ func NewTemplateExpander( "label": func(label string, s *sample) string { return s.Labels[label] }, - "externalLabel": func(label string) string { - if config.CurrentConfig == nil { - return "" - } - config.CurrentConfigMutex.RLock() - defer config.CurrentConfigMutex.RUnlock() - for eln, elv := range (*config.CurrentConfig).GlobalConfig.ExternalLabels { - if label == string(eln) { - return string(elv) - } - } - return "" - }, "value": func(s *sample) float64 { return s.Value }, diff --git a/web/api/v1/api_test.go b/web/api/v1/api_test.go index 5082c1172..b2b8939de 100644 --- a/web/api/v1/api_test.go +++ b/web/api/v1/api_test.go @@ -142,6 +142,7 @@ func (m rulesRetrieverMock) AlertingRules() []*rules.AlertingRule { time.Second, labels.Labels{}, labels.Labels{}, + labels.Labels{}, true, log.NewNopLogger(), ) @@ -151,6 +152,7 @@ func (m rulesRetrieverMock) AlertingRules() []*rules.AlertingRule { time.Second, labels.Labels{}, labels.Labels{}, + labels.Labels{}, true, log.NewNopLogger(), ) diff --git a/web/web.go b/web/web.go index 4762bb6dd..77a059725 100644 --- a/web/web.go +++ b/web/web.go @@ -542,22 +542,22 @@ func (h *Handler) consoles(w http.ResponseWriter, r *http.Request) { params[k] = v[0] } - // External labels - el := make(map[string]string) - if config.CurrentConfig != nil { - config.CurrentConfigMutex.RLock() - for eln, elv := range (*config.CurrentConfig).GlobalConfig.ExternalLabels { - el[string(eln)] = string(elv) - } - config.CurrentConfigMutex.RUnlock() + externalLabels := map[string]string{} + h.mtx.RLock() + els := h.config.GlobalConfig.ExternalLabels + h.mtx.RUnlock() + for _, el := range els { + externalLabels[el.Name] = el.Value } // Inject some convenience variables that are easier to remember for users // who are not used to Go's templating system. - defs := "{{$rawParams := .RawParams }}" - defs += "{{$params := .Params}}" - defs += "{{$path := .Path}}" - defs += "{{$externalLabels := .ExternalLabels}}" + defs := []string{ + "{{$rawParams := .RawParams }}", + "{{$params := .Params}}", + "{{$path := .Path}}", + "{{$externalLabels := .ExternalLabels}}", + } data := struct { RawParams url.Values @@ -568,12 +568,12 @@ func (h *Handler) consoles(w http.ResponseWriter, r *http.Request) { RawParams: rawParams, Params: params, Path: strings.TrimLeft(name, "/"), - ExternalLabels: el, + ExternalLabels: externalLabels, } tmpl := template.NewTemplateExpander( h.context, - defs+string(text), + strings.Join(append(defs, string(text)), ""), "__console_"+name, data, h.now(),