Rework #5009 after comments

Signed-off-by: Bjoern Rabenstein <bjoern@rabenste.in>
This commit is contained in:
Bjoern Rabenstein 2019-04-15 18:52:58 +02:00
parent a92ef68dd8
commit 38d518c0fe
13 changed files with 98 additions and 98 deletions

View file

@ -443,7 +443,11 @@ func main() {
}
files = append(files, fs...)
}
return ruleManager.Update(time.Duration(cfg.GlobalConfig.EvaluationInterval), files)
return ruleManager.Update(
time.Duration(cfg.GlobalConfig.EvaluationInterval),
files,
cfg.GlobalConfig.ExternalLabels,
)
},
}
@ -747,13 +751,8 @@ func reloadConfig(filename string, logger log.Logger, rls ...func(*config.Config
if failed {
return errors.Errorf("one or more errors occurred while applying the new configuration (--config.file=%q)", filename)
}
promql.SetDefaultEvaluationInterval(time.Duration(conf.GlobalConfig.EvaluationInterval))
// Register conf as current config.
config.CurrentConfigMutex.Lock()
config.CurrentConfig = conf
config.CurrentConfigMutex.Unlock()
level.Info(logger).Log("msg", "Completed loading of configuration file", "filename", filename)
return nil
}

View file

@ -163,7 +163,8 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou
Logger: &dummyLogger{},
}
m := rules.NewManager(opts)
groupsMap, ers := m.LoadGroups(tg.Interval, ruleFiles...)
// TODO(beorn7): Provide a way to pass in external labels.
groupsMap, ers := m.LoadGroups(tg.Interval, nil, ruleFiles...)
if ers != nil {
return ers
}

View file

@ -20,7 +20,6 @@ import (
"path/filepath"
"regexp"
"strings"
"sync"
"time"
"github.com/pkg/errors"
@ -69,10 +68,6 @@ func LoadFile(filename string) (*Config, error) {
// The defaults applied before parsing the respective config sections.
var (
// CurrentConfig is a pointer to the current configuration.
CurrentConfig *Config
CurrentConfigMutex sync.RWMutex
// DefaultConfig is the default top-level configuration.
DefaultConfig = Config{
GlobalConfig: DefaultGlobalConfig,

View file

@ -42,9 +42,11 @@ The `annotations` clause specifies a set of informational labels that can be use
#### Templating
Label and annotation values can be templated using [console templates](https://prometheus.io/docs/visualization/consoles).
The `$labels` & `$externalLabels` variables hold the label key/value pairs of an alert instance
and `$value` holds the evaluated value of an alert instance.
Label and annotation values can be templated using [console
templates](https://prometheus.io/docs/visualization/consoles). The `$labels`
variable holds the label key/value pairs of an alert instance. The configured
external labels can be accessed via the `$externalLabels` variable. The
`$value` variable holds the evaluated value of an alert instance.
# To insert a firing element's label values:
{{ $labels.<labelname> }}

View file

@ -89,8 +89,10 @@ parameterize templates, and have a few other differences.
### Alert field templates
`.Value` and `.Labels` contain the alert value and labels. They are also exposed
as the `$value` and `$labels` variables for convenience.
`.Value`, `.Labels`, and `ExternalLabels` contain the alert value, the alert
labels, and the globally configured external labels, respectively. They are
also exposed as the `$value`, `$labels`, and `$externalLabels` variables for
convenience.
### Console templates
@ -104,7 +106,9 @@ auto-escaping. To bypass the auto-escaping use the `safe*` functions.,
URL parameters are available as a map in `.Params`. To access multiple URL
parameters by the same name, `.RawParams` is a map of the list values for each
parameter. The URL path is available in `.Path`, excluding the `/consoles/`
prefix.
prefix. The globally configured external labels are available as
`.ExternalLabels`. There are also convenience variables for all four:
`$rawParams`, `$params`, `$path`, and `$externalLabels`.
Consoles also have access to all the templates defined with `{{define
"templateName"}}...{{end}}` found in `*.lib` files in the directory pointed to

View file

@ -16,6 +16,7 @@ package rulefmt
import (
"context"
"io/ioutil"
"strings"
"time"
"github.com/pkg/errors"
@ -155,12 +156,16 @@ func testTemplateParsing(rl *Rule) (errs []error) {
}
// Trying to parse templates.
tmplData := template.AlertTemplateData(make(map[string]string), make(map[string]string), 0)
defs := "{{$labels := .Labels}}{{$value := .Value}}"
tmplData := template.AlertTemplateData(map[string]string{}, map[string]string{}, 0)
defs := []string{
"{{$labels := .Labels}}",
"{{$externalLabels := .ExternalLabels}}",
"{{$value := .Value}}",
}
parseTest := func(text string) error {
tmpl := template.NewTemplateExpander(
context.TODO(),
defs+text,
strings.Join(append(defs, text), ""),
"__alert_"+rl.Alert,
tmplData,
model.Time(timestamp.FromTime(time.Now())),

View file

@ -17,6 +17,7 @@ import (
"context"
"fmt"
"net/url"
"strings"
"sync"
"time"
@ -28,7 +29,6 @@ import (
"github.com/go-kit/kit/log/level"
"github.com/pkg/errors"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/config"
"github.com/prometheus/prometheus/pkg/labels"
"github.com/prometheus/prometheus/pkg/rulefmt"
@ -80,9 +80,8 @@ func (s AlertState) String() string {
type Alert struct {
State AlertState
Labels labels.Labels
ExternalLabels labels.Labels
Annotations labels.Labels
Labels labels.Labels
Annotations labels.Labels
// The value at the last evaluation of the alerting expression.
Value float64
@ -121,6 +120,8 @@ type AlertingRule struct {
labels labels.Labels
// Non-identifying key/value pairs.
annotations labels.Labels
// External labels from the global config.
externalLabels map[string]string
// true if old state has been restored. We start persisting samples for ALERT_FOR_STATE
// only after the restoration.
restored bool
@ -142,17 +143,27 @@ type AlertingRule struct {
}
// NewAlertingRule constructs a new AlertingRule.
func NewAlertingRule(name string, vec promql.Expr, hold time.Duration, lbls, anns labels.Labels, restored bool, logger log.Logger) *AlertingRule {
func NewAlertingRule(
name string, vec promql.Expr, hold time.Duration,
labels, annotations, externalLabels labels.Labels,
restored bool, logger log.Logger,
) *AlertingRule {
el := make(map[string]string, len(externalLabels))
for _, lbl := range externalLabels {
el[lbl.Name] = lbl.Value
}
return &AlertingRule{
name: name,
vector: vec,
holdDuration: hold,
labels: lbls,
annotations: anns,
health: HealthUnknown,
active: map[uint64]*Alert{},
logger: logger,
restored: restored,
name: name,
vector: vec,
holdDuration: hold,
labels: labels,
annotations: annotations,
externalLabels: el,
health: HealthUnknown,
active: map[uint64]*Alert{},
logger: logger,
restored: restored,
}
}
@ -302,32 +313,24 @@ func (r *AlertingRule) Eval(ctx context.Context, ts time.Time, query QueryFunc,
var vec promql.Vector
for _, smpl := range res {
// Provide the alert information to the template.
l := make(map[string]string)
l := make(map[string]string, len(smpl.Metric))
for _, lbl := range smpl.Metric {
l[lbl.Name] = lbl.Value
}
// Add external labels.
el := make(map[string]string)
if config.CurrentConfig != nil {
config.CurrentConfigMutex.RLock()
for eln, elv := range (*config.CurrentConfig).GlobalConfig.ExternalLabels {
el[string(eln)] = string(elv)
}
config.CurrentConfigMutex.RUnlock()
}
tmplData := template.AlertTemplateData(l, el, smpl.V)
tmplData := template.AlertTemplateData(l, r.externalLabels, smpl.V)
// Inject some convenience variables that are easier to remember for users
// who are not used to Go's templating system.
defs := "{{$labels := .Labels}}"
defs += "{{$externalLabels := .ExternalLabels}}"
defs += "{{$value := .Value}}"
defs := []string{
"{{$labels := .Labels}}",
"{{$externalLabels := .ExternalLabels}}",
"{{$value := .Value}}",
}
expand := func(text string) string {
tmpl := template.NewTemplateExpander(
ctx,
defs+text,
strings.Join(append(defs, text), ""),
"__alert_"+r.Name(),
tmplData,
model.Time(timestamp.FromTime(ts)),
@ -463,7 +466,7 @@ func (r *AlertingRule) ForEachActiveAlert(f func(*Alert)) {
}
func (r *AlertingRule) sendAlerts(ctx context.Context, ts time.Time, resendDelay time.Duration, interval time.Duration, notifyFunc NotifyFunc) {
alerts := make([]*Alert, 0)
alerts := []*Alert{}
r.ForEachActiveAlert(func(alert *Alert) {
if alert.needsSending(ts, resendDelay) {
alert.LastSentAt = ts

View file

@ -26,7 +26,7 @@ import (
func TestAlertingRuleHTMLSnippet(t *testing.T) {
expr, err := promql.ParseExpr(`foo{html="<b>BOLD<b>"}`)
testutil.Ok(t, err)
rule := NewAlertingRule("testrule", expr, 0, labels.FromStrings("html", "<b>BOLD</b>"), labels.FromStrings("html", "<b>BOLD</b>"), false, nil)
rule := NewAlertingRule("testrule", expr, 0, labels.FromStrings("html", "<b>BOLD</b>"), labels.FromStrings("html", "<b>BOLD</b>"), nil, false, nil)
const want = `alert: <a href="/test/prefix/graph?g0.expr=ALERTS%7Balertname%3D%22testrule%22%7D&g0.tab=1">testrule</a>
expr: <a href="/test/prefix/graph?g0.expr=foo%7Bhtml%3D%22%3Cb%3EBOLD%3Cb%3E%22%7D&g0.tab=1">foo{html=&#34;&lt;b&gt;BOLD&lt;b&gt;&#34;}</a>
@ -62,7 +62,7 @@ func TestAlertingRuleLabelsUpdate(t *testing.T) {
// If an alert is going back and forth between two label values it will never fire.
// Instead, you should write two alerts with constant labels.
labels.FromStrings("severity", "{{ if lt $value 80.0 }}critical{{ else }}warning{{ end }}"),
nil, true, nil,
nil, nil, true, nil,
)
results := []promql.Vector{

View file

@ -751,11 +751,11 @@ func (m *Manager) Stop() {
// Update the rule manager's state as the config requires. If
// loading the new rules failed the old rule set is restored.
func (m *Manager) Update(interval time.Duration, files []string) error {
func (m *Manager) Update(interval time.Duration, files []string, externalLabels labels.Labels) error {
m.mtx.Lock()
defer m.mtx.Unlock()
groups, errs := m.LoadGroups(interval, files...)
groups, errs := m.LoadGroups(interval, externalLabels, files...)
if errs != nil {
for _, e := range errs {
level.Error(m.logger).Log("msg", "loading groups failed", "err", e)
@ -803,7 +803,9 @@ func (m *Manager) Update(interval time.Duration, files []string) error {
}
// LoadGroups reads groups from a list of files.
func (m *Manager) LoadGroups(interval time.Duration, filenames ...string) (map[string]*Group, []error) {
func (m *Manager) LoadGroups(
interval time.Duration, externalLabels labels.Labels, filenames ...string,
) (map[string]*Group, []error) {
groups := make(map[string]*Group)
shouldRestore := !m.restored
@ -834,6 +836,7 @@ func (m *Manager) LoadGroups(interval time.Duration, filenames ...string) (map[s
time.Duration(r.For),
labels.FromMap(r.Labels),
labels.FromMap(r.Annotations),
externalLabels,
m.restored,
log.With(m.logger, "alert", r.Alert),
))

View file

@ -51,7 +51,7 @@ func TestAlertingRule(t *testing.T) {
expr,
time.Minute,
labels.FromStrings("severity", "{{\"c\"}}ritical"),
nil, true, nil,
nil, nil, true, nil,
)
result := promql.Vector{
{
@ -192,7 +192,7 @@ func TestForStateAddSamples(t *testing.T) {
expr,
time.Minute,
labels.FromStrings("severity", "{{\"c\"}}ritical"),
nil, true, nil,
nil, nil, true, nil,
)
result := promql.Vector{
{
@ -366,7 +366,7 @@ func TestForStateRestore(t *testing.T) {
expr,
alertForDuration,
labels.FromStrings("severity", "critical"),
nil, true, nil,
nil, nil, true, nil,
)
group := NewGroup("default", "", time.Second, []Rule{rule}, true, opts)
@ -426,7 +426,7 @@ func TestForStateRestore(t *testing.T) {
expr,
alertForDuration,
labels.FromStrings("severity", "critical"),
nil, false, nil,
nil, nil, false, nil,
)
newGroup := NewGroup("default", "", time.Second, []Rule{newRule}, true, opts)
@ -581,12 +581,12 @@ func readSeriesSet(ss storage.SeriesSet) (map[string][]promql.Point, error) {
func TestCopyState(t *testing.T) {
oldGroup := &Group{
rules: []Rule{
NewAlertingRule("alert", nil, 0, nil, nil, true, nil),
NewAlertingRule("alert", nil, 0, nil, nil, nil, true, nil),
NewRecordingRule("rule1", nil, nil),
NewRecordingRule("rule2", nil, nil),
NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v1"}}),
NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v2"}}),
NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, true, nil),
NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, nil, true, nil),
},
seriesInPreviousEval: []map[string]labels.Labels{
{"a": nil},
@ -604,10 +604,10 @@ func TestCopyState(t *testing.T) {
NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v0"}}),
NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v1"}}),
NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v2"}}),
NewAlertingRule("alert", nil, 0, nil, nil, true, nil),
NewAlertingRule("alert", nil, 0, nil, nil, nil, true, nil),
NewRecordingRule("rule1", nil, nil),
NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v0"}}, nil, true, nil),
NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, true, nil),
NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v0"}}, nil, nil, true, nil),
NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, nil, true, nil),
NewRecordingRule("rule4", nil, nil),
},
seriesInPreviousEval: make([]map[string]labels.Labels, 8),
@ -654,7 +654,7 @@ func TestUpdate(t *testing.T) {
ruleManager.Run()
defer ruleManager.Stop()
err := ruleManager.Update(10*time.Second, files)
err := ruleManager.Update(10*time.Second, files, nil)
testutil.Ok(t, err)
testutil.Assert(t, len(ruleManager.groups) > 0, "expected non-empty rule groups")
for _, g := range ruleManager.groups {
@ -663,7 +663,7 @@ func TestUpdate(t *testing.T) {
}
}
err = ruleManager.Update(10*time.Second, files)
err = ruleManager.Update(10*time.Second, files, nil)
testutil.Ok(t, err)
for _, g := range ruleManager.groups {
for _, actual := range g.seriesInPreviousEval {
@ -699,7 +699,7 @@ func TestNotify(t *testing.T) {
expr, err := promql.ParseExpr("a > 1")
testutil.Ok(t, err)
rule := NewAlertingRule("aTooHigh", expr, 0, labels.Labels{}, labels.Labels{}, true, log.NewNopLogger())
rule := NewAlertingRule("aTooHigh", expr, 0, labels.Labels{}, labels.Labels{}, nil, true, log.NewNopLogger())
group := NewGroup("alert", "", time.Second, []Rule{rule}, true, opts)
app, _ := storage.Appender()

View file

@ -31,7 +31,6 @@ import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/config"
"github.com/prometheus/prometheus/promql"
"github.com/prometheus/prometheus/util/strutil"
)
@ -134,19 +133,6 @@ func NewTemplateExpander(
"label": func(label string, s *sample) string {
return s.Labels[label]
},
"externalLabel": func(label string) string {
if config.CurrentConfig == nil {
return ""
}
config.CurrentConfigMutex.RLock()
defer config.CurrentConfigMutex.RUnlock()
for eln, elv := range (*config.CurrentConfig).GlobalConfig.ExternalLabels {
if label == string(eln) {
return string(elv)
}
}
return ""
},
"value": func(s *sample) float64 {
return s.Value
},

View file

@ -142,6 +142,7 @@ func (m rulesRetrieverMock) AlertingRules() []*rules.AlertingRule {
time.Second,
labels.Labels{},
labels.Labels{},
labels.Labels{},
true,
log.NewNopLogger(),
)
@ -151,6 +152,7 @@ func (m rulesRetrieverMock) AlertingRules() []*rules.AlertingRule {
time.Second,
labels.Labels{},
labels.Labels{},
labels.Labels{},
true,
log.NewNopLogger(),
)

View file

@ -542,22 +542,22 @@ func (h *Handler) consoles(w http.ResponseWriter, r *http.Request) {
params[k] = v[0]
}
// External labels
el := make(map[string]string)
if config.CurrentConfig != nil {
config.CurrentConfigMutex.RLock()
for eln, elv := range (*config.CurrentConfig).GlobalConfig.ExternalLabels {
el[string(eln)] = string(elv)
}
config.CurrentConfigMutex.RUnlock()
externalLabels := map[string]string{}
h.mtx.RLock()
els := h.config.GlobalConfig.ExternalLabels
h.mtx.RUnlock()
for _, el := range els {
externalLabels[el.Name] = el.Value
}
// Inject some convenience variables that are easier to remember for users
// who are not used to Go's templating system.
defs := "{{$rawParams := .RawParams }}"
defs += "{{$params := .Params}}"
defs += "{{$path := .Path}}"
defs += "{{$externalLabels := .ExternalLabels}}"
defs := []string{
"{{$rawParams := .RawParams }}",
"{{$params := .Params}}",
"{{$path := .Path}}",
"{{$externalLabels := .ExternalLabels}}",
}
data := struct {
RawParams url.Values
@ -568,12 +568,12 @@ func (h *Handler) consoles(w http.ResponseWriter, r *http.Request) {
RawParams: rawParams,
Params: params,
Path: strings.TrimLeft(name, "/"),
ExternalLabels: el,
ExternalLabels: externalLabels,
}
tmpl := template.NewTemplateExpander(
h.context,
defs+string(text),
strings.Join(append(defs, string(text)), ""),
"__console_"+name,
data,
h.now(),