diff --git a/rules/manager.go b/rules/manager.go index d779979b7..2fea605a9 100644 --- a/rules/manager.go +++ b/rules/manager.go @@ -36,11 +36,7 @@ import ( ) // Constants for instrumentation. -const ( - namespace = "prometheus" - - ruleTypeLabel = "rule_type" -) +const namespace = "prometheus" var ( evalDuration = prometheus.NewSummaryVec( @@ -49,21 +45,23 @@ var ( Name: "rule_evaluation_duration_seconds", Help: "The duration for a rule to execute.", }, - []string{ruleTypeLabel}, + []string{"rule_type"}, ) - evalFailures = prometheus.NewCounter( + evalFailures = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: namespace, Name: "rule_evaluation_failures_total", Help: "The total number of rule evaluation failures.", }, + []string{"rule_type"}, ) - evalTotal = prometheus.NewCounter( + evalTotal = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: namespace, Name: "rule_evaluations_total", Help: "The total number of rule evaluations.", }, + []string{"rule_type"}, ) iterationDuration = prometheus.NewSummary(prometheus.SummaryOpts{ Namespace: namespace, @@ -74,6 +72,11 @@ var ( ) func init() { + evalTotal.WithLabelValues(string(ruleTypeAlert)) + evalTotal.WithLabelValues(string(ruleTypeRecording)) + evalFailures.WithLabelValues(string(ruleTypeAlert)) + evalFailures.WithLabelValues(string(ruleTypeRecording)) + prometheus.MustRegister(iterationDuration) prometheus.MustRegister(evalFailures) prometheus.MustRegister(evalDuration) @@ -200,6 +203,16 @@ func (g *Group) copyState(from *Group) { } } +func typeForRule(r Rule) ruleType { + switch r.(type) { + case *AlertingRule: + return ruleTypeAlert + case *RecordingRule: + return ruleTypeRecording + } + panic(fmt.Errorf("unknown rule type: %T", r)) +} + // eval runs a single evaluation cycle in which all rules are evaluated in parallel. // In the future a single group will be evaluated sequentially to properly handle // rule dependency. @@ -210,13 +223,18 @@ func (g *Group) eval() { ) for _, rule := range g.rules { + rtyp := string(typeForRule(rule)) + wg.Add(1) // BUG(julius): Look at fixing thundering herd. go func(rule Rule) { defer wg.Done() - start := time.Now() - evalTotal.Inc() + defer func(t time.Time) { + evalDuration.WithLabelValues(rtyp).Observe(float64(time.Since(t)) / float64(time.Second)) + }(time.Now()) + + evalTotal.WithLabelValues(rtyp).Inc() vector, err := rule.eval(now, g.opts.QueryEngine) if err != nil { @@ -225,26 +243,13 @@ func (g *Group) eval() { if _, ok := err.(promql.ErrQueryCanceled); !ok { log.Warnf("Error while evaluating rule %q: %s", rule, err) } - evalFailures.Inc() - } - var rtyp ruleType - - switch r := rule.(type) { - case *AlertingRule: - rtyp = ruleTypeRecording - g.sendAlerts(r, now) - - case *RecordingRule: - rtyp = ruleTypeAlert - - default: - panic(fmt.Errorf("unknown rule type: %T", rule)) + evalFailures.WithLabelValues(rtyp).Inc() + return } - evalDuration.WithLabelValues(string(rtyp)).Observe( - float64(time.Since(start)) / float64(time.Second), - ) - + if ar, ok := rule.(*AlertingRule); ok { + g.sendAlerts(ar, now) + } for _, s := range vector { g.opts.SampleAppender.Append(s) } diff --git a/web/ui/bindata.go b/web/ui/bindata.go index 3f9f88cba..3cc9690f5 100644 --- a/web/ui/bindata.go +++ b/web/ui/bindata.go @@ -791,38 +791,38 @@ func AssetNames() []string { // _bindata is a table, holding each asset generator, mapped to its name. var _bindata = map[string]func() (*asset, error){ - "web/ui/templates/_base.html": webUiTemplates_baseHtml, - "web/ui/templates/alerts.html": webUiTemplatesAlertsHtml, - "web/ui/templates/graph.html": webUiTemplatesGraphHtml, - "web/ui/templates/status.html": webUiTemplatesStatusHtml, - "web/ui/static/css/alerts.css": webUiStaticCssAlertsCss, - "web/ui/static/css/graph.css": webUiStaticCssGraphCss, - "web/ui/static/css/prom_console.css": webUiStaticCssProm_consoleCss, - "web/ui/static/css/prometheus.css": webUiStaticCssPrometheusCss, - "web/ui/static/img/ajax-loader.gif": webUiStaticImgAjaxLoaderGif, - "web/ui/static/js/alerts.js": webUiStaticJsAlertsJs, - "web/ui/static/js/graph.js": webUiStaticJsGraphJs, - "web/ui/static/js/graph_template.handlebar": webUiStaticJsGraph_templateHandlebar, - "web/ui/static/js/prom_console.js": webUiStaticJsProm_consoleJs, - "web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap-theme.min.css": webUiStaticVendorBootstrap331CssBootstrapThemeMinCss, - "web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap.min.css": webUiStaticVendorBootstrap331CssBootstrapMinCss, - "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.eot": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot, - "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.svg": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg, - "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.ttf": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf, - "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.woff": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularWoff, - "web/ui/static/vendor/bootstrap-3.3.1/js/bootstrap.min.js": webUiStaticVendorBootstrap331JsBootstrapMinJs, - "web/ui/static/vendor/bootstrap-3.3.1/js/npm.js": webUiStaticVendorBootstrap331JsNpmJs, - "web/ui/static/vendor/bootstrap-datetimepicker/bootstrap-datetimepicker.js": webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs, + "web/ui/templates/_base.html": webUiTemplates_baseHtml, + "web/ui/templates/alerts.html": webUiTemplatesAlertsHtml, + "web/ui/templates/graph.html": webUiTemplatesGraphHtml, + "web/ui/templates/status.html": webUiTemplatesStatusHtml, + "web/ui/static/css/alerts.css": webUiStaticCssAlertsCss, + "web/ui/static/css/graph.css": webUiStaticCssGraphCss, + "web/ui/static/css/prom_console.css": webUiStaticCssProm_consoleCss, + "web/ui/static/css/prometheus.css": webUiStaticCssPrometheusCss, + "web/ui/static/img/ajax-loader.gif": webUiStaticImgAjaxLoaderGif, + "web/ui/static/js/alerts.js": webUiStaticJsAlertsJs, + "web/ui/static/js/graph.js": webUiStaticJsGraphJs, + "web/ui/static/js/graph_template.handlebar": webUiStaticJsGraph_templateHandlebar, + "web/ui/static/js/prom_console.js": webUiStaticJsProm_consoleJs, + "web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap-theme.min.css": webUiStaticVendorBootstrap331CssBootstrapThemeMinCss, + "web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap.min.css": webUiStaticVendorBootstrap331CssBootstrapMinCss, + "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.eot": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot, + "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.svg": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg, + "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.ttf": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf, + "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.woff": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularWoff, + "web/ui/static/vendor/bootstrap-3.3.1/js/bootstrap.min.js": webUiStaticVendorBootstrap331JsBootstrapMinJs, + "web/ui/static/vendor/bootstrap-3.3.1/js/npm.js": webUiStaticVendorBootstrap331JsNpmJs, + "web/ui/static/vendor/bootstrap-datetimepicker/bootstrap-datetimepicker.js": webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs, "web/ui/static/vendor/bootstrap-datetimepicker/bootstrap-datetimepicker.min.css": webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerMinCss, - "web/ui/static/vendor/bootstrap3-typeahead/bootstrap3-typeahead.min.js": webUiStaticVendorBootstrap3TypeaheadBootstrap3TypeaheadMinJs, - "web/ui/static/vendor/js/handlebars.js": webUiStaticVendorJsHandlebarsJs, - "web/ui/static/vendor/js/jquery.hotkeys.js": webUiStaticVendorJsJqueryHotkeysJs, - "web/ui/static/vendor/js/jquery.min.js": webUiStaticVendorJsJqueryMinJs, - "web/ui/static/vendor/js/jquery.selection.js": webUiStaticVendorJsJquerySelectionJs, - "web/ui/static/vendor/rickshaw/rickshaw.min.css": webUiStaticVendorRickshawRickshawMinCss, - "web/ui/static/vendor/rickshaw/rickshaw.min.js": webUiStaticVendorRickshawRickshawMinJs, - "web/ui/static/vendor/rickshaw/vendor/d3.layout.min.js": webUiStaticVendorRickshawVendorD3LayoutMinJs, - "web/ui/static/vendor/rickshaw/vendor/d3.v3.js": webUiStaticVendorRickshawVendorD3V3Js, + "web/ui/static/vendor/bootstrap3-typeahead/bootstrap3-typeahead.min.js": webUiStaticVendorBootstrap3TypeaheadBootstrap3TypeaheadMinJs, + "web/ui/static/vendor/js/handlebars.js": webUiStaticVendorJsHandlebarsJs, + "web/ui/static/vendor/js/jquery.hotkeys.js": webUiStaticVendorJsJqueryHotkeysJs, + "web/ui/static/vendor/js/jquery.min.js": webUiStaticVendorJsJqueryMinJs, + "web/ui/static/vendor/js/jquery.selection.js": webUiStaticVendorJsJquerySelectionJs, + "web/ui/static/vendor/rickshaw/rickshaw.min.css": webUiStaticVendorRickshawRickshawMinCss, + "web/ui/static/vendor/rickshaw/rickshaw.min.js": webUiStaticVendorRickshawRickshawMinJs, + "web/ui/static/vendor/rickshaw/vendor/d3.layout.min.js": webUiStaticVendorRickshawVendorD3LayoutMinJs, + "web/ui/static/vendor/rickshaw/vendor/d3.v3.js": webUiStaticVendorRickshawVendorD3V3Js, } // AssetDir returns the file names below a certain @@ -864,69 +864,70 @@ type bintree struct { Func func() (*asset, error) Children map[string]*bintree } + var _bintree = &bintree{nil, map[string]*bintree{ "web": &bintree{nil, map[string]*bintree{ "ui": &bintree{nil, map[string]*bintree{ "static": &bintree{nil, map[string]*bintree{ "css": &bintree{nil, map[string]*bintree{ - "alerts.css": &bintree{webUiStaticCssAlertsCss, map[string]*bintree{}}, - "graph.css": &bintree{webUiStaticCssGraphCss, map[string]*bintree{}}, + "alerts.css": &bintree{webUiStaticCssAlertsCss, map[string]*bintree{}}, + "graph.css": &bintree{webUiStaticCssGraphCss, map[string]*bintree{}}, "prom_console.css": &bintree{webUiStaticCssProm_consoleCss, map[string]*bintree{}}, - "prometheus.css": &bintree{webUiStaticCssPrometheusCss, map[string]*bintree{}}, + "prometheus.css": &bintree{webUiStaticCssPrometheusCss, map[string]*bintree{}}, }}, "img": &bintree{nil, map[string]*bintree{ "ajax-loader.gif": &bintree{webUiStaticImgAjaxLoaderGif, map[string]*bintree{}}, }}, "js": &bintree{nil, map[string]*bintree{ - "alerts.js": &bintree{webUiStaticJsAlertsJs, map[string]*bintree{}}, - "graph.js": &bintree{webUiStaticJsGraphJs, map[string]*bintree{}}, + "alerts.js": &bintree{webUiStaticJsAlertsJs, map[string]*bintree{}}, + "graph.js": &bintree{webUiStaticJsGraphJs, map[string]*bintree{}}, "graph_template.handlebar": &bintree{webUiStaticJsGraph_templateHandlebar, map[string]*bintree{}}, - "prom_console.js": &bintree{webUiStaticJsProm_consoleJs, map[string]*bintree{}}, + "prom_console.js": &bintree{webUiStaticJsProm_consoleJs, map[string]*bintree{}}, }}, "vendor": &bintree{nil, map[string]*bintree{ "bootstrap-3.3.1": &bintree{nil, map[string]*bintree{ "css": &bintree{nil, map[string]*bintree{ "bootstrap-theme.min.css": &bintree{webUiStaticVendorBootstrap331CssBootstrapThemeMinCss, map[string]*bintree{}}, - "bootstrap.min.css": &bintree{webUiStaticVendorBootstrap331CssBootstrapMinCss, map[string]*bintree{}}, + "bootstrap.min.css": &bintree{webUiStaticVendorBootstrap331CssBootstrapMinCss, map[string]*bintree{}}, }}, "fonts": &bintree{nil, map[string]*bintree{ - "glyphicons-halflings-regular.eot": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot, map[string]*bintree{}}, - "glyphicons-halflings-regular.svg": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg, map[string]*bintree{}}, - "glyphicons-halflings-regular.ttf": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf, map[string]*bintree{}}, + "glyphicons-halflings-regular.eot": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot, map[string]*bintree{}}, + "glyphicons-halflings-regular.svg": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg, map[string]*bintree{}}, + "glyphicons-halflings-regular.ttf": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf, map[string]*bintree{}}, "glyphicons-halflings-regular.woff": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularWoff, map[string]*bintree{}}, }}, "js": &bintree{nil, map[string]*bintree{ "bootstrap.min.js": &bintree{webUiStaticVendorBootstrap331JsBootstrapMinJs, map[string]*bintree{}}, - "npm.js": &bintree{webUiStaticVendorBootstrap331JsNpmJs, map[string]*bintree{}}, + "npm.js": &bintree{webUiStaticVendorBootstrap331JsNpmJs, map[string]*bintree{}}, }}, }}, "bootstrap-datetimepicker": &bintree{nil, map[string]*bintree{ - "bootstrap-datetimepicker.js": &bintree{webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs, map[string]*bintree{}}, + "bootstrap-datetimepicker.js": &bintree{webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs, map[string]*bintree{}}, "bootstrap-datetimepicker.min.css": &bintree{webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerMinCss, map[string]*bintree{}}, }}, "bootstrap3-typeahead": &bintree{nil, map[string]*bintree{ "bootstrap3-typeahead.min.js": &bintree{webUiStaticVendorBootstrap3TypeaheadBootstrap3TypeaheadMinJs, map[string]*bintree{}}, }}, "js": &bintree{nil, map[string]*bintree{ - "handlebars.js": &bintree{webUiStaticVendorJsHandlebarsJs, map[string]*bintree{}}, - "jquery.hotkeys.js": &bintree{webUiStaticVendorJsJqueryHotkeysJs, map[string]*bintree{}}, - "jquery.min.js": &bintree{webUiStaticVendorJsJqueryMinJs, map[string]*bintree{}}, + "handlebars.js": &bintree{webUiStaticVendorJsHandlebarsJs, map[string]*bintree{}}, + "jquery.hotkeys.js": &bintree{webUiStaticVendorJsJqueryHotkeysJs, map[string]*bintree{}}, + "jquery.min.js": &bintree{webUiStaticVendorJsJqueryMinJs, map[string]*bintree{}}, "jquery.selection.js": &bintree{webUiStaticVendorJsJquerySelectionJs, map[string]*bintree{}}, }}, "rickshaw": &bintree{nil, map[string]*bintree{ "rickshaw.min.css": &bintree{webUiStaticVendorRickshawRickshawMinCss, map[string]*bintree{}}, - "rickshaw.min.js": &bintree{webUiStaticVendorRickshawRickshawMinJs, map[string]*bintree{}}, + "rickshaw.min.js": &bintree{webUiStaticVendorRickshawRickshawMinJs, map[string]*bintree{}}, "vendor": &bintree{nil, map[string]*bintree{ "d3.layout.min.js": &bintree{webUiStaticVendorRickshawVendorD3LayoutMinJs, map[string]*bintree{}}, - "d3.v3.js": &bintree{webUiStaticVendorRickshawVendorD3V3Js, map[string]*bintree{}}, + "d3.v3.js": &bintree{webUiStaticVendorRickshawVendorD3V3Js, map[string]*bintree{}}, }}, }}, }}, }}, "templates": &bintree{nil, map[string]*bintree{ - "_base.html": &bintree{webUiTemplates_baseHtml, map[string]*bintree{}}, + "_base.html": &bintree{webUiTemplates_baseHtml, map[string]*bintree{}}, "alerts.html": &bintree{webUiTemplatesAlertsHtml, map[string]*bintree{}}, - "graph.html": &bintree{webUiTemplatesGraphHtml, map[string]*bintree{}}, + "graph.html": &bintree{webUiTemplatesGraphHtml, map[string]*bintree{}}, "status.html": &bintree{webUiTemplatesStatusHtml, map[string]*bintree{}}, }}, }}, @@ -979,4 +980,3 @@ func _filePath(dir, name string) string { cannonicalName := strings.Replace(name, "\\", "/", -1) return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...) } -