Merge pull request #1330 from prometheus/swapfix

Fix swapped constants, improve instrumentation
This commit is contained in:
Fabian Reinartz 2016-01-21 13:06:55 +01:00
commit c56f35274e
2 changed files with 84 additions and 79 deletions

View file

@ -36,11 +36,7 @@ import (
)
// Constants for instrumentation.
const (
namespace = "prometheus"
ruleTypeLabel = "rule_type"
)
const namespace = "prometheus"
var (
evalDuration = prometheus.NewSummaryVec(
@ -49,21 +45,23 @@ var (
Name: "rule_evaluation_duration_seconds",
Help: "The duration for a rule to execute.",
},
[]string{ruleTypeLabel},
[]string{"rule_type"},
)
evalFailures = prometheus.NewCounter(
evalFailures = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: namespace,
Name: "rule_evaluation_failures_total",
Help: "The total number of rule evaluation failures.",
},
[]string{"rule_type"},
)
evalTotal = prometheus.NewCounter(
evalTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: namespace,
Name: "rule_evaluations_total",
Help: "The total number of rule evaluations.",
},
[]string{"rule_type"},
)
iterationDuration = prometheus.NewSummary(prometheus.SummaryOpts{
Namespace: namespace,
@ -74,6 +72,11 @@ var (
)
func init() {
evalTotal.WithLabelValues(string(ruleTypeAlert))
evalTotal.WithLabelValues(string(ruleTypeRecording))
evalFailures.WithLabelValues(string(ruleTypeAlert))
evalFailures.WithLabelValues(string(ruleTypeRecording))
prometheus.MustRegister(iterationDuration)
prometheus.MustRegister(evalFailures)
prometheus.MustRegister(evalDuration)
@ -200,6 +203,16 @@ func (g *Group) copyState(from *Group) {
}
}
func typeForRule(r Rule) ruleType {
switch r.(type) {
case *AlertingRule:
return ruleTypeAlert
case *RecordingRule:
return ruleTypeRecording
}
panic(fmt.Errorf("unknown rule type: %T", r))
}
// eval runs a single evaluation cycle in which all rules are evaluated in parallel.
// In the future a single group will be evaluated sequentially to properly handle
// rule dependency.
@ -210,13 +223,18 @@ func (g *Group) eval() {
)
for _, rule := range g.rules {
rtyp := string(typeForRule(rule))
wg.Add(1)
// BUG(julius): Look at fixing thundering herd.
go func(rule Rule) {
defer wg.Done()
start := time.Now()
evalTotal.Inc()
defer func(t time.Time) {
evalDuration.WithLabelValues(rtyp).Observe(float64(time.Since(t)) / float64(time.Second))
}(time.Now())
evalTotal.WithLabelValues(rtyp).Inc()
vector, err := rule.eval(now, g.opts.QueryEngine)
if err != nil {
@ -225,26 +243,13 @@ func (g *Group) eval() {
if _, ok := err.(promql.ErrQueryCanceled); !ok {
log.Warnf("Error while evaluating rule %q: %s", rule, err)
}
evalFailures.Inc()
}
var rtyp ruleType
switch r := rule.(type) {
case *AlertingRule:
rtyp = ruleTypeRecording
g.sendAlerts(r, now)
case *RecordingRule:
rtyp = ruleTypeAlert
default:
panic(fmt.Errorf("unknown rule type: %T", rule))
evalFailures.WithLabelValues(rtyp).Inc()
return
}
evalDuration.WithLabelValues(string(rtyp)).Observe(
float64(time.Since(start)) / float64(time.Second),
)
if ar, ok := rule.(*AlertingRule); ok {
g.sendAlerts(ar, now)
}
for _, s := range vector {
g.opts.SampleAppender.Append(s)
}

View file

@ -791,38 +791,38 @@ func AssetNames() []string {
// _bindata is a table, holding each asset generator, mapped to its name.
var _bindata = map[string]func() (*asset, error){
"web/ui/templates/_base.html": webUiTemplates_baseHtml,
"web/ui/templates/alerts.html": webUiTemplatesAlertsHtml,
"web/ui/templates/graph.html": webUiTemplatesGraphHtml,
"web/ui/templates/status.html": webUiTemplatesStatusHtml,
"web/ui/static/css/alerts.css": webUiStaticCssAlertsCss,
"web/ui/static/css/graph.css": webUiStaticCssGraphCss,
"web/ui/static/css/prom_console.css": webUiStaticCssProm_consoleCss,
"web/ui/static/css/prometheus.css": webUiStaticCssPrometheusCss,
"web/ui/static/img/ajax-loader.gif": webUiStaticImgAjaxLoaderGif,
"web/ui/static/js/alerts.js": webUiStaticJsAlertsJs,
"web/ui/static/js/graph.js": webUiStaticJsGraphJs,
"web/ui/static/js/graph_template.handlebar": webUiStaticJsGraph_templateHandlebar,
"web/ui/static/js/prom_console.js": webUiStaticJsProm_consoleJs,
"web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap-theme.min.css": webUiStaticVendorBootstrap331CssBootstrapThemeMinCss,
"web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap.min.css": webUiStaticVendorBootstrap331CssBootstrapMinCss,
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.eot": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot,
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.svg": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg,
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.ttf": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf,
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.woff": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularWoff,
"web/ui/static/vendor/bootstrap-3.3.1/js/bootstrap.min.js": webUiStaticVendorBootstrap331JsBootstrapMinJs,
"web/ui/static/vendor/bootstrap-3.3.1/js/npm.js": webUiStaticVendorBootstrap331JsNpmJs,
"web/ui/static/vendor/bootstrap-datetimepicker/bootstrap-datetimepicker.js": webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs,
"web/ui/templates/_base.html": webUiTemplates_baseHtml,
"web/ui/templates/alerts.html": webUiTemplatesAlertsHtml,
"web/ui/templates/graph.html": webUiTemplatesGraphHtml,
"web/ui/templates/status.html": webUiTemplatesStatusHtml,
"web/ui/static/css/alerts.css": webUiStaticCssAlertsCss,
"web/ui/static/css/graph.css": webUiStaticCssGraphCss,
"web/ui/static/css/prom_console.css": webUiStaticCssProm_consoleCss,
"web/ui/static/css/prometheus.css": webUiStaticCssPrometheusCss,
"web/ui/static/img/ajax-loader.gif": webUiStaticImgAjaxLoaderGif,
"web/ui/static/js/alerts.js": webUiStaticJsAlertsJs,
"web/ui/static/js/graph.js": webUiStaticJsGraphJs,
"web/ui/static/js/graph_template.handlebar": webUiStaticJsGraph_templateHandlebar,
"web/ui/static/js/prom_console.js": webUiStaticJsProm_consoleJs,
"web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap-theme.min.css": webUiStaticVendorBootstrap331CssBootstrapThemeMinCss,
"web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap.min.css": webUiStaticVendorBootstrap331CssBootstrapMinCss,
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.eot": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot,
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.svg": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg,
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.ttf": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf,
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.woff": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularWoff,
"web/ui/static/vendor/bootstrap-3.3.1/js/bootstrap.min.js": webUiStaticVendorBootstrap331JsBootstrapMinJs,
"web/ui/static/vendor/bootstrap-3.3.1/js/npm.js": webUiStaticVendorBootstrap331JsNpmJs,
"web/ui/static/vendor/bootstrap-datetimepicker/bootstrap-datetimepicker.js": webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs,
"web/ui/static/vendor/bootstrap-datetimepicker/bootstrap-datetimepicker.min.css": webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerMinCss,
"web/ui/static/vendor/bootstrap3-typeahead/bootstrap3-typeahead.min.js": webUiStaticVendorBootstrap3TypeaheadBootstrap3TypeaheadMinJs,
"web/ui/static/vendor/js/handlebars.js": webUiStaticVendorJsHandlebarsJs,
"web/ui/static/vendor/js/jquery.hotkeys.js": webUiStaticVendorJsJqueryHotkeysJs,
"web/ui/static/vendor/js/jquery.min.js": webUiStaticVendorJsJqueryMinJs,
"web/ui/static/vendor/js/jquery.selection.js": webUiStaticVendorJsJquerySelectionJs,
"web/ui/static/vendor/rickshaw/rickshaw.min.css": webUiStaticVendorRickshawRickshawMinCss,
"web/ui/static/vendor/rickshaw/rickshaw.min.js": webUiStaticVendorRickshawRickshawMinJs,
"web/ui/static/vendor/rickshaw/vendor/d3.layout.min.js": webUiStaticVendorRickshawVendorD3LayoutMinJs,
"web/ui/static/vendor/rickshaw/vendor/d3.v3.js": webUiStaticVendorRickshawVendorD3V3Js,
"web/ui/static/vendor/bootstrap3-typeahead/bootstrap3-typeahead.min.js": webUiStaticVendorBootstrap3TypeaheadBootstrap3TypeaheadMinJs,
"web/ui/static/vendor/js/handlebars.js": webUiStaticVendorJsHandlebarsJs,
"web/ui/static/vendor/js/jquery.hotkeys.js": webUiStaticVendorJsJqueryHotkeysJs,
"web/ui/static/vendor/js/jquery.min.js": webUiStaticVendorJsJqueryMinJs,
"web/ui/static/vendor/js/jquery.selection.js": webUiStaticVendorJsJquerySelectionJs,
"web/ui/static/vendor/rickshaw/rickshaw.min.css": webUiStaticVendorRickshawRickshawMinCss,
"web/ui/static/vendor/rickshaw/rickshaw.min.js": webUiStaticVendorRickshawRickshawMinJs,
"web/ui/static/vendor/rickshaw/vendor/d3.layout.min.js": webUiStaticVendorRickshawVendorD3LayoutMinJs,
"web/ui/static/vendor/rickshaw/vendor/d3.v3.js": webUiStaticVendorRickshawVendorD3V3Js,
}
// AssetDir returns the file names below a certain
@ -864,69 +864,70 @@ type bintree struct {
Func func() (*asset, error)
Children map[string]*bintree
}
var _bintree = &bintree{nil, map[string]*bintree{
"web": &bintree{nil, map[string]*bintree{
"ui": &bintree{nil, map[string]*bintree{
"static": &bintree{nil, map[string]*bintree{
"css": &bintree{nil, map[string]*bintree{
"alerts.css": &bintree{webUiStaticCssAlertsCss, map[string]*bintree{}},
"graph.css": &bintree{webUiStaticCssGraphCss, map[string]*bintree{}},
"alerts.css": &bintree{webUiStaticCssAlertsCss, map[string]*bintree{}},
"graph.css": &bintree{webUiStaticCssGraphCss, map[string]*bintree{}},
"prom_console.css": &bintree{webUiStaticCssProm_consoleCss, map[string]*bintree{}},
"prometheus.css": &bintree{webUiStaticCssPrometheusCss, map[string]*bintree{}},
"prometheus.css": &bintree{webUiStaticCssPrometheusCss, map[string]*bintree{}},
}},
"img": &bintree{nil, map[string]*bintree{
"ajax-loader.gif": &bintree{webUiStaticImgAjaxLoaderGif, map[string]*bintree{}},
}},
"js": &bintree{nil, map[string]*bintree{
"alerts.js": &bintree{webUiStaticJsAlertsJs, map[string]*bintree{}},
"graph.js": &bintree{webUiStaticJsGraphJs, map[string]*bintree{}},
"alerts.js": &bintree{webUiStaticJsAlertsJs, map[string]*bintree{}},
"graph.js": &bintree{webUiStaticJsGraphJs, map[string]*bintree{}},
"graph_template.handlebar": &bintree{webUiStaticJsGraph_templateHandlebar, map[string]*bintree{}},
"prom_console.js": &bintree{webUiStaticJsProm_consoleJs, map[string]*bintree{}},
"prom_console.js": &bintree{webUiStaticJsProm_consoleJs, map[string]*bintree{}},
}},
"vendor": &bintree{nil, map[string]*bintree{
"bootstrap-3.3.1": &bintree{nil, map[string]*bintree{
"css": &bintree{nil, map[string]*bintree{
"bootstrap-theme.min.css": &bintree{webUiStaticVendorBootstrap331CssBootstrapThemeMinCss, map[string]*bintree{}},
"bootstrap.min.css": &bintree{webUiStaticVendorBootstrap331CssBootstrapMinCss, map[string]*bintree{}},
"bootstrap.min.css": &bintree{webUiStaticVendorBootstrap331CssBootstrapMinCss, map[string]*bintree{}},
}},
"fonts": &bintree{nil, map[string]*bintree{
"glyphicons-halflings-regular.eot": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot, map[string]*bintree{}},
"glyphicons-halflings-regular.svg": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg, map[string]*bintree{}},
"glyphicons-halflings-regular.ttf": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf, map[string]*bintree{}},
"glyphicons-halflings-regular.eot": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot, map[string]*bintree{}},
"glyphicons-halflings-regular.svg": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg, map[string]*bintree{}},
"glyphicons-halflings-regular.ttf": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf, map[string]*bintree{}},
"glyphicons-halflings-regular.woff": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularWoff, map[string]*bintree{}},
}},
"js": &bintree{nil, map[string]*bintree{
"bootstrap.min.js": &bintree{webUiStaticVendorBootstrap331JsBootstrapMinJs, map[string]*bintree{}},
"npm.js": &bintree{webUiStaticVendorBootstrap331JsNpmJs, map[string]*bintree{}},
"npm.js": &bintree{webUiStaticVendorBootstrap331JsNpmJs, map[string]*bintree{}},
}},
}},
"bootstrap-datetimepicker": &bintree{nil, map[string]*bintree{
"bootstrap-datetimepicker.js": &bintree{webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs, map[string]*bintree{}},
"bootstrap-datetimepicker.js": &bintree{webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs, map[string]*bintree{}},
"bootstrap-datetimepicker.min.css": &bintree{webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerMinCss, map[string]*bintree{}},
}},
"bootstrap3-typeahead": &bintree{nil, map[string]*bintree{
"bootstrap3-typeahead.min.js": &bintree{webUiStaticVendorBootstrap3TypeaheadBootstrap3TypeaheadMinJs, map[string]*bintree{}},
}},
"js": &bintree{nil, map[string]*bintree{
"handlebars.js": &bintree{webUiStaticVendorJsHandlebarsJs, map[string]*bintree{}},
"jquery.hotkeys.js": &bintree{webUiStaticVendorJsJqueryHotkeysJs, map[string]*bintree{}},
"jquery.min.js": &bintree{webUiStaticVendorJsJqueryMinJs, map[string]*bintree{}},
"handlebars.js": &bintree{webUiStaticVendorJsHandlebarsJs, map[string]*bintree{}},
"jquery.hotkeys.js": &bintree{webUiStaticVendorJsJqueryHotkeysJs, map[string]*bintree{}},
"jquery.min.js": &bintree{webUiStaticVendorJsJqueryMinJs, map[string]*bintree{}},
"jquery.selection.js": &bintree{webUiStaticVendorJsJquerySelectionJs, map[string]*bintree{}},
}},
"rickshaw": &bintree{nil, map[string]*bintree{
"rickshaw.min.css": &bintree{webUiStaticVendorRickshawRickshawMinCss, map[string]*bintree{}},
"rickshaw.min.js": &bintree{webUiStaticVendorRickshawRickshawMinJs, map[string]*bintree{}},
"rickshaw.min.js": &bintree{webUiStaticVendorRickshawRickshawMinJs, map[string]*bintree{}},
"vendor": &bintree{nil, map[string]*bintree{
"d3.layout.min.js": &bintree{webUiStaticVendorRickshawVendorD3LayoutMinJs, map[string]*bintree{}},
"d3.v3.js": &bintree{webUiStaticVendorRickshawVendorD3V3Js, map[string]*bintree{}},
"d3.v3.js": &bintree{webUiStaticVendorRickshawVendorD3V3Js, map[string]*bintree{}},
}},
}},
}},
}},
"templates": &bintree{nil, map[string]*bintree{
"_base.html": &bintree{webUiTemplates_baseHtml, map[string]*bintree{}},
"_base.html": &bintree{webUiTemplates_baseHtml, map[string]*bintree{}},
"alerts.html": &bintree{webUiTemplatesAlertsHtml, map[string]*bintree{}},
"graph.html": &bintree{webUiTemplatesGraphHtml, map[string]*bintree{}},
"graph.html": &bintree{webUiTemplatesGraphHtml, map[string]*bintree{}},
"status.html": &bintree{webUiTemplatesStatusHtml, map[string]*bintree{}},
}},
}},
@ -979,4 +980,3 @@ func _filePath(dir, name string) string {
cannonicalName := strings.Replace(name, "\\", "/", -1)
return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...)
}