mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2025-03-05 20:59:13 -08:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Merge pull request #1644 from prometheus/beorn7/logging

Browse source 
Add missing logging of out-of-order samples
This commit is contained in:
Fabian Reinartz 2016-05-20 05:52:00 -07:00
parent dec56838fc 45e5775f9b
commit f7ed2ff706
2 changed files with 29 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
retrieval/scrape.go
View file

@ -496,6 +496,10 @@ func (sl *scrapeLoop) report(start time.Time, duration time.Duration, err error)
Value: model.SampleValue(float64(duration) / float64(time.Second)), Value: model.SampleValue(float64(duration) / float64(time.Second)),
} }
sl.reportAppender.Append(healthSample) if err := sl.reportAppender.Append(healthSample); err != nil {
sl.reportAppender.Append(durationSample) log.With("sample", healthSample).With("error", err).Warn("Scrape health sample discarded")
}
if err := sl.reportAppender.Append(durationSample); err != nil {
log.With("sample", durationSample).With("error", err).Warn("Scrape duration sample discarded")
}
} }

24
rules/manager.go
View file

@ -31,6 +31,7 @@ import (
"github.com/prometheus/prometheus/notifier" "github.com/prometheus/prometheus/notifier"
"github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/promql"
"github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/storage"
"github.com/prometheus/prometheus/storage/local"
"github.com/prometheus/prometheus/template" "github.com/prometheus/prometheus/template"
"github.com/prometheus/prometheus/util/strutil" "github.com/prometheus/prometheus/util/strutil"
) )
@ -270,8 +271,29 @@ func (g *Group) eval() {
if ar, ok := rule.(*AlertingRule); ok { if ar, ok := rule.(*AlertingRule); ok {
g.sendAlerts(ar, now) g.sendAlerts(ar, now)
} }
var (
numOutOfOrder = 0
numDuplicates = 0
)
for _, s := range vector { for _, s := range vector {
g.opts.SampleAppender.Append(s) if err := g.opts.SampleAppender.Append(s); err != nil {
switch err {
case local.ErrOutOfOrderSample:
numOutOfOrder++
log.With("sample", s).With("error", err).Debug("Rule evaluation result discarded")
case local.ErrDuplicateSampleForTimestamp:
numDuplicates++
log.With("sample", s).With("error", err).Debug("Rule evaluation result discarded")
default:
log.With("sample", s).With("error", err).Warn("Rule evaluation result discarded")
}
}
}
if numOutOfOrder > 0 {
log.With("numDropped", numOutOfOrder).Warn("Error on ingesting out-of-order result from rule evaluation")
}
if numDuplicates > 0 {
log.With("numDropped", numDuplicates).Warn("Error on ingesting results from rule evaluation with different value but same timestamp")
} }
}(rule) }(rule)
} }