mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2025-01-11 05:47:27 -08:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Merge pull request #11774 from codesome/rules-ooo

Browse source 
rules: Consider ErrTooOldSample in expected errors
This commit is contained in:
Ganesh Vernekar 2023-01-06 14:56:14 +05:30 committed by GitHub
parent c49a28bb97 f1a332c496
commit 57cab58c53
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 13 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
rules/manager.go
View file

@ -649,6 +649,7 @@ func (g *Group) Eval(ctx context.Context, ts time.Time) {
} }
var ( var (
numOutOfOrder = 0 numOutOfOrder = 0
numTooOld = 0
numDuplicates = 0 numDuplicates = 0
) )
@ -680,6 +681,9 @@ func (g *Group) Eval(ctx context.Context, ts time.Time) {
case errors.Is(unwrappedErr, storage.ErrOutOfOrderSample): case errors.Is(unwrappedErr, storage.ErrOutOfOrderSample):
numOutOfOrder++ numOutOfOrder++
level.Debug(g.logger).Log("name", rule.Name(), "index", i, "msg", "Rule evaluation result discarded", "err", err, "sample", s) level.Debug(g.logger).Log("name", rule.Name(), "index", i, "msg", "Rule evaluation result discarded", "err", err, "sample", s)
case errors.Is(unwrappedErr, storage.ErrTooOldSample):
numTooOld++
level.Debug(g.logger).Log("name", rule.Name(), "index", i, "msg", "Rule evaluation result discarded", "err", err, "sample", s)
case errors.Is(unwrappedErr, storage.ErrDuplicateSampleForTimestamp): case errors.Is(unwrappedErr, storage.ErrDuplicateSampleForTimestamp):
numDuplicates++ numDuplicates++
level.Debug(g.logger).Log("name", rule.Name(), "index", i, "msg", "Rule evaluation result discarded", "err", err, "sample", s) level.Debug(g.logger).Log("name", rule.Name(), "index", i, "msg", "Rule evaluation result discarded", "err", err, "sample", s)
@ -694,6 +698,9 @@ func (g *Group) Eval(ctx context.Context, ts time.Time) {
if numOutOfOrder > 0 { if numOutOfOrder > 0 {
level.Warn(g.logger).Log("name", rule.Name(), "index", i, "msg", "Error on ingesting out-of-order result from rule evaluation", "numDropped", numOutOfOrder) level.Warn(g.logger).Log("name", rule.Name(), "index", i, "msg", "Error on ingesting out-of-order result from rule evaluation", "numDropped", numOutOfOrder)
} }
if numTooOld > 0 {
level.Warn(g.logger).Log("name", rule.Name(), "index", i, "msg", "Error on ingesting too old result from rule evaluation", "numDropped", numTooOld)
}
if numDuplicates > 0 { if numDuplicates > 0 {
level.Warn(g.logger).Log("name", rule.Name(), "index", i, "msg", "Error on ingesting results from rule evaluation with different value but same timestamp", "numDropped", numDuplicates) level.Warn(g.logger).Log("name", rule.Name(), "index", i, "msg", "Error on ingesting results from rule evaluation with different value but same timestamp", "numDropped", numDuplicates)
} }
@ -708,7 +715,9 @@ func (g *Group) Eval(ctx context.Context, ts time.Time) {
} }
switch { switch {
case unwrappedErr == nil: case unwrappedErr == nil:
case errors.Is(unwrappedErr, storage.ErrOutOfOrderSample), errors.Is(unwrappedErr, storage.ErrDuplicateSampleForTimestamp): case errors.Is(unwrappedErr, storage.ErrOutOfOrderSample),
errors.Is(unwrappedErr, storage.ErrTooOldSample),
errors.Is(unwrappedErr, storage.ErrDuplicateSampleForTimestamp):
// Do not count these in logging, as this is expected if series // Do not count these in logging, as this is expected if series
// is exposed from a different rule. // is exposed from a different rule.
default: default:
@ -738,7 +747,9 @@ func (g *Group) cleanupStaleSeries(ctx context.Context, ts time.Time) {
} }
switch { switch {
case unwrappedErr == nil: case unwrappedErr == nil:
case errors.Is(unwrappedErr, storage.ErrOutOfOrderSample), errors.Is(unwrappedErr, storage.ErrDuplicateSampleForTimestamp): case errors.Is(unwrappedErr, storage.ErrOutOfOrderSample),
errors.Is(unwrappedErr, storage.ErrTooOldSample),
errors.Is(unwrappedErr, storage.ErrDuplicateSampleForTimestamp):
// Do not count these in logging, as this is expected if series // Do not count these in logging, as this is expected if series
// is exposed from a different rule. // is exposed from a different rule.
default: default: