mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2024-12-25 05:34:05 -08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Run rule evaluation with timestamps precisely evaluation_interval apart (#4201)

Browse source 
* Run rule evaluation with timestamps precisely evaluation_interval apart from one another.

Signed-off-by: Alin Sinpalean <alin.sinpalean@gmail.com>
This commit is contained in:
Alin Sinpalean 2018-06-01 16:23:07 +02:00 committed by Brian Brazil
parent 05cf674279
commit 9dc763cc03
1 changed files with 19 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
rules/manager.go
View file

@ -192,8 +192,9 @@ func (g *Group) run(ctx context.Context) {
defer close(g.terminated)
// Wait an initial amount to have consistently slotted intervals.
evalTimestamp := g.evalTimestamp().Add(g.interval)
select {
case <-time.After(g.offset()):
case <-time.After(time.Until(evalTimestamp)):
case <-g.done:
return
}
@ -202,17 +203,20 @@ func (g *Group) run(ctx context.Context) {
iterationsScheduled.Inc()
start := time.Now()
g.Eval(ctx, start)
g.Eval(ctx, evalTimestamp)
timeSinceStart := time.Since(start)
iterationDuration.Observe(time.Since(start).Seconds())
g.SetEvaluationTime(time.Since(start))
iterationDuration.Observe(timeSinceStart.Seconds())
g.SetEvaluationTime(timeSinceStart)
}
lastTriggered := time.Now()
iter()
// The assumption here is that since the ticker was started after having
// waited for `evalTimestamp` to pass, the ticks will trigger soon
// after each `evalTimestamp + N * g.interval` occurrence.
tick := time.NewTicker(g.interval)
defer tick.Stop()
iter()
for {
select {
case <-g.done:
@ -222,12 +226,12 @@ func (g *Group) run(ctx context.Context) {
case <-g.done:
return
case <-tick.C:
missed := (time.Since(lastTriggered).Nanoseconds() / g.interval.Nanoseconds()) - 1
missed := (time.Since(evalTimestamp) / g.interval) - 1
if missed > 0 {
iterationsMissed.Add(float64(missed))
iterationsScheduled.Add(float64(missed))
}
lastTriggered = time.Now()
evalTimestamp = evalTimestamp.Add((missed + 1) * g.interval)
iter()
}
}
@ -261,20 +265,16 @@ func (g *Group) SetEvaluationTime(dur time.Duration) {
g.evaluationTime = dur
}
// offset returns until the next consistently slotted evaluation interval.
func (g *Group) offset() time.Duration {
now := time.Now().UnixNano()
// evalTimestamp returns the immediately preceding consistently slotted evaluation time.
func (g *Group) evalTimestamp() time.Time {
var (
base = now - (now % int64(g.interval))
offset = g.hash() % uint64(g.interval)
next = base + int64(offset)
offset = int64(g.hash() % uint64(g.interval))
now = time.Now().UnixNano()
adjNow = now - offset
base = adjNow - (adjNow % int64(g.interval))
)
if next < now {
next += int64(g.interval)
}
return time.Duration(next - now)
return time.Unix(0, base+offset)
}
// copyState copies the alerting rule and staleness related state from the given group.