diff --git a/retrieval/target.go b/retrieval/target.go index c7187d4ecb..b0352bfe7e 100644 --- a/retrieval/target.go +++ b/retrieval/target.go @@ -138,7 +138,8 @@ func (t *target) Scrape(earliest time.Time, results chan format.Result) (err err done <- true }() - resp, err := http.Get(t.Address()) + var resp *http.Response // Don't shadow "err" from the enclosing function. + resp, err = http.Get(t.Address()) if err != nil { return } diff --git a/retrieval/targetpool.go b/retrieval/targetpool.go index ccc3647804..8fb5f81773 100644 --- a/retrieval/targetpool.go +++ b/retrieval/targetpool.go @@ -90,7 +90,11 @@ func (p *TargetPool) runIteration(results chan format.Result, interval time.Dura if target.scheduledFor().After(now) { heap.Push(p, target) - + // None of the remaining targets are ready to be scheduled. Signal that + // we're done processing them in this scrape iteration. + for j := i; j < targetCount; j++ { + finished <- true + } break } @@ -107,6 +111,6 @@ func (p *TargetPool) runIteration(results chan format.Result, interval time.Dura close(finished) - duration := float64(time.Now().Sub(begin) / time.Millisecond) + duration := float64(time.Since(begin) / time.Millisecond) retrievalDurations.Add(map[string]string{intervalKey: interval.String()}, duration) } diff --git a/retrieval/targetpool_test.go b/retrieval/targetpool_test.go index 71afb7ef59..56ca363813 100644 --- a/retrieval/targetpool_test.go +++ b/retrieval/targetpool_test.go @@ -15,6 +15,7 @@ package retrieval import ( "container/heap" + "github.com/prometheus/prometheus/retrieval/format" "github.com/prometheus/prometheus/utility/test" "testing" "time" @@ -45,8 +46,8 @@ func testTargetPool(t test.Tester) { var scenarios = []struct { name string - outputs []output inputs []input + outputs []output }{ { name: "empty", @@ -160,6 +161,28 @@ func TestTargetPool(t *testing.T) { testTargetPool(t) } +func TestTargetPoolIterationWithUnhealthyTargetsFinishes(t *testing.T) { + pool := TargetPool{} + target := &target{ + address: "http://example.com/metrics.json", + scheduler: literalScheduler(time.Date(9999, 1, 1, 0, 0, 0, 0, time.UTC)), + } + pool.Push(target) + + done := make(chan bool) + go func() { + pool.runIteration(make(chan format.Result), time.Duration(0)) + done <- true + }() + + select { + case <-done: + break + case <-time.After(time.Duration(1) * time.Second): + t.Fatalf("Targetpool iteration is stuck") + } +} + func BenchmarkTargetPool(b *testing.B) { for i := 0; i < b.N; i++ { testTargetPool(b)