From 52f52a7ee2405634ab19ff1b95bf8fe79b8e178f Mon Sep 17 00:00:00 2001 From: "Matt T. Proud" Date: Fri, 4 Jan 2013 17:55:58 +0100 Subject: [PATCH] Include nascent instrumentation of stack. --- main.go | 10 +++- retrieval/instrumentation.go | 41 +++++++++++++++ retrieval/target.go | 19 +++++-- storage/metric/leveldb/instrumentation.go | 36 +++++++++++++ storage/metric/leveldb/mutable.go | 63 +++++++++++++---------- 5 files changed, 138 insertions(+), 31 deletions(-) create mode 100644 retrieval/instrumentation.go create mode 100644 storage/metric/leveldb/instrumentation.go diff --git a/main.go b/main.go index cc586dcb3..1a29e6ad4 100644 --- a/main.go +++ b/main.go @@ -14,10 +14,11 @@ package main import ( - "fmt" + "github.com/matttproud/golang_instrumentation" "github.com/matttproud/prometheus/retrieval" "github.com/matttproud/prometheus/storage/metric/leveldb" "log" + "net/http" "os" "time" ) @@ -44,9 +45,14 @@ func main() { manager := retrieval.NewTargetManager(results, 1) manager.Add(t) + go func() { + exporter := registry.DefaultRegistry.YieldExporter() + http.Handle("/metrics.json", exporter) + http.ListenAndServe(":9090", nil) + }() + for { result := <-results - fmt.Printf("result -> %s\n", result) for _, s := range result.Samples { m.AppendSample(&s) } diff --git a/retrieval/instrumentation.go b/retrieval/instrumentation.go new file mode 100644 index 000000000..3513ce18e --- /dev/null +++ b/retrieval/instrumentation.go @@ -0,0 +1,41 @@ +// Copyright 2013 Prometheus Team +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package retrieval + +import ( + "github.com/matttproud/golang_instrumentation" + "github.com/matttproud/golang_instrumentation/maths" + "github.com/matttproud/golang_instrumentation/metrics" +) + +var ( + networkLatencyHistogram = &metrics.HistogramSpecification{ + Starts: metrics.LogarithmicSizedBucketsFor(0, 1000), + BucketMaker: metrics.AccumulatingBucketBuilder(metrics.EvictAndReplaceWith(10, maths.Average), 100), + ReportablePercentiles: []float64{0.01, 0.05, 0.5, 0.90, 0.99}, + } + + targetsHealthy = &metrics.CounterMetric{} + targetsUnhealthy = &metrics.CounterMetric{} + + scrapeLatencyHealthy = metrics.CreateHistogram(networkLatencyHistogram) + scrapeLatencyUnhealthy = metrics.CreateHistogram(networkLatencyHistogram) +) + +func init() { + registry.Register("targets_healthy_total", targetsHealthy) + registry.Register("targets_unhealthy_total", targetsUnhealthy) + registry.Register("targets_healthy_scrape_latency_ms", scrapeLatencyHealthy) + registry.Register("targets_unhealthy_scrape_latency_ms", scrapeLatencyUnhealthy) +} diff --git a/retrieval/target.go b/retrieval/target.go index 57454418a..2a897d5f4 100644 --- a/retrieval/target.go +++ b/retrieval/target.go @@ -15,6 +15,7 @@ package retrieval import ( "encoding/json" "fmt" + "github.com/matttproud/golang_instrumentation/metrics" "github.com/matttproud/prometheus/model" "io/ioutil" "log" @@ -62,6 +63,7 @@ func (t *Target) reschedule(s TargetState) { switch s { case ALIVE: t.unreachableCount = 0 + targetsHealthy.Increment() case UNREACHABLE: backoff := MAXIMUM_BACKOFF exponential := time.Duration(math.Pow(2, float64(t.unreachableCount))) * time.Second @@ -71,7 +73,6 @@ func (t *Target) reschedule(s TargetState) { t.scheduledFor = time.Now().Add(backoff) t.unreachableCount++ - log.Printf("%s unavailable %s times deferred for %s.", t, t.unreachableCount, backoff) default: } @@ -79,6 +80,7 @@ func (t *Target) reschedule(s TargetState) { switch s { case UNREACHABLE: t.unreachableCount++ + targetsUnhealthy.Increment() } default: } @@ -111,7 +113,7 @@ func (t *Target) Scrape(results chan Result) (err error) { done := make(chan bool) - go func() { + request := func() { ti := time.Now() resp, err := http.Get(t.Address) if err != nil { @@ -197,7 +199,18 @@ func (t *Target) Scrape(results chan Result) (err error) { } done <- true - }() + } + + accumulator := func(d time.Duration) { + ms := float64(d) / float64(time.Millisecond) + if err == nil { + scrapeLatencyHealthy.Add(ms) + } else { + scrapeLatencyUnhealthy.Add(ms) + } + } + + go metrics.InstrumentCall(request, accumulator) select { case <-done: diff --git a/storage/metric/leveldb/instrumentation.go b/storage/metric/leveldb/instrumentation.go new file mode 100644 index 000000000..df1779b42 --- /dev/null +++ b/storage/metric/leveldb/instrumentation.go @@ -0,0 +1,36 @@ +// Copyright 2012 Prometheus Team +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package leveldb + +import ( + "github.com/matttproud/golang_instrumentation" + "github.com/matttproud/golang_instrumentation/maths" + "github.com/matttproud/golang_instrumentation/metrics" +) + +var ( + diskLatencyHistogram = &metrics.HistogramSpecification{ + Starts: metrics.LogarithmicSizedBucketsFor(0, 5000), + BucketMaker: metrics.AccumulatingBucketBuilder(metrics.EvictAndReplaceWith(10, maths.Average), 100), + ReportablePercentiles: []float64{0.01, 0.05, 0.5, 0.90, 0.99}, + } + + targetsHealthy = &metrics.CounterMetric{} + + appendLatency = metrics.CreateHistogram(diskLatencyHistogram) +) + +func init() { + registry.Register("sample_append_disk_latency_microseconds", appendLatency) +} diff --git a/storage/metric/leveldb/mutable.go b/storage/metric/leveldb/mutable.go index 62e75f981..61ad2495d 100644 --- a/storage/metric/leveldb/mutable.go +++ b/storage/metric/leveldb/mutable.go @@ -21,6 +21,7 @@ import ( "github.com/matttproud/prometheus/coding/indexable" "github.com/matttproud/prometheus/model" dto "github.com/matttproud/prometheus/model/generated" + "time" ) var ( @@ -149,44 +150,54 @@ func (l *LevelDBMetricPersistence) AppendSample(sample *model.Sample) (err error m.Increment() }() - metricDTO := model.SampleToMetricDTO(sample) + operation := func() { + metricDTO := model.SampleToMetricDTO(sample) - indexHas, err := l.hasIndexMetric(metricDTO) - if err != nil { - return - } - - if !indexHas { - err = l.indexMetric(metricDTO) + indexHas, err := l.hasIndexMetric(metricDTO) if err != nil { return } - err = l.appendFingerprints(metricDTO) + if !indexHas { + err = l.indexMetric(metricDTO) + if err != nil { + return + } + + err = l.appendFingerprints(metricDTO) + if err != nil { + return + } + } + + fingerprintDTO, err := model.MessageToFingerprintDTO(metricDTO) + if err != nil { + return + } + + sampleKeyDTO := &dto.SampleKey{ + Fingerprint: fingerprintDTO, + Timestamp: indexable.EncodeTime(sample.Timestamp), + } + sampleValueDTO := &dto.SampleValue{ + Value: proto.Float32(float32(sample.Value)), + } + sampleKeyEncoded := coding.NewProtocolBufferEncoder(sampleKeyDTO) + sampleValueEncoded := coding.NewProtocolBufferEncoder(sampleValueDTO) + + err = l.metricSamples.Put(sampleKeyEncoded, sampleValueEncoded) if err != nil { return } } - fingerprintDTO, err := model.MessageToFingerprintDTO(metricDTO) - if err != nil { - return + // XXX: Problematic with panics. + accumulator := func(d time.Duration) { + ms := float64(d) / float64(time.Microsecond) + appendLatency.Add(ms) } - sampleKeyDTO := &dto.SampleKey{ - Fingerprint: fingerprintDTO, - Timestamp: indexable.EncodeTime(sample.Timestamp), - } - sampleValueDTO := &dto.SampleValue{ - Value: proto.Float32(float32(sample.Value)), - } - sampleKeyEncoded := coding.NewProtocolBufferEncoder(sampleKeyDTO) - sampleValueEncoded := coding.NewProtocolBufferEncoder(sampleValueDTO) - - err = l.metricSamples.Put(sampleKeyEncoded, sampleValueEncoded) - if err != nil { - return - } + metrics.InstrumentCall(operation, accumulator) return }