Merge pull request #20 from matttproud/feature/instrument-internals

Feature/instrument internals

main.go

@@ -14,10 +14,11 @@
 package main
 
 import (
-	"fmt"
+	"github.com/matttproud/golang_instrumentation"
 	"github.com/matttproud/prometheus/retrieval"
 	"github.com/matttproud/prometheus/storage/metric/leveldb"
 	"log"
+	"net/http"
 	"os"
 	"time"
 )
@@ -44,9 +45,14 @@ func main() {
 	manager := retrieval.NewTargetManager(results, 1)
 	manager.Add(t)
 
+	go func() {
+		exporter := registry.DefaultRegistry.YieldExporter()
+		http.Handle("/metrics.json", exporter)
+		http.ListenAndServe(":9090", nil)
+	}()
+
 	for {
 		result := <-results
-		fmt.Printf("result -> %s\n", result)
 		for _, s := range result.Samples {
 			m.AppendSample(&s)
 		}

retrieval/instrumentation.go

@@ -0,0 +1,41 @@
+// Copyright 2013 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package retrieval
+
+import (
+	"github.com/matttproud/golang_instrumentation"
+	"github.com/matttproud/golang_instrumentation/maths"
+	"github.com/matttproud/golang_instrumentation/metrics"
+)
+
+var (
+	networkLatencyHistogram = &metrics.HistogramSpecification{
+		Starts:                metrics.LogarithmicSizedBucketsFor(0, 1000),
+		BucketMaker:           metrics.AccumulatingBucketBuilder(metrics.EvictAndReplaceWith(10, maths.Average), 100),
+		ReportablePercentiles: []float64{0.01, 0.05, 0.5, 0.90, 0.99},
+	}
+
+	targetsHealthy   = &metrics.CounterMetric{}
+	targetsUnhealthy = &metrics.CounterMetric{}
+
+	scrapeLatencyHealthy   = metrics.CreateHistogram(networkLatencyHistogram)
+	scrapeLatencyUnhealthy = metrics.CreateHistogram(networkLatencyHistogram)
+)
+
+func init() {
+	registry.Register("targets_healthy_total", targetsHealthy)
+	registry.Register("targets_unhealthy_total", targetsUnhealthy)
+	registry.Register("targets_healthy_scrape_latency_ms", scrapeLatencyHealthy)
+	registry.Register("targets_unhealthy_scrape_latency_ms", scrapeLatencyUnhealthy)
+}

retrieval/target.go

@@ -15,6 +15,7 @@ package retrieval
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/matttproud/golang_instrumentation/metrics"
 	"github.com/matttproud/prometheus/model"
 	"io/ioutil"
 	"log"
@@ -62,6 +63,7 @@ func (t *Target) reschedule(s TargetState) {
 		switch s {
 		case ALIVE:
 			t.unreachableCount = 0
+			targetsHealthy.Increment()
 		case UNREACHABLE:
 			backoff := MAXIMUM_BACKOFF
 			exponential := time.Duration(math.Pow(2, float64(t.unreachableCount))) * time.Second
@@ -71,7 +73,6 @@ func (t *Target) reschedule(s TargetState) {
 
 			t.scheduledFor = time.Now().Add(backoff)
 			t.unreachableCount++
-
 			log.Printf("%s unavailable %s times deferred for %s.", t, t.unreachableCount, backoff)
 		default:
 		}
@@ -79,6 +80,7 @@ func (t *Target) reschedule(s TargetState) {
 		switch s {
 		case UNREACHABLE:
 			t.unreachableCount++
+			targetsUnhealthy.Increment()
 		}
 	default:
 	}
@@ -111,7 +113,7 @@ func (t *Target) Scrape(results chan Result) (err error) {
 
 	done := make(chan bool)
 
-	go func() {
+	request := func() {
 		ti := time.Now()
 		resp, err := http.Get(t.Address)
 		if err != nil {
@@ -197,7 +199,18 @@ func (t *Target) Scrape(results chan Result) (err error) {
 		}
 
 		done <- true
-	}()
+	}
+
+	accumulator := func(d time.Duration) {
+		ms := float64(d) / float64(time.Millisecond)
+		if err == nil {
+			scrapeLatencyHealthy.Add(ms)
+		} else {
+			scrapeLatencyUnhealthy.Add(ms)
+		}
+	}
+
+	go metrics.InstrumentCall(request, accumulator)
 
 	select {
 	case <-done:

storage/metric/leveldb/instrumentation.go

@@ -0,0 +1,36 @@
+// Copyright 2012 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package leveldb
+
+import (
+	"github.com/matttproud/golang_instrumentation"
+	"github.com/matttproud/golang_instrumentation/maths"
+	"github.com/matttproud/golang_instrumentation/metrics"
+)
+
+var (
+	diskLatencyHistogram = &metrics.HistogramSpecification{
+		Starts:                metrics.LogarithmicSizedBucketsFor(0, 5000),
+		BucketMaker:           metrics.AccumulatingBucketBuilder(metrics.EvictAndReplaceWith(10, maths.Average), 100),
+		ReportablePercentiles: []float64{0.01, 0.05, 0.5, 0.90, 0.99},
+	}
+
+	targetsHealthy = &metrics.CounterMetric{}
+
+	appendLatency = metrics.CreateHistogram(diskLatencyHistogram)
+)
+
+func init() {
+	registry.Register("sample_append_disk_latency_microseconds", appendLatency)
+}

storage/metric/leveldb/mutable.go

@@ -21,6 +21,7 @@ import (
 	"github.com/matttproud/prometheus/coding/indexable"
 	"github.com/matttproud/prometheus/model"
 	dto "github.com/matttproud/prometheus/model/generated"
+	"time"
 )
 
 var (
@@ -149,44 +150,54 @@ func (l *LevelDBMetricPersistence) AppendSample(sample *model.Sample) (err error
 		m.Increment()
 	}()
 
-	metricDTO := model.SampleToMetricDTO(sample)
+	operation := func() {
+		metricDTO := model.SampleToMetricDTO(sample)
 
-	indexHas, err := l.hasIndexMetric(metricDTO)
+		indexHas, err := l.hasIndexMetric(metricDTO)
-	if err != nil {
-		return
-	}
-
-	if !indexHas {
-		err = l.indexMetric(metricDTO)
 		if err != nil {
 			return
 		}
 
-		err = l.appendFingerprints(metricDTO)
+		if !indexHas {
+			err = l.indexMetric(metricDTO)
+			if err != nil {
+				return
+			}
+
+			err = l.appendFingerprints(metricDTO)
+			if err != nil {
+				return
+			}
+		}
+
+		fingerprintDTO, err := model.MessageToFingerprintDTO(metricDTO)
+		if err != nil {
+			return
+		}
+
+		sampleKeyDTO := &dto.SampleKey{
+			Fingerprint: fingerprintDTO,
+			Timestamp:   indexable.EncodeTime(sample.Timestamp),
+		}
+		sampleValueDTO := &dto.SampleValue{
+			Value: proto.Float32(float32(sample.Value)),
+		}
+		sampleKeyEncoded := coding.NewProtocolBufferEncoder(sampleKeyDTO)
+		sampleValueEncoded := coding.NewProtocolBufferEncoder(sampleValueDTO)
+
+		err = l.metricSamples.Put(sampleKeyEncoded, sampleValueEncoded)
 		if err != nil {
 			return
 		}
 	}
 
-	fingerprintDTO, err := model.MessageToFingerprintDTO(metricDTO)
+	// XXX: Problematic with panics.
-	if err != nil {
+	accumulator := func(d time.Duration) {
-		return
+		ms := float64(d) / float64(time.Microsecond)
+		appendLatency.Add(ms)
 	}
 
-	sampleKeyDTO := &dto.SampleKey{
+	metrics.InstrumentCall(operation, accumulator)
-		Fingerprint: fingerprintDTO,
-		Timestamp:   indexable.EncodeTime(sample.Timestamp),
-	}
-	sampleValueDTO := &dto.SampleValue{
-		Value: proto.Float32(float32(sample.Value)),
-	}
-	sampleKeyEncoded := coding.NewProtocolBufferEncoder(sampleKeyDTO)
-	sampleValueEncoded := coding.NewProtocolBufferEncoder(sampleValueDTO)
-
-	err = l.metricSamples.Put(sampleKeyEncoded, sampleValueEncoded)
-	if err != nil {
-		return
-	}
 
 	return
 }