From db92620163930a4ef85c39c199ca77a002647140 Mon Sep 17 00:00:00 2001
From: Julius Volz <julius@soundcloud.com>
Date: Mon, 6 Oct 2014 00:00:40 +0200
Subject: [PATCH] Instrument eviction and purge durations.

Change-Id: Ia5b2319363ad2644674c9b7a94162a89bcc296fb
---
 storage/local/instrumentation.go | 12 +++++++++++-
 storage/local/storage.go         |  8 ++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/storage/local/instrumentation.go b/storage/local/instrumentation.go
index 2febf71b8..87aa492eb 100644
--- a/storage/local/instrumentation.go
+++ b/storage/local/instrumentation.go
@@ -36,6 +36,14 @@ var (
 		Name: "prometheus_stored_samples_total",
 		Help: "The total number of stored samples.",
 	})
+	evictionDuration = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "prometheus_memory_eviction_duration_milliseconds",
+		Help: "The duration of the last memory eviction iteration in milliseconds.",
+	})
+	purgeDuration = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "prometheus_storage_purge_duration_milliseconds",
+		Help: "The duration of the last storage purge iteration in milliseconds.",
+	})
 
 	numChunks = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name: "prometheus_used_chunks_count",
@@ -59,7 +67,7 @@ var (
 	})
 
 	persistLatencies = prometheus.NewSummaryVec(prometheus.SummaryOpts{
-		Name: "prometheus_persist_latency_ms",
+		Name: "prometheus_persist_latency_milliseconds",
 		Help: "A summary of latencies for persisting each chunk.",
 	}, []string{outcome})
 	persistQueueLength = prometheus.NewGauge(prometheus.GaugeOpts{
@@ -75,6 +83,8 @@ var (
 func init() {
 	prometheus.MustRegister(numSeries)
 	prometheus.MustRegister(numSamples)
+	prometheus.MustRegister(evictionDuration)
+	prometheus.MustRegister(purgeDuration)
 	prometheus.MustRegister(numChunks)
 	prometheus.MustRegister(numChunkGives)
 	prometheus.MustRegister(numChunkGets)
diff --git a/storage/local/storage.go b/storage/local/storage.go
index e8ea220e9..47f2345c4 100644
--- a/storage/local/storage.go
+++ b/storage/local/storage.go
@@ -204,6 +204,10 @@ func (s *memorySeriesStorage) evictMemoryChunks(ttl time.Duration) {
 	s.mtx.RLock()
 	defer s.mtx.RUnlock()
 
+	defer func(begin time.Time) {
+		evictionDuration.Set(float64(time.Since(begin) / time.Millisecond))
+	}(time.Now())
+
 	for fp, series := range s.fingerprintToSeries {
 		if series.evictOlderThan(clientmodel.TimestampFromTime(time.Now()).Add(-1 * ttl)) {
 			if err := s.persistence.ArchiveMetric(
@@ -294,6 +298,9 @@ func (s *memorySeriesStorage) purgePeriodically(stop <-chan bool) {
 		case <-purgeTicker.C:
 			glog.Info("Purging old series data...")
 			s.mtx.RLock()
+
+			begin := time.Now()
+
 			fps := make([]clientmodel.Fingerprint, 0, len(s.fingerprintToSeries))
 			for fp := range s.fingerprintToSeries {
 				fps = append(fps, fp)
@@ -324,6 +331,7 @@ func (s *memorySeriesStorage) purgePeriodically(stop <-chan bool) {
 					s.purgeSeries(fp, ts)
 				}
 			}
+			purgeDuration.Set(float64(time.Since(begin) / time.Millisecond))
 			glog.Info("Done purging old series data.")
 		}
 	}