diff --git a/storage/local/instrumentation.go b/storage/local/instrumentation.go index 2febf71b8..87aa492eb 100644 --- a/storage/local/instrumentation.go +++ b/storage/local/instrumentation.go @@ -36,6 +36,14 @@ var ( Name: "prometheus_stored_samples_total", Help: "The total number of stored samples.", }) + evictionDuration = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "prometheus_memory_eviction_duration_milliseconds", + Help: "The duration of the last memory eviction iteration in milliseconds.", + }) + purgeDuration = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "prometheus_storage_purge_duration_milliseconds", + Help: "The duration of the last storage purge iteration in milliseconds.", + }) numChunks = prometheus.NewGauge(prometheus.GaugeOpts{ Name: "prometheus_used_chunks_count", @@ -59,7 +67,7 @@ var ( }) persistLatencies = prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Name: "prometheus_persist_latency_ms", + Name: "prometheus_persist_latency_milliseconds", Help: "A summary of latencies for persisting each chunk.", }, []string{outcome}) persistQueueLength = prometheus.NewGauge(prometheus.GaugeOpts{ @@ -75,6 +83,8 @@ var ( func init() { prometheus.MustRegister(numSeries) prometheus.MustRegister(numSamples) + prometheus.MustRegister(evictionDuration) + prometheus.MustRegister(purgeDuration) prometheus.MustRegister(numChunks) prometheus.MustRegister(numChunkGives) prometheus.MustRegister(numChunkGets) diff --git a/storage/local/storage.go b/storage/local/storage.go index e8ea220e9..47f2345c4 100644 --- a/storage/local/storage.go +++ b/storage/local/storage.go @@ -204,6 +204,10 @@ func (s *memorySeriesStorage) evictMemoryChunks(ttl time.Duration) { s.mtx.RLock() defer s.mtx.RUnlock() + defer func(begin time.Time) { + evictionDuration.Set(float64(time.Since(begin) / time.Millisecond)) + }(time.Now()) + for fp, series := range s.fingerprintToSeries { if series.evictOlderThan(clientmodel.TimestampFromTime(time.Now()).Add(-1 * ttl)) { if err := s.persistence.ArchiveMetric( @@ -294,6 +298,9 @@ func (s *memorySeriesStorage) purgePeriodically(stop <-chan bool) { case <-purgeTicker.C: glog.Info("Purging old series data...") s.mtx.RLock() + + begin := time.Now() + fps := make([]clientmodel.Fingerprint, 0, len(s.fingerprintToSeries)) for fp := range s.fingerprintToSeries { fps = append(fps, fp) @@ -324,6 +331,7 @@ func (s *memorySeriesStorage) purgePeriodically(stop <-chan bool) { s.purgeSeries(fp, ts) } } + purgeDuration.Set(float64(time.Since(begin) / time.Millisecond)) glog.Info("Done purging old series data.") } }