mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2025-03-05 20:59:13 -08:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

storage: Increment s.persistErrors on all persist errors

Browse source 
Fixes #2091
This commit is contained in:
beorn7 2017-04-06 01:36:29 +02:00
parent 5f764d9940
commit ed5f68f382
1 changed files with 9 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
storage/local/storage.go
View file

@ -231,7 +231,7 @@ func NewMemorySeriesStorage(o *MemorySeriesStorageOptions) *MemorySeriesStorage
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "persist_errors_total", Name: "persist_errors_total",
Help: "The total number of errors while persisting chunks.", Help: "The total number of errors while writing to the persistence layer.",
}), }),
queuedChunksToPersist: prometheus.NewCounter(prometheus.CounterOpts{ queuedChunksToPersist: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: namespace, Namespace: namespace,
@ -1449,6 +1449,7 @@ func (s *MemorySeriesStorage) loop() {
s.dirtySeries.Set(0) s.dirtySeries.Set(0)
err := s.persistence.checkpointSeriesMapAndHeads(s.fpToSeries, s.fpLocker) err := s.persistence.checkpointSeriesMapAndHeads(s.fpToSeries, s.fpLocker)
if err != nil { if err != nil {
s.persistErrors.Inc()
log.Errorln("Error while checkpointing:", err) log.Errorln("Error while checkpointing:", err)
} }
// If a checkpoint takes longer than checkpointInterval, unluckily timed // If a checkpoint takes longer than checkpointInterval, unluckily timed
@ -1713,14 +1714,20 @@ func (s *MemorySeriesStorage) maintainArchivedSeries(fp model.Fingerprint, befor
newFirstTime, _, _, allDropped, err := s.persistence.dropAndPersistChunks(fp, beforeTime, nil) newFirstTime, _, _, allDropped, err := s.persistence.dropAndPersistChunks(fp, beforeTime, nil)
if err != nil { if err != nil {
// TODO(beorn7): Should quarantine the series.
s.persistErrors.Inc()
log.Error("Error dropping persisted chunks: ", err) log.Error("Error dropping persisted chunks: ", err)
} }
if allDropped { if allDropped {
s.persistence.purgeArchivedMetric(fp) // Ignoring error. Nothing we can do. if err := s.persistence.purgeArchivedMetric(fp); err != nil {
s.persistErrors.Inc()
// purgeArchivedMetric logs the error already.
}
s.seriesOps.WithLabelValues(archivePurge).Inc() s.seriesOps.WithLabelValues(archivePurge).Inc()
return return
} }
if err := s.persistence.updateArchivedTimeRange(fp, newFirstTime, lastTime); err != nil { if err := s.persistence.updateArchivedTimeRange(fp, newFirstTime, lastTime); err != nil {
s.persistErrors.Inc()
log.Errorf("Error updating archived time range for fingerprint %v: %s", fp, err) log.Errorf("Error updating archived time range for fingerprint %v: %s", fp, err)
} }
} }