Various fixes to locking & shutdown for WAL-based remote write.

- Remove datarace in the exported highest scrape timestamp.
- Backoff on enqueue should be per-sample - reset the result for each sample.
- Remove diffKeys, unused ctx and cancelfunc in WALWatcher, 'name' from writeTo interface, and pass it to constructor.
- Reorder functions in WALWatcher depth-first according to call graph.
- Fix vendor/modules.txt.
- Split out the various timer periods into consts at the top of the file.
- Move w.currentSegmentMetric.Set close to where we set the currentSegment.
- Combine r.Next() and isClosed(w.quit) into a single loop.
- Unnest some ifs in WALWatcher.watch, propagate erros in decodeRecord, add some new lines to make it easier to read.
- Reorganise checkpoint handling to reduce nesting and make it easier to follow.

Signed-off-by: Tom Wilkie <tom.wilkie@gmail.com>

storage/remote/queue_manager.go

@@ -215,9 +215,8 @@ type QueueManager struct {
 	shards      *shards
 	numShards   int
 	reshardChan chan int
-
+	quit        chan struct{}
-	quit chan struct{}
+	wg          sync.WaitGroup
-	wg   sync.WaitGroup
 
 	samplesIn, samplesOut, samplesOutDuration *ewmaRate
 	integralAccumulator                       float64
@@ -259,7 +258,7 @@ func NewQueueManager(logger log.Logger, walDir string, samplesIn *ewmaRate, high
 	t.highestSentTimestampMetric = queueHighestSentTimestamp.WithLabelValues(t.queueName)
 	t.pendingSamplesMetric = queuePendingSamples.WithLabelValues(t.queueName)
 	t.enqueueRetriesMetric = enqueueRetriesTotal.WithLabelValues(t.queueName)
-	t.watcher = NewWALWatcher(logger, t, walDir, startTime)
+	t.watcher = NewWALWatcher(logger, client.Name(), t, walDir, startTime)
 	t.shards = t.newShards()
 
 	numShards.WithLabelValues(t.queueName).Set(float64(t.numShards))
@@ -310,11 +309,10 @@ func (t *QueueManager) Append(s []tsdb.RefSample) bool {
 	}
 	t.seriesMtx.Unlock()
 
-	backoff := t.cfg.MinBackoff
 outer:
 	for _, sample := range tempSamples {
-		// This will result in spin/busy waiting if the queues are being resharded
+		// This will only loop if the queues are being resharded.
-		// or shutting down.  TODO backoff.
+		backoff := t.cfg.MinBackoff
 		for {
 			select {
 			case <-t.quit:
@@ -325,6 +323,7 @@ outer:
 			if t.shards.enqueue(sample.ref, sample.ts) {
 				continue outer
 			}
+
 			t.enqueueRetriesMetric.Inc()
 			time.Sleep(time.Duration(backoff))
 			backoff = backoff * 2
@@ -359,18 +358,6 @@ func (t *QueueManager) Stop() {
 	t.wg.Wait()
 }
 
-func (t *QueueManager) Name() string {
-	return t.queueName
-}
-
-// Find out which series are dropped after relabelling and make sure we have a metric label for them.
-func (t *QueueManager) diffKeys(ref uint64, original, relabelled model.LabelSet) {
-	numDropped := len(original) - len(relabelled)
-	if numDropped == 0 {
-		return
-	}
-}
-
 // StoreSeries keeps track of which series we know about for lookups when sending samples to remote.
 func (t *QueueManager) StoreSeries(series []tsdb.RefSeries, index int) {
 	temp := make(map[uint64][]prompb.Label, len(series))
@@ -381,8 +368,6 @@ func (t *QueueManager) StoreSeries(series []tsdb.RefSeries, index int) {
 		}
 		t.processExternalLabels(ls)
 		rl := relabel.Process(ls, t.relabelConfigs...)
-
-		t.diffKeys(s.Ref, ls, rl)
 		if len(rl) == 0 {
 			t.droppedSeries[s.Ref] = struct{}{}
 			continue

storage/remote/storage.go

@@ -42,6 +42,7 @@ type Storage struct {
 	queues                 []*QueueManager
 	samplesIn              *ewmaRate
 	samplesInMetric        prometheus.Counter
+	highestTimestampMtx    sync.Mutex
 	highestTimestamp       int64
 	highestTimestampMetric prometheus.Gauge
 

storage/remote/wal_watcher.go

@@ -14,9 +14,7 @@
 package remote
 
 import (
-	"context"
 	"fmt"
-	"io"
 	"os"
 	"path"
 	"strconv"
@@ -33,6 +31,12 @@ import (
 	"github.com/prometheus/tsdb/wal"
 )
 
+const (
+	readPeriod         = 10 * time.Millisecond
+	checkpointPeriod   = 5 * time.Second
+	segmentCheckPeriod = 100 * time.Millisecond
+)
+
 var (
 	watcherSamplesRecordsRead = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
@@ -123,14 +127,15 @@ type writeTo interface {
 	Append([]tsdb.RefSample) bool
 	StoreSeries([]tsdb.RefSeries, int)
 	SeriesReset(int)
-	Name() string
 }
 
 // WALWatcher watches the TSDB WAL for a given WriteTo.
 type WALWatcher struct {
-	writer         writeTo
+	name   string
-	logger         log.Logger
+	writer writeTo
-	walDir         string
+	logger log.Logger
+	walDir string
+
 	currentSegment int
 	lastCheckpoint string
 	startTime      int64
@@ -144,298 +149,44 @@ type WALWatcher struct {
 	samplesSentPreTailing   prometheus.Counter
 	currentSegmentMetric    prometheus.Gauge
 
-	ctx    context.Context
+	quit chan struct{}
-	cancel context.CancelFunc
-	quit   chan struct{}
 }
 
 // NewWALWatcher creates a new WAL watcher for a given WriteTo.
-func NewWALWatcher(logger log.Logger, writer writeTo, walDir string, startTime int64) *WALWatcher {
+func NewWALWatcher(logger log.Logger, name string, writer writeTo, walDir string, startTime int64) *WALWatcher {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
-	ctx, cancel := context.WithCancel(context.Background())
 	w := &WALWatcher{
 		logger:    logger,
 		writer:    writer,
 		walDir:    path.Join(walDir, "wal"),
 		startTime: startTime,
-		ctx:       ctx,
-		cancel:    cancel,
 		quit:      make(chan struct{}),
 	}
 
-	w.samplesReadMetric = watcherSamplesRecordsRead.WithLabelValues(w.writer.Name())
+	w.samplesReadMetric = watcherSamplesRecordsRead.WithLabelValues(name)
-	w.seriesReadMetric = watcherSeriesRecordsRead.WithLabelValues(w.writer.Name())
+	w.seriesReadMetric = watcherSeriesRecordsRead.WithLabelValues(name)
-	w.tombstonesReadMetric = watcherTombstoneRecordsRead.WithLabelValues(w.writer.Name())
+	w.tombstonesReadMetric = watcherTombstoneRecordsRead.WithLabelValues(name)
-	w.unknownReadMetric = watcherUnknownTypeRecordsRead.WithLabelValues(w.writer.Name())
+	w.unknownReadMetric = watcherUnknownTypeRecordsRead.WithLabelValues(name)
-	w.invalidReadMetric = watcherInvalidRecordsRead.WithLabelValues(w.writer.Name())
+	w.invalidReadMetric = watcherInvalidRecordsRead.WithLabelValues(name)
-	w.recordDecodeFailsMetric = watcherRecordDecodeFails.WithLabelValues(w.writer.Name())
+	w.recordDecodeFailsMetric = watcherRecordDecodeFails.WithLabelValues(name)
-	w.samplesSentPreTailing = watcherSamplesSentPreTailing.WithLabelValues(w.writer.Name())
+	w.samplesSentPreTailing = watcherSamplesSentPreTailing.WithLabelValues(name)
-	w.currentSegmentMetric = watcherCurrentSegment.WithLabelValues(w.writer.Name())
+	w.currentSegmentMetric = watcherCurrentSegment.WithLabelValues(name)
 
 	return w
 }
 
 func (w *WALWatcher) Start() {
-	level.Info(w.logger).Log("msg", "starting WAL watcher", "queue", w.writer.Name())
+	level.Info(w.logger).Log("msg", "starting WAL watcher", "queue", w.name)
 	go w.runWatcher()
 }
 
 func (w *WALWatcher) Stop() {
-	level.Info(w.logger).Log("msg", "stopping WAL watcher", "queue", w.writer.Name())
+	level.Info(w.logger).Log("msg", "stopping WAL watcher", "queue", w.name)
 	close(w.quit)
 }
 
-// TODO: fix the exit logic for this function
-// The stop param is used to stop at the end of the existing WAL on startup,
-// since scraped samples may be written to the latest segment before we finish reading it.
-func (w *WALWatcher) readSeriesRecords(r *wal.LiveReader, index int, stop int64) {
-	var (
-		dec     tsdb.RecordDecoder
-		series  []tsdb.RefSeries
-		samples []tsdb.RefSample
-		ret     bool
-	)
-	for !isClosed(w.quit) {
-		for r.Next() {
-			series = series[:0]
-			rec := r.Record()
-			// If the timestamp is > start then we should Append this sample and exit readSeriesRecords,
-			// because this is the first sample written to the WAL after the WAL watcher was started.
-			typ := dec.Type(rec)
-			if typ == tsdb.RecordSamples {
-				samples, err := dec.Samples(rec, samples[:0])
-				if err != nil {
-					continue
-				}
-				for _, s := range samples {
-					if s.T > w.startTime {
-						w.writer.Append(samples)
-						ret = true
-						w.samplesSentPreTailing.Inc()
-					}
-				}
-				if ret {
-					level.Info(w.logger).Log("msg", "found a sample with a timestamp after the WAL watcher start")
-					level.Info(w.logger).Log("msg", "read all series records in segment/checkpoint", "index", index)
-					return
-				}
-			}
-			if typ != tsdb.RecordSeries {
-				continue
-			}
-
-			series, err := dec.Series(rec, nil)
-			if err != nil {
-				level.Error(log.With(w.logger)).Log("err", err)
-				break
-			}
-
-			w.writer.StoreSeries(series, index)
-		}
-		// Since we only call readSeriesRecords on fully written WAL segments or checkpoints,
-		// Error() will only return an error if something actually went wrong when reading
-		// a record, either it was invalid or it was only partially written to the WAL.
-		if err := r.Err(); err != nil {
-			level.Error(w.logger).Log("err", err)
-			return
-		}
-		// Ensure we read all of the bytes in the segment or checkpoint.
-		if r.TotalRead() >= stop {
-			level.Info(w.logger).Log("msg", "read all series records in segment/checkpoint", "index", index)
-			return
-		}
-	}
-}
-
-// Read all the series records from a Checkpoint directory.
-func (w *WALWatcher) readCheckpoint(checkpointDir string) error {
-	sr, err := wal.NewSegmentsReader(checkpointDir)
-	if err != nil {
-		return errors.Wrap(err, "open checkpoint")
-	}
-	defer sr.Close()
-
-	split := strings.Split(checkpointDir, ".")
-	if len(split) != 2 {
-		return errors.Errorf("checkpoint dir name is not in the right format: %s", checkpointDir)
-	}
-
-	i, err := strconv.Atoi(split[1])
-	if err != nil {
-		i = w.currentSegment - 1
-	}
-
-	size, err := getCheckpointSize(checkpointDir)
-	if err != nil {
-		level.Error(w.logger).Log("msg", "error getting checkpoint size", "checkpoint", checkpointDir)
-	}
-
-	w.readSeriesRecords(wal.NewLiveReader(sr), i, size)
-	level.Debug(w.logger).Log("msg", "read series references from checkpoint", "checkpoint", checkpointDir)
-	w.writer.SeriesReset(i)
-	return nil
-}
-
-// When starting the WAL watcher, there is potentially an existing WAL. In that case, we
-// should read to the end of the newest existing segment before reading new records that
-// are written to it, storing data from series records along the way.
-// Unfortunately this function is duplicates some of TSDB Head.Init().
-func (w *WALWatcher) readToEnd(walDir string, firstSegment, lastSegment int) (*wal.Segment, *wal.LiveReader, error) {
-	// Backfill from the checkpoint first if it exists.
-	defer level.Debug(w.logger).Log("msg", "done reading existing WAL")
-	dir, startFrom, err := tsdb.LastCheckpoint(walDir)
-	if err != nil && err != tsdb.ErrNotFound {
-		return nil, nil, errors.Wrap(err, "find last checkpoint")
-	}
-	level.Debug(w.logger).Log("msg", "reading checkpoint", "dir", dir)
-	if err == nil {
-		w.lastCheckpoint = dir
-		err = w.readCheckpoint(dir)
-		if err != nil {
-			return nil, nil, err
-		}
-		startFrom++
-	}
-
-	// Backfill segments from the last checkpoint onwards if at least 2 segments exist.
-	if lastSegment > 0 {
-		for i := firstSegment; i < lastSegment; i++ {
-			seg, err := wal.OpenReadSegment(wal.SegmentName(walDir, i))
-			if err != nil {
-				return nil, nil, err
-			}
-			sz, _ := getSegmentSize(walDir, i)
-			w.readSeriesRecords(wal.NewLiveReader(seg), i, sz)
-		}
-	}
-
-	// We want to start the WAL Watcher from the end of the last segment on start,
-	// so we make sure to return the wal.Segment pointer
-	segment, err := wal.OpenReadSegment(wal.SegmentName(w.walDir, lastSegment))
-	if err != nil {
-		return nil, nil, err
-	}
-
-	r := wal.NewLiveReader(segment)
-	sz, _ := getSegmentSize(walDir, lastSegment)
-	w.readSeriesRecords(r, lastSegment, sz)
-	return segment, r, nil
-}
-
-func (w *WALWatcher) decodeRecord(rec []byte) error {
-	var (
-		dec     tsdb.RecordDecoder
-		series  []tsdb.RefSeries
-		samples []tsdb.RefSample
-	)
-	switch dec.Type(rec) {
-	case tsdb.RecordSeries:
-		series, err := dec.Series(rec, series[:0])
-		if err != nil {
-			w.recordDecodeFailsMetric.Inc()
-			level.Error(log.With(w.logger)).Log("err", err)
-			break
-		}
-		w.seriesReadMetric.Add(float64(len(series)))
-		w.writer.StoreSeries(series, w.currentSegment)
-	case tsdb.RecordSamples:
-		samples, err := dec.Samples(rec, samples[:0])
-		if err != nil {
-			w.recordDecodeFailsMetric.Inc()
-			level.Error(log.With(w.logger)).Log("err", err)
-			break
-		}
-		w.samplesReadMetric.Add(float64(len(samples)))
-		// Blocks  until the sample is sent to all remote write endpoints or closed (because enqueue blocks).
-		w.writer.Append(samples)
-	case tsdb.RecordTombstones:
-		w.tombstonesReadMetric.Add(float64(len(samples)))
-	case tsdb.RecordInvalid:
-		w.invalidReadMetric.Add(float64(len(samples)))
-		return errors.New("invalid record")
-	default:
-		level.Info(w.logger).Log("msg", "unknown TSDB record type in decodeSegment")
-		return errors.New("unknown TSDB record type")
-	}
-	return nil
-}
-
-func (w *WALWatcher) readSegment(r *wal.LiveReader) {
-	for r.Next() && !isClosed(w.quit) {
-		err := w.decodeRecord(r.Record())
-		if err != nil {
-			level.Error(w.logger).Log("err", err)
-		}
-	}
-	if err := r.Err(); err != nil && err != io.EOF {
-		level.Error(w.logger).Log("err", err)
-	}
-}
-
-func (w *WALWatcher) watch(wl *wal.WAL, reader *wal.LiveReader) error {
-	readTimeout := 10 * time.Millisecond
-	readTicker := time.NewTicker(readTimeout)
-	defer readTicker.Stop()
-	checkpointTicker := time.NewTicker(5 * time.Second)
-	defer checkpointTicker.Stop()
-	segmentTicker := time.NewTicker(100 * time.Millisecond)
-	defer segmentTicker.Stop()
-
-	currentSegmentName := fmt.Sprintf("%08d", w.currentSegment)
-	w.currentSegmentMetric.Set(float64(w.currentSegment))
-
-	for {
-		select {
-		case <-w.quit:
-			level.Info(w.logger).Log("msg", "quitting WAL watcher watch loop")
-			return errors.New("quit channel")
-		case <-checkpointTicker.C:
-			// check if there is a new checkpoint
-			dir, _, err := tsdb.LastCheckpoint(w.walDir)
-			if err != nil && err != tsdb.ErrNotFound {
-				continue
-			}
-			cn, err := checkpointNum(dir)
-			if err != nil {
-				continue
-			}
-			// TODO: callum, simplify the nesting here
-			if err == nil && dir != w.lastCheckpoint {
-				level.Info(w.logger).Log("msg", "new checkpoint detected", "last", w.lastCheckpoint, "new", dir)
-				d, err := strconv.Atoi(cn)
-				if err != nil {
-					level.Error(w.logger).Log("err", err)
-				} else if d < w.currentSegment {
-					w.lastCheckpoint = dir
-					// This potentially takes a long time, should we run it in another go routine?
-					err = w.readCheckpoint(w.lastCheckpoint)
-					if err != nil {
-						level.Error(w.logger).Log("err", err)
-					}
-				} else {
-					level.Info(w.logger).Log("msg", "current segment is behind the checkpoint, skipping reading of checkpoint for now", "current", currentSegmentName, "checkpoint", dir)
-				}
-			}
-		case <-segmentTicker.C:
-			// check if new segments exist
-			_, last, err := wl.Segments()
-			if err != nil {
-				level.Error(w.logger).Log("err", err)
-				continue
-			}
-			if last > w.currentSegment {
-				w.readSegment(reader)
-				level.Info(w.logger).Log("msg", "a new segment exists, we should start reading it", "current", currentSegmentName, "new", fmt.Sprintf("%08d", last))
-				return nil
-			}
-		case <-readTicker.C:
-			w.readSegment(reader)
-		}
-	}
-}
-
 func (w *WALWatcher) runWatcher() {
 	// The WAL dir may not exist when Prometheus first starts up.
 	for {
@@ -472,14 +223,11 @@ func (w *WALWatcher) runWatcher() {
 	}
 
 	w.currentSegment = last
-
+	w.currentSegmentMetric.Set(float64(w.currentSegment))
-	if err != nil {
-		level.Error(w.logger).Log("err", err)
-		return
-	}
 
 	for {
 		level.Info(w.logger).Log("msg", "watching segment", "segment", w.currentSegment)
+
 		// On start, after reading the existing WAL for series records, we have a pointer to what is the latest segment.
 		// On subsequent calls to this function, currentSegment will have been incremented and we should open that segment.
 		err := w.watch(nw, reader)
@@ -488,7 +236,10 @@ func (w *WALWatcher) runWatcher() {
 			level.Error(w.logger).Log("msg", "runWatcher is ending", "err", err)
 			return
 		}
+
 		w.currentSegment++
+		w.currentSegmentMetric.Set(float64(w.currentSegment))
+
 		segment, err = wal.OpenReadSegment(wal.SegmentName(w.walDir, w.currentSegment))
 		reader = wal.NewLiveReader(segment)
 		// TODO: callum, is this error really fatal?
@@ -499,13 +250,292 @@ func (w *WALWatcher) runWatcher() {
 	}
 }
 
-func checkpointNum(dir string) (string, error) {
+// When starting the WAL watcher, there is potentially an existing WAL. In that case, we
+// should read to the end of the newest existing segment before reading new records that
+// are written to it, storing data from series records along the way.
+// Unfortunately this function is duplicates some of TSDB Head.Init().
+func (w *WALWatcher) readToEnd(walDir string, firstSegment, lastSegment int) (*wal.Segment, *wal.LiveReader, error) {
+	// Backfill from the checkpoint first if it exists.
+	defer level.Debug(w.logger).Log("msg", "done reading existing WAL")
+	dir, startFrom, err := tsdb.LastCheckpoint(walDir)
+	if err != nil && err != tsdb.ErrNotFound {
+		return nil, nil, errors.Wrap(err, "find last checkpoint")
+	}
+
+	level.Debug(w.logger).Log("msg", "reading checkpoint", "dir", dir)
+	if err == nil {
+		w.lastCheckpoint = dir
+		err = w.readCheckpoint(dir)
+		if err != nil {
+			return nil, nil, err
+		}
+		startFrom++
+	}
+
+	// Backfill segments from the last checkpoint onwards if at least 2 segments exist.
+	if lastSegment > 0 {
+		for i := firstSegment; i < lastSegment; i++ {
+			seg, err := wal.OpenReadSegment(wal.SegmentName(walDir, i))
+			if err != nil {
+				return nil, nil, err
+			}
+			sz, _ := getSegmentSize(walDir, i)
+			w.readSeriesRecords(wal.NewLiveReader(seg), i, sz)
+		}
+	}
+
+	// We want to start the WAL Watcher from the end of the last segment on start,
+	// so we make sure to return the wal.Segment pointer
+	segment, err := wal.OpenReadSegment(wal.SegmentName(w.walDir, lastSegment))
+	if err != nil {
+		return nil, nil, err
+	}
+
+	r := wal.NewLiveReader(segment)
+	sz, _ := getSegmentSize(walDir, lastSegment)
+	w.readSeriesRecords(r, lastSegment, sz)
+	return segment, r, nil
+}
+
+// TODO: fix the exit logic for this function
+// The stop param is used to stop at the end of the existing WAL on startup,
+// since scraped samples may be written to the latest segment before we finish reading it.
+func (w *WALWatcher) readSeriesRecords(r *wal.LiveReader, index int, stop int64) {
+	var (
+		dec     tsdb.RecordDecoder
+		series  []tsdb.RefSeries
+		samples []tsdb.RefSample
+		ret     bool
+	)
+
+	for r.Next() && !isClosed(w.quit) {
+		series = series[:0]
+		rec := r.Record()
+		// If the timestamp is > start then we should Append this sample and exit readSeriesRecords,
+		// because this is the first sample written to the WAL after the WAL watcher was started.
+		typ := dec.Type(rec)
+		if typ == tsdb.RecordSamples {
+			samples, err := dec.Samples(rec, samples[:0])
+			if err != nil {
+				continue
+			}
+			for _, s := range samples {
+				if s.T > w.startTime {
+					w.writer.Append(samples)
+					ret = true
+					w.samplesSentPreTailing.Inc()
+				}
+			}
+			if ret {
+				level.Info(w.logger).Log("msg", "found a sample with a timestamp after the WAL watcher start")
+				level.Info(w.logger).Log("msg", "read all series records in segment/checkpoint", "index", index)
+				return
+			}
+		}
+		if typ != tsdb.RecordSeries {
+			continue
+		}
+
+		series, err := dec.Series(rec, nil)
+		if err != nil {
+			level.Error(log.With(w.logger)).Log("err", err)
+			break
+		}
+
+		w.writer.StoreSeries(series, index)
+	}
+
+	// Since we only call readSeriesRecords on fully written WAL segments or checkpoints,
+	// Error() will only return an error if something actually went wrong when reading
+	// a record, either it was invalid or it was only partially written to the WAL.
+	if err := r.Err(); err != nil {
+		level.Error(w.logger).Log("err", err)
+		return
+	}
+
+	// Ensure we read all of the bytes in the segment or checkpoint.
+	if r.TotalRead() >= stop {
+		level.Info(w.logger).Log("msg", "read all series records in segment/checkpoint", "index", index)
+		return
+	}
+}
+
+func (w *WALWatcher) watch(wl *wal.WAL, reader *wal.LiveReader) error {
+
+	readTicker := time.NewTicker(readPeriod)
+	defer readTicker.Stop()
+
+	checkpointTicker := time.NewTicker(checkpointPeriod)
+	defer checkpointTicker.Stop()
+
+	segmentTicker := time.NewTicker(segmentCheckPeriod)
+	defer segmentTicker.Stop()
+
+	for {
+		select {
+		case <-w.quit:
+			level.Info(w.logger).Log("msg", "quitting WAL watcher watch loop")
+			return errors.New("quit channel")
+
+		case <-checkpointTicker.C:
+			// Periodically check if there is a new checkpoint.
+			// As this is considered an optimisation, we ignore errors during
+			// checkpoint processing.
+
+			dir, _, err := tsdb.LastCheckpoint(w.walDir)
+			if err != nil && err != tsdb.ErrNotFound {
+				level.Error(w.logger).Log("msg", "error getting last checkpoint", "err", err)
+				continue
+			}
+
+			if dir == w.lastCheckpoint {
+				continue
+			}
+
+			level.Info(w.logger).Log("msg", "new checkpoint detected", "last", w.lastCheckpoint, "new", dir)
+
+			d, err := checkpointNum(dir)
+			if err != nil {
+				level.Error(w.logger).Log("msg", "error parsing checkpoint", "err", err)
+				continue
+			}
+
+			if d >= w.currentSegment {
+				level.Info(w.logger).Log("msg", "current segment is behind the checkpoint, skipping reading of checkpoint", "current", fmt.Sprintf("%08d", w.currentSegment), "checkpoint", dir)
+				continue
+			}
+
+			w.lastCheckpoint = dir
+			// This potentially takes a long time, should we run it in another go routine?
+			err = w.readCheckpoint(w.lastCheckpoint)
+			if err != nil {
+				level.Error(w.logger).Log("err", err)
+			}
+
+		case <-segmentTicker.C:
+			// check if new segments exist
+			_, last, err := wl.Segments()
+			if err != nil {
+				return errors.Wrap(err, "segments")
+			}
+
+			if last <= w.currentSegment {
+				continue
+			}
+
+			if err := w.readSegment(reader); err != nil {
+				// Ignore errors reading to end of segment, as we're going to move to
+				// next segment now.
+				level.Error(w.logger).Log("msg", "error reading to end of segment", "err", err)
+			}
+
+			level.Info(w.logger).Log("msg", "a new segment exists, we should start reading it", "current", fmt.Sprintf("%08d", w.currentSegment), "new", fmt.Sprintf("%08d", last))
+			return nil
+
+		case <-readTicker.C:
+			if err := w.readSegment(reader); err != nil {
+				level.Error(w.logger).Log("err", err)
+				return err
+			}
+		}
+	}
+}
+
+func (w *WALWatcher) readSegment(r *wal.LiveReader) error {
+	for r.Next() && !isClosed(w.quit) {
+		err := w.decodeRecord(r.Record())
+
+		// Intentionally skip over record decode errors.
+		if err != nil {
+			level.Error(w.logger).Log("err", err)
+		}
+	}
+	return r.Err()
+}
+
+func (w *WALWatcher) decodeRecord(rec []byte) error {
+	var (
+		dec     tsdb.RecordDecoder
+		series  []tsdb.RefSeries
+		samples []tsdb.RefSample
+	)
+	switch dec.Type(rec) {
+	case tsdb.RecordSeries:
+		series, err := dec.Series(rec, series[:0])
+		if err != nil {
+			w.recordDecodeFailsMetric.Inc()
+			return err
+		}
+		w.seriesReadMetric.Add(float64(len(series)))
+		w.writer.StoreSeries(series, w.currentSegment)
+
+	case tsdb.RecordSamples:
+		samples, err := dec.Samples(rec, samples[:0])
+		if err != nil {
+			w.recordDecodeFailsMetric.Inc()
+			return err
+		}
+		w.samplesReadMetric.Add(float64(len(samples)))
+		// Blocks  until the sample is sent to all remote write endpoints or closed (because enqueue blocks).
+		w.writer.Append(samples)
+
+	case tsdb.RecordTombstones:
+		w.tombstonesReadMetric.Add(float64(len(samples)))
+
+	case tsdb.RecordInvalid:
+		w.invalidReadMetric.Add(float64(len(samples)))
+		return errors.New("invalid record")
+
+	default:
+		w.recordDecodeFailsMetric.Inc()
+		return errors.New("unknown TSDB record type")
+	}
+	return nil
+}
+
+// Read all the series records from a Checkpoint directory.
+func (w *WALWatcher) readCheckpoint(checkpointDir string) error {
+	sr, err := wal.NewSegmentsReader(checkpointDir)
+	if err != nil {
+		return errors.Wrap(err, "open checkpoint")
+	}
+	defer sr.Close()
+
+	split := strings.Split(checkpointDir, ".")
+	if len(split) != 2 {
+		return errors.Errorf("checkpoint dir name is not in the right format: %s", checkpointDir)
+	}
+
+	i, err := strconv.Atoi(split[1])
+	if err != nil {
+		i = w.currentSegment - 1
+	}
+
+	size, err := getCheckpointSize(checkpointDir)
+	if err != nil {
+		level.Error(w.logger).Log("msg", "error getting checkpoint size", "checkpoint", checkpointDir)
+		return errors.Wrap(err, "get checkpoint size")
+	}
+
+	w.readSeriesRecords(wal.NewLiveReader(sr), i, size)
+	level.Debug(w.logger).Log("msg", "read series references from checkpoint", "checkpoint", checkpointDir)
+	w.writer.SeriesReset(i)
+	return nil
+}
+
+func checkpointNum(dir string) (int, error) {
 	// Checkpoint dir names are in the format checkpoint.000001
 	chunks := strings.Split(dir, ".")
 	if len(chunks) != 2 {
-		return "", errors.Errorf("invalid checkpoint dir string: %s", dir)
+		return 0, errors.Errorf("invalid checkpoint dir string: %s", dir)
 	}
-	return chunks[1], nil
+
+	result, err := strconv.Atoi(chunks[1])
+	if err != nil {
+		return 0, errors.Errorf("invalid checkpoint dir string: %s", dir)
+	}
+
+	return result, nil
 }
 
 func getCheckpointSize(dir string) (int64, error) {

storage/remote/wal_watcher_test.go

@@ -70,10 +70,6 @@ func (wtm *writeToMock) SeriesReset(index int) {
 	}
 }
 
-func (wtm *writeToMock) Name() string {
-	return ""
-}
-
 func newWriteToMock() *writeToMock {
 	return &writeToMock{
 		seriesLabels:         make(map[uint64][]prompb.Label),
@@ -140,7 +136,7 @@ func Test_readToEnd_noCheckpoint(t *testing.T) {
 
 	wt := newWriteToMock()
 	st := timestamp.FromTime(time.Now())
-	watcher := NewWALWatcher(nil, wt, dir, st)
+	watcher := NewWALWatcher(nil, "", wt, dir, st)
 	_, _, err = watcher.readToEnd(wdir, first, last)
 	testutil.Ok(t, err)
 	testutil.Equals(t, seriesCount, len(wt.seriesLabels))
@@ -218,7 +214,7 @@ func Test_readToEnd_withCheckpoint(t *testing.T) {
 
 	wt := newWriteToMock()
 	st := timestamp.FromTime(time.Now())
-	watcher := NewWALWatcher(nil, wt, dir, st)
+	watcher := NewWALWatcher(nil, "", wt, dir, st)
 	_, _, err = watcher.readToEnd(wdir, first, last)
 	testutil.Ok(t, err)
 	testutil.Equals(t, seriesCount*10*2, len(wt.seriesLabels))
@@ -274,7 +270,7 @@ func Test_readCheckpoint(t *testing.T) {
 
 	wt := newWriteToMock()
 	st := timestamp.FromTime(time.Now())
-	watcher := NewWALWatcher(nil, wt, dir, st)
+	watcher := NewWALWatcher(nil, "", wt, dir, st)
 	_, _, err = watcher.readToEnd(wdir, first, last)
 	testutil.Ok(t, err)
 	testutil.Equals(t, seriesCount*10, len(wt.seriesLabels))
@@ -327,7 +323,7 @@ func Test_checkpoint_seriesReset(t *testing.T) {
 
 	wt := newWriteToMock()
 	st := timestamp.FromTime(time.Now())
-	watcher := NewWALWatcher(nil, wt, dir, st)
+	watcher := NewWALWatcher(nil, "", wt, dir, st)
 	_, _, err = watcher.readToEnd(wdir, first, last)
 	testutil.Ok(t, err)
 	testutil.Equals(t, seriesCount*10, len(wt.seriesLabels))
@@ -352,7 +348,7 @@ func Test_decodeRecord(t *testing.T) {
 
 	wt := newWriteToMock()
 	st := timestamp.FromTime(time.Now())
-	watcher := NewWALWatcher(nil, wt, dir, st)
+	watcher := NewWALWatcher(nil, "", wt, dir, st)
 
 	// decode a series record
 	enc := tsdb.RecordEncoder{}

storage/remote/write.go

@@ -20,32 +20,47 @@ import (
 
 // Appender implements scrape.Appendable.
 func (s *Storage) Appender() (storage.Appender, error) {
-	return s, nil
+	return &timestampTracker{
+		storage: s,
+	}, nil
+}
+
+type timestampTracker struct {
+	storage          *Storage
+	samples          int64
+	highestTimestamp int64
 }
 
 // Add implements storage.Appender.
-func (s *Storage) Add(l labels.Labels, t int64, v float64) (uint64, error) {
+func (t *timestampTracker) Add(_ labels.Labels, ts int64, v float64) (uint64, error) {
-	s.samplesIn.incr(1)
+	t.samples++
-	s.samplesInMetric.Inc()
+	if ts > t.highestTimestamp {
-	if t > s.highestTimestamp {
+		t.highestTimestamp = ts
-		s.highestTimestamp = t
 	}
 	return 0, nil
 }
 
 // AddFast implements storage.Appender.
-func (s *Storage) AddFast(l labels.Labels, _ uint64, t int64, v float64) error {
+func (t *timestampTracker) AddFast(l labels.Labels, _ uint64, ts int64, v float64) error {
-	_, err := s.Add(l, t, v)
+	_, err := t.Add(l, ts, v)
 	return err
 }
 
 // Commit implements storage.Appender.
-func (s *Storage) Commit() error {
+func (t *timestampTracker) Commit() error {
-	s.highestTimestampMetric.Set(float64(s.highestTimestamp))
+	t.storage.samplesIn.incr(t.samples)
+	t.storage.samplesInMetric.Add(float64(t.samples))
+
+	t.storage.highestTimestampMtx.Lock()
+	defer t.storage.highestTimestampMtx.Unlock()
+	if t.highestTimestamp > t.storage.highestTimestamp {
+		t.storage.highestTimestamp = t.highestTimestamp
+		t.storage.highestTimestampMetric.Set(float64(t.highestTimestamp))
+	}
 	return nil
 }
 
 // Rollback implements storage.Appender.
-func (*Storage) Rollback() error {
+func (*timestampTracker) Rollback() error {
 	return nil
 }

vendor/modules.txt

@@ -243,12 +243,12 @@ github.com/prometheus/procfs/xfs
 github.com/prometheus/procfs/internal/util
 # github.com/prometheus/tsdb v0.4.0
 github.com/prometheus/tsdb
+github.com/prometheus/tsdb/fileutil
+github.com/prometheus/tsdb/wal
 github.com/prometheus/tsdb/labels
 github.com/prometheus/tsdb/chunkenc
 github.com/prometheus/tsdb/chunks
-github.com/prometheus/tsdb/fileutil
 github.com/prometheus/tsdb/index
-github.com/prometheus/tsdb/wal
 # github.com/samuel/go-zookeeper v0.0.0-20161028232340-1d7be4effb13
 github.com/samuel/go-zookeeper/zk
 # github.com/sasha-s/go-deadlock v0.0.0-20161201235124-341000892f3d