diff --git a/tsdb/wlog/wlog.go b/tsdb/wlog/wlog.go
index 577057fd4..668fbb5fb 100644
--- a/tsdb/wlog/wlog.go
+++ b/tsdb/wlog/wlog.go
@@ -228,10 +228,28 @@ type wlMetrics struct {
 	currentSegment  prometheus.Gauge
 	writesFailed    prometheus.Counter
 	walFileSize     prometheus.GaugeFunc
+
+	r prometheus.Registerer
+}
+
+func (w *wlMetrics) Unregister() {
+	if w.r == nil {
+		return
+	}
+	w.r.Unregister(w.fsyncDuration)
+	w.r.Unregister(w.pageFlushes)
+	w.r.Unregister(w.pageCompletions)
+	w.r.Unregister(w.truncateFail)
+	w.r.Unregister(w.truncateTotal)
+	w.r.Unregister(w.currentSegment)
+	w.r.Unregister(w.writesFailed)
+	w.r.Unregister(w.walFileSize)
 }
 
 func newWLMetrics(w *WL, r prometheus.Registerer) *wlMetrics {
-	m := &wlMetrics{}
+	m := &wlMetrics{
+		r: r,
+	}
 
 	m.fsyncDuration = prometheus.NewSummary(prometheus.SummaryOpts{
 		Name:       "fsync_duration_seconds",
@@ -877,6 +895,8 @@ func (w *WL) Close() (err error) {
 	if err := w.segment.Close(); err != nil {
 		level.Error(w.logger).Log("msg", "close previous segment", "err", err)
 	}
+
+	w.metrics.Unregister()
 	w.closed = true
 	return nil
 }
diff --git a/tsdb/wlog/wlog_test.go b/tsdb/wlog/wlog_test.go
index 7d9692011..165d2758f 100644
--- a/tsdb/wlog/wlog_test.go
+++ b/tsdb/wlog/wlog_test.go
@@ -23,6 +23,8 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	client_testutil "github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
@@ -561,3 +563,13 @@ func BenchmarkWAL_Log(b *testing.B) {
 		})
 	}
 }
+
+func TestUnregisterMetrics(t *testing.T) {
+	reg := prometheus.NewRegistry()
+
+	for i := 0; i < 2; i++ {
+		wl, err := New(log.NewNopLogger(), reg, t.TempDir(), CompressionNone)
+		require.NoError(t, err)
+		require.NoError(t, wl.Close())
+	}
+}