Merge pull request #519 from grafana/compactor-stringlabels

compactor: avoid memory blow-up with stringlabels

tsdb/addsymbol.go

@@ -0,0 +1,10 @@
+//go:build !stringlabels
+
+// Split out function which needs to be coded differently for stringlabels case.
+
+package tsdb
+
+func (sw *symbolsBatcher) addSymbol(sym string) error {
+	sw.buffer[sym] = struct{}{}
+	return sw.flushSymbols(false)
+}

tsdb/addsymbol_stringlabels.go

@@ -0,0 +1,15 @@
+//go:build stringlabels
+
+// Split out function which needs to be coded differently for stringlabels case.
+
+package tsdb
+
+import "strings"
+
+func (sw *symbolsBatcher) addSymbol(sym string) error {
+	if _, found := sw.buffer[sym]; !found {
+		sym = strings.Clone(sym) // So we don't retain reference to the entire labels block.
+		sw.buffer[sym] = struct{}{}
+	}
+	return sw.flushSymbols(false)
+}

tsdb/symbols_batch.go

@@ -145,11 +145,6 @@ func newSymbolsBatcher(limit int, dir string, flushers *symbolFlushers) *symbols
 	}
 }
 
-func (sw *symbolsBatcher) addSymbol(sym string) error {
-	sw.buffer[sym] = struct{}{}
-	return sw.flushSymbols(false)
-}
-
 func (sw *symbolsBatcher) flushSymbols(force bool) error {
 	if !force && len(sw.buffer) < sw.limit {
 		return nil