move label read function int labels pkg

block_test.go

@@ -45,8 +45,8 @@ func createEmptyBlock(t *testing.T, dir string) *Block {
 	testutil.Ok(t, writeMetaFile(dir, &BlockMeta{}))
 
 	ir, err := index.NewWriter(filepath.Join(dir, indexFilename))
-	Ok(t, err)
+	testutil.Ok(t, err)
-	Ok(t, ir.Close())
+	testutil.Ok(t, ir.Close())
 
 	testutil.Ok(t, os.MkdirAll(chunkDir(dir), 0777))
 

head_test.go

@@ -14,22 +14,18 @@
 package tsdb
 
 import (
-	"bufio"
 	"math/rand"
-	"os"
-	"sort"
-	"strings"
 	"testing"
 
-	"github.com/pkg/errors"
 	"github.com/prometheus/tsdb/chunkenc"
+	"github.com/prometheus/tsdb/chunks"
 	"github.com/prometheus/tsdb/index"
 	"github.com/prometheus/tsdb/labels"
 	"github.com/prometheus/tsdb/testutil"
 )
 
 func BenchmarkCreateSeries(b *testing.B) {
-	lbls, err := readPrometheusLabels("testdata/all.series", b.N)
+	lbls, err := labels.ReadLabels("testdata/all.series", b.N)
 	testutil.Ok(b, err)
 
 	h, err := NewHead(nil, nil, nil, 10000)
@@ -46,49 +42,6 @@ func BenchmarkCreateSeries(b *testing.B) {
 	}
 }
 
-func readPrometheusLabels(fn string, n int) ([]labels.Labels, error) {
-	f, err := os.Open(fn)
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-
-	scanner := bufio.NewScanner(f)
-
-	var mets []labels.Labels
-	hashes := map[uint64]struct{}{}
-	i := 0
-
-	for scanner.Scan() && i < n {
-		m := make(labels.Labels, 0, 10)
-
-		r := strings.NewReplacer("\"", "", "{", "", "}", "")
-		s := r.Replace(scanner.Text())
-
-		labelChunks := strings.Split(s, ",")
-		for _, labelChunk := range labelChunks {
-			split := strings.Split(labelChunk, ":")
-			m = append(m, labels.Label{Name: split[0], Value: split[1]})
-		}
-		// Order of the k/v labels matters, don't assume we'll always receive them already sorted.
-		sort.Sort(m)
-		h := m.Hash()
-		if _, ok := hashes[h]; ok {
-			continue
-		}
-		mets = append(mets, m)
-		hashes[h] = struct{}{}
-		i++
-	}
-	if err != nil {
-		return nil, err
-	}
-	if i != n {
-		return mets, errors.Errorf("requested %d metrics but found %d", n, i)
-	}
-	return mets, nil
-}
-
 type memoryWAL struct {
 	nopWAL
 	entries []interface{}
@@ -727,7 +680,7 @@ func TestGCChunkAccess(t *testing.T) {
 	idx := h.indexRange(0, 1500)
 	var (
 		lset   labels.Labels
-		chunks []ChunkMeta
+		chunks []chunks.Meta
 	)
 	testutil.Ok(t, idx.Series(1, &lset, &chunks))
 
@@ -767,7 +720,7 @@ func TestGCSeriesAccess(t *testing.T) {
 	idx := h.indexRange(0, 2000)
 	var (
 		lset   labels.Labels
-		chunks []ChunkMeta
+		chunks []chunks.Meta
 	)
 	testutil.Ok(t, idx.Series(1, &lset, &chunks))
 

index/index_test.go

@@ -20,11 +20,8 @@ import (
 	"path/filepath"
 	"sort"
 	"testing"
-	"unsafe"
 
 	"github.com/pkg/errors"
-	promlabels "github.com/prometheus/prometheus/pkg/labels"
-	"github.com/prometheus/prometheus/pkg/textparse"
 	"github.com/prometheus/tsdb/chunkenc"
 	"github.com/prometheus/tsdb/chunks"
 	"github.com/prometheus/tsdb/labels"
@@ -242,7 +239,7 @@ func TestPersistence_index_e2e(t *testing.T) {
 	testutil.Ok(t, err)
 	defer os.RemoveAll(dir)
 
-	lbls, err := readPrometheusLabels("testdata/20kseries.json", 20000)
+	lbls, err := labels.ReadLabels("../testdata/20kseries.json", 20000)
 	testutil.Ok(t, err)
 
 	// Sort labels as the index writer expects series in sorted order.
@@ -307,7 +304,11 @@ func TestPersistence_index_e2e(t *testing.T) {
 	}
 
 	for k, v := range values {
-		vals := v.slice()
+		var vals []string
+		for e := range v {
+			vals = append(vals, e)
+		}
+		sort.Strings(vals)
 
 		testutil.Ok(t, iw.WriteLabelIndex([]string{k}, vals))
 		testutil.Ok(t, mi.WriteLabelIndex([]string{k}, vals))
@@ -322,15 +323,15 @@ func TestPersistence_index_e2e(t *testing.T) {
 	mi.WritePostings("", "", newListPostings(all))
 
 	for l := range postings.m {
-		err = iw.WritePostings(l.Name, l.Value, postings.get(l.Name, l.Value))
+		err = iw.WritePostings(l.Name, l.Value, postings.Get(l.Name, l.Value))
 		testutil.Ok(t, err)
-		mi.WritePostings(l.Name, l.Value, postings.get(l.Name, l.Value))
+		mi.WritePostings(l.Name, l.Value, postings.Get(l.Name, l.Value))
 	}
 
 	err = iw.Close()
 	testutil.Ok(t, err)
 
-	ir, err := NewFileIndexReader(filepath.Join(dir, "index"))
+	ir, err := NewFileReader(filepath.Join(dir, "index"))
 	testutil.Ok(t, err)
 
 	for p := range mi.postings {
@@ -359,7 +360,7 @@ func TestPersistence_index_e2e(t *testing.T) {
 	}
 
 	for k, v := range mi.labelIndex {
-		tplsExp, err := newStringTuples(v, 1)
+		tplsExp, err := NewStringTuples(v, 1)
 		testutil.Ok(t, err)
 
 		tplsRes, err := ir.LabelValues(k)
@@ -379,41 +380,3 @@ func TestPersistence_index_e2e(t *testing.T) {
 
 	testutil.Ok(t, ir.Close())
 }
-
-func readPrometheusLabels(fn string, n int) ([]labels.Labels, error) {
-	f, err := os.Open(fn)
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-
-	b, err := ioutil.ReadAll(f)
-	if err != nil {
-		return nil, err
-	}
-
-	p := textparse.New(b)
-	i := 0
-	var mets []labels.Labels
-	hashes := map[uint64]struct{}{}
-
-	for p.Next() && i < n {
-		m := make(labels.Labels, 0, 10)
-		p.Metric((*promlabels.Labels)(unsafe.Pointer(&m)))
-
-		h := m.Hash()
-		if _, ok := hashes[h]; ok {
-			continue
-		}
-		mets = append(mets, m)
-		hashes[h] = struct{}{}
-		i++
-	}
-	if err := p.Err(); err != nil {
-		return nil, err
-	}
-	if i != n {
-		return mets, errors.Errorf("requested %d metrics but found %d", n, i)
-	}
-	return mets, nil
-}

labels/labels.go

@@ -14,12 +14,15 @@
 package labels
 
 import (
+	"bufio"
 	"bytes"
+	"os"
 	"sort"
 	"strconv"
 	"strings"
 
 	"github.com/cespare/xxhash"
+	"github.com/pkg/errors"
 )
 
 const sep = '\xff'
@@ -161,3 +164,49 @@ type Slice []Labels
 func (s Slice) Len() int           { return len(s) }
 func (s Slice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
 func (s Slice) Less(i, j int) bool { return Compare(s[i], s[j]) < 0 }
+
+// ReadLabels reads up to n label sets in a JSON formatted file fn. It is mostly useful
+// to load testing data.
+func ReadLabels(fn string, n int) ([]Labels, error) {
+	f, err := os.Open(fn)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+
+	var mets []Labels
+	hashes := map[uint64]struct{}{}
+	i := 0
+
+	for scanner.Scan() && i < n {
+		m := make(Labels, 0, 10)
+
+		r := strings.NewReplacer("\"", "", "{", "", "}", "")
+		s := r.Replace(scanner.Text())
+
+		labelChunks := strings.Split(s, ",")
+		for _, labelChunk := range labelChunks {
+			split := strings.Split(labelChunk, ":")
+			m = append(m, Label{Name: split[0], Value: split[1]})
+		}
+		// Order of the k/v labels matters, don't assume we'll always receive them already sorted.
+		sort.Sort(m)
+
+		h := m.Hash()
+		if _, ok := hashes[h]; ok {
+			continue
+		}
+		mets = append(mets, m)
+		hashes[h] = struct{}{}
+		i++
+	}
+	if err != nil {
+		return nil, err
+	}
+	if i != n {
+		return mets, errors.Errorf("requested %d metrics but found %d", n, i)
+	}
+	return mets, nil
+}

labels/labels_test.go

@@ -14,12 +14,9 @@
 package labels
 
 import (
-	"bufio"
 	"fmt"
 	"math/rand"
-	"os"
 	"sort"
-	"strings"
 	"testing"
 
 	"github.com/prometheus/tsdb/testutil"
@@ -90,7 +87,7 @@ func TestCompareAndEquals(t *testing.T) {
 }
 
 func BenchmarkSliceSort(b *testing.B) {
-	lbls, err := readPrometheusLabels("../testdata/1m.series", 900000)
+	lbls, err := ReadLabels("../testdata/1m.series", 900000)
 	testutil.Ok(b, err)
 
 	for len(lbls) < 20e6 {
@@ -119,43 +116,6 @@ func BenchmarkSliceSort(b *testing.B) {
 	}
 }
 
-func readPrometheusLabels(fn string, n int) ([]Labels, error) {
-	f, err := os.Open(fn)
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-
-	scanner := bufio.NewScanner(f)
-
-	var mets []Labels
-	hashes := map[uint64]struct{}{}
-	i := 0
-
-	for scanner.Scan() && i < n {
-		m := make(Labels, 0, 10)
-
-		r := strings.NewReplacer("\"", "", "{", "", "}", "")
-		s := r.Replace(scanner.Text())
-
-		labelChunks := strings.Split(s, ",")
-		for _, labelChunk := range labelChunks {
-			split := strings.Split(labelChunk, ":")
-			m = append(m, Label{Name: split[0], Value: split[1]})
-		}
-		// Order of the k/v labels matters, don't assume we'll always receive them already sorted.
-		sort.Sort(m)
-		h := m.Hash()
-		if _, ok := hashes[h]; ok {
-			continue
-		}
-		mets = append(mets, m)
-		hashes[h] = struct{}{}
-		i++
-	}
-	return mets, nil
-}
-
 func BenchmarkLabelSetFromMap(b *testing.B) {
 	m := map[string]string{
 		"job":       "node",

querier_test.go

@@ -1248,7 +1248,7 @@ func BenchmarkMergedSeriesSet(b *testing.B) {
 	} {
 		for _, j := range []int{1, 2, 4, 8, 16, 32} {
 			b.Run(fmt.Sprintf("series=%d,blocks=%d", k, j), func(b *testing.B) {
-				lbls, err := readPrometheusLabels("testdata/1m.series", k)
+				lbls, err := labels.ReadLabels("testdata/1m.series", k)
 				testutil.Ok(b, err)
 
 				sort.Sort(labels.Slice(lbls))
@@ -1299,7 +1299,7 @@ func (cr mockChunkReader) Close() error {
 }
 
 func TestDeletedIterator(t *testing.T) {
-	chk := chunks.NewXORChunk()
+	chk := chunkenc.NewXORChunk()
 	app, err := chk.Appender()
 	testutil.Ok(t, err)
 	// Insert random stuff from (0, 1000).

wal_test.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/go-kit/kit/log"
 	"github.com/prometheus/tsdb/fileutil"
+	"github.com/prometheus/tsdb/labels"
 	"github.com/prometheus/tsdb/testutil"
 )
 
@@ -71,7 +72,7 @@ func TestSegmentWAL_Truncate(t *testing.T) {
 		numMetrics = 20000
 		batch      = 100
 	)
-	series, err := readPrometheusLabels("testdata/20kseries.json", numMetrics)
+	series, err := labels.ReadLabels("testdata/20kseries.json", numMetrics)
 	testutil.Ok(t, err)
 
 	dir, err := ioutil.TempDir("", "test_wal_log_truncate")
@@ -150,7 +151,7 @@ func TestSegmentWAL_Log_Restore(t *testing.T) {
 	)
 	// Generate testing data. It does not make semantical sense but
 	// for the purpose of this test.
-	series, err := readPrometheusLabels("testdata/20kseries.json", numMetrics)
+	series, err := labels.ReadLabels("testdata/20kseries.json", numMetrics)
 	testutil.Ok(t, err)
 
 	dir, err := ioutil.TempDir("", "test_wal_log_restore")