mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2024-12-24 05:04:05 -08:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

move label read function int labels pkg

Browse source
This commit is contained in:
Fabian Reinartz 2017-12-21 11:55:58 +01:00
parent 67f0ca8f0e
commit adb6be73dd
7 changed files with 71 additions and 145 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
block_test.go
View file

@ -45,8 +45,8 @@ func createEmptyBlock(t *testing.T, dir string) *Block {
testutil.Ok(t, writeMetaFile(dir, &BlockMeta{}))
ir, err := index.NewWriter(filepath.Join(dir, indexFilename))
Ok(t, err)
Ok(t, ir.Close())
testutil.Ok(t, err)
testutil.Ok(t, ir.Close())
testutil.Ok(t, os.MkdirAll(chunkDir(dir), 0777))

55
head_test.go
View file

@ -14,22 +14,18 @@
package tsdb
import (
"bufio"
"math/rand"
"os"
"sort"
"strings"
"testing"
"github.com/pkg/errors"
"github.com/prometheus/tsdb/chunkenc"
"github.com/prometheus/tsdb/chunks"
"github.com/prometheus/tsdb/index"
"github.com/prometheus/tsdb/labels"
"github.com/prometheus/tsdb/testutil"
)
func BenchmarkCreateSeries(b *testing.B) {
lbls, err := readPrometheusLabels("testdata/all.series", b.N)
lbls, err := labels.ReadLabels("testdata/all.series", b.N)
testutil.Ok(b, err)
h, err := NewHead(nil, nil, nil, 10000)
@ -46,49 +42,6 @@ func BenchmarkCreateSeries(b *testing.B) {
}
}
func readPrometheusLabels(fn string, n int) ([]labels.Labels, error) {
f, err := os.Open(fn)
if err != nil {
return nil, err
}
defer f.Close()
scanner := bufio.NewScanner(f)
var mets []labels.Labels
hashes := map[uint64]struct{}{}
i := 0
for scanner.Scan() && i < n {
m := make(labels.Labels, 0, 10)
r := strings.NewReplacer("\"", "", "{", "", "}", "")
s := r.Replace(scanner.Text())
labelChunks := strings.Split(s, ",")
for _, labelChunk := range labelChunks {
split := strings.Split(labelChunk, ":")
m = append(m, labels.Label{Name: split[0], Value: split[1]})
}
// Order of the k/v labels matters, don't assume we'll always receive them already sorted.
sort.Sort(m)
h := m.Hash()
if _, ok := hashes[h]; ok {
continue
}
mets = append(mets, m)
hashes[h] = struct{}{}
i++
}
if err != nil {
return nil, err
}
if i != n {
return mets, errors.Errorf("requested %d metrics but found %d", n, i)
}
return mets, nil
}
type memoryWAL struct {
nopWAL
entries []interface{}
@ -727,7 +680,7 @@ func TestGCChunkAccess(t *testing.T) {
idx := h.indexRange(0, 1500)
var (
lset labels.Labels
chunks []ChunkMeta
chunks []chunks.Meta
)
testutil.Ok(t, idx.Series(1, &lset, &chunks))
@ -767,7 +720,7 @@ func TestGCSeriesAccess(t *testing.T) {
idx := h.indexRange(0, 2000)
var (
lset labels.Labels
chunks []ChunkMeta
chunks []chunks.Meta
)
testutil.Ok(t, idx.Series(1, &lset, &chunks))

57
index/index_test.go
View file

@ -20,11 +20,8 @@ import (
"path/filepath"
"sort"
"testing"
"unsafe"
"github.com/pkg/errors"
promlabels "github.com/prometheus/prometheus/pkg/labels"
"github.com/prometheus/prometheus/pkg/textparse"
"github.com/prometheus/tsdb/chunkenc"
"github.com/prometheus/tsdb/chunks"
"github.com/prometheus/tsdb/labels"
@ -242,7 +239,7 @@ func TestPersistence_index_e2e(t *testing.T) {
testutil.Ok(t, err)
defer os.RemoveAll(dir)
lbls, err := readPrometheusLabels("testdata/20kseries.json", 20000)
lbls, err := labels.ReadLabels("../testdata/20kseries.json", 20000)
testutil.Ok(t, err)
// Sort labels as the index writer expects series in sorted order.
@ -307,7 +304,11 @@ func TestPersistence_index_e2e(t *testing.T) {
}
for k, v := range values {
vals := v.slice()
var vals []string
for e := range v {
vals = append(vals, e)
}
sort.Strings(vals)
testutil.Ok(t, iw.WriteLabelIndex([]string{k}, vals))
testutil.Ok(t, mi.WriteLabelIndex([]string{k}, vals))
@ -322,15 +323,15 @@ func TestPersistence_index_e2e(t *testing.T) {
mi.WritePostings("", "", newListPostings(all))
for l := range postings.m {
err = iw.WritePostings(l.Name, l.Value, postings.get(l.Name, l.Value))
err = iw.WritePostings(l.Name, l.Value, postings.Get(l.Name, l.Value))
testutil.Ok(t, err)
mi.WritePostings(l.Name, l.Value, postings.get(l.Name, l.Value))
mi.WritePostings(l.Name, l.Value, postings.Get(l.Name, l.Value))
}
err = iw.Close()
testutil.Ok(t, err)
ir, err := NewFileIndexReader(filepath.Join(dir, "index"))
ir, err := NewFileReader(filepath.Join(dir, "index"))
testutil.Ok(t, err)
for p := range mi.postings {
@ -359,7 +360,7 @@ func TestPersistence_index_e2e(t *testing.T) {
}
for k, v := range mi.labelIndex {
tplsExp, err := newStringTuples(v, 1)
tplsExp, err := NewStringTuples(v, 1)
testutil.Ok(t, err)
tplsRes, err := ir.LabelValues(k)
@ -379,41 +380,3 @@ func TestPersistence_index_e2e(t *testing.T) {
testutil.Ok(t, ir.Close())
}
func readPrometheusLabels(fn string, n int) ([]labels.Labels, error) {
f, err := os.Open(fn)
if err != nil {
return nil, err
}
defer f.Close()
b, err := ioutil.ReadAll(f)
if err != nil {
return nil, err
}
p := textparse.New(b)
i := 0
var mets []labels.Labels
hashes := map[uint64]struct{}{}
for p.Next() && i < n {
m := make(labels.Labels, 0, 10)
p.Metric((*promlabels.Labels)(unsafe.Pointer(&m)))
h := m.Hash()
if _, ok := hashes[h]; ok {
continue
}
mets = append(mets, m)
hashes[h] = struct{}{}
i++
}
if err := p.Err(); err != nil {
return nil, err
}
if i != n {
return mets, errors.Errorf("requested %d metrics but found %d", n, i)
}
return mets, nil
}

49
labels/labels.go
View file

@ -14,12 +14,15 @@
package labels
import (
"bufio"
"bytes"
"os"
"sort"
"strconv"
"strings"
"github.com/cespare/xxhash"
"github.com/pkg/errors"
)
const sep = '\xff'
@ -161,3 +164,49 @@ type Slice []Labels
func (s Slice) Len() int { return len(s) }
func (s Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s Slice) Less(i, j int) bool { return Compare(s[i], s[j]) < 0 }
// ReadLabels reads up to n label sets in a JSON formatted file fn. It is mostly useful
// to load testing data.
func ReadLabels(fn string, n int) ([]Labels, error) {
f, err := os.Open(fn)
if err != nil {
return nil, err
}
defer f.Close()
scanner := bufio.NewScanner(f)
var mets []Labels
hashes := map[uint64]struct{}{}
i := 0
for scanner.Scan() && i < n {
m := make(Labels, 0, 10)
r := strings.NewReplacer("\"", "", "{", "", "}", "")
s := r.Replace(scanner.Text())
labelChunks := strings.Split(s, ",")
for _, labelChunk := range labelChunks {
split := strings.Split(labelChunk, ":")
m = append(m, Label{Name: split[0], Value: split[1]})
}
// Order of the k/v labels matters, don't assume we'll always receive them already sorted.
sort.Sort(m)
h := m.Hash()
if _, ok := hashes[h]; ok {
continue
}
mets = append(mets, m)
hashes[h] = struct{}{}
i++
}
if err != nil {
return nil, err
}
if i != n {
return mets, errors.Errorf("requested %d metrics but found %d", n, i)
}
return mets, nil
}

42
labels/labels_test.go
View file

@ -14,12 +14,9 @@
package labels
import (
"bufio"
"fmt"
"math/rand"
"os"
"sort"
"strings"
"testing"
"github.com/prometheus/tsdb/testutil"
@ -90,7 +87,7 @@ func TestCompareAndEquals(t *testing.T) {
}
func BenchmarkSliceSort(b *testing.B) {
lbls, err := readPrometheusLabels("../testdata/1m.series", 900000)
lbls, err := ReadLabels("../testdata/1m.series", 900000)
testutil.Ok(b, err)
for len(lbls) < 20e6 {
@ -119,43 +116,6 @@ func BenchmarkSliceSort(b *testing.B) {
}
}
func readPrometheusLabels(fn string, n int) ([]Labels, error) {
f, err := os.Open(fn)
if err != nil {
return nil, err
}
defer f.Close()
scanner := bufio.NewScanner(f)
var mets []Labels
hashes := map[uint64]struct{}{}
i := 0
for scanner.Scan() && i < n {
m := make(Labels, 0, 10)
r := strings.NewReplacer("\"", "", "{", "", "}", "")
s := r.Replace(scanner.Text())
labelChunks := strings.Split(s, ",")
for _, labelChunk := range labelChunks {
split := strings.Split(labelChunk, ":")
m = append(m, Label{Name: split[0], Value: split[1]})
}
// Order of the k/v labels matters, don't assume we'll always receive them already sorted.
sort.Sort(m)
h := m.Hash()
if _, ok := hashes[h]; ok {
continue
}
mets = append(mets, m)
hashes[h] = struct{}{}
i++
}
return mets, nil
}
func BenchmarkLabelSetFromMap(b *testing.B) {
m := map[string]string{
"job": "node",

4
querier_test.go
View file

@ -1248,7 +1248,7 @@ func BenchmarkMergedSeriesSet(b *testing.B) {
} {
for _, j := range []int{1, 2, 4, 8, 16, 32} {
b.Run(fmt.Sprintf("series=%d,blocks=%d", k, j), func(b *testing.B) {
lbls, err := readPrometheusLabels("testdata/1m.series", k)
lbls, err := labels.ReadLabels("testdata/1m.series", k)
testutil.Ok(b, err)
sort.Sort(labels.Slice(lbls))
@ -1299,7 +1299,7 @@ func (cr mockChunkReader) Close() error {
}
func TestDeletedIterator(t *testing.T) {
chk := chunks.NewXORChunk()
chk := chunkenc.NewXORChunk()
app, err := chk.Appender()
testutil.Ok(t, err)
// Insert random stuff from (0, 1000).

5
wal_test.go
View file

@ -22,6 +22,7 @@ import (
"github.com/go-kit/kit/log"
"github.com/prometheus/tsdb/fileutil"
"github.com/prometheus/tsdb/labels"
"github.com/prometheus/tsdb/testutil"
)
@ -71,7 +72,7 @@ func TestSegmentWAL_Truncate(t *testing.T) {
numMetrics = 20000
batch = 100
)
series, err := readPrometheusLabels("testdata/20kseries.json", numMetrics)
series, err := labels.ReadLabels("testdata/20kseries.json", numMetrics)
testutil.Ok(t, err)
dir, err := ioutil.TempDir("", "test_wal_log_truncate")
@ -150,7 +151,7 @@ func TestSegmentWAL_Log_Restore(t *testing.T) {
)
// Generate testing data. It does not make semantical sense but
// for the purpose of this test.
series, err := readPrometheusLabels("testdata/20kseries.json", numMetrics)
series, err := labels.ReadLabels("testdata/20kseries.json", numMetrics)
testutil.Ok(t, err)
dir, err := ioutil.TempDir("", "test_wal_log_restore")