From 720442d3d3a70a377206d52806a38e5736d7901b Mon Sep 17 00:00:00 2001 From: "Matt T. Proud" Date: Sun, 25 Nov 2012 16:07:05 +0100 Subject: [PATCH] Use index singleton and relocate encoder package. --- .gitignore | 22 ------------ encoder.go => coding/general.go | 2 +- .../protocol_buffer.go | 2 +- index.go | 10 ++++-- levigo_index.go | 21 +++++------ levigo_metric_persistence.go | 35 ++++++++++--------- levigo_persistence.go | 9 ++--- persistence.go | 12 ++++--- 8 files changed, 51 insertions(+), 62 deletions(-) delete mode 100644 .gitignore rename encoder.go => coding/general.go (78%) rename protocol_buffer_encoder.go => coding/protocol_buffer.go (95%) diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 00268614f..000000000 --- a/.gitignore +++ /dev/null @@ -1,22 +0,0 @@ -# Compiled Object files, Static and Dynamic libs (Shared Objects) -*.o -*.a -*.so - -# Folders -_obj -_test - -# Architecture specific extensions/prefixes -*.[568vq] -[568vq].out - -*.cgo1.go -*.cgo2.c -_cgo_defun.c -_cgo_gotypes.go -_cgo_export.* - -_testmain.go - -*.exe diff --git a/encoder.go b/coding/general.go similarity index 78% rename from encoder.go rename to coding/general.go index 96824f179..a0d826904 100644 --- a/encoder.go +++ b/coding/general.go @@ -1,4 +1,4 @@ -package main +package coding type Encoder interface { Encode() ([]byte, error) diff --git a/protocol_buffer_encoder.go b/coding/protocol_buffer.go similarity index 95% rename from protocol_buffer_encoder.go rename to coding/protocol_buffer.go index e9b7a3741..a8332751c 100644 --- a/protocol_buffer_encoder.go +++ b/coding/protocol_buffer.go @@ -1,4 +1,4 @@ -package main +package coding import ( "code.google.com/p/goprotobuf/proto" diff --git a/index.go b/index.go index e8d7144e8..ad203d9ba 100644 --- a/index.go +++ b/index.go @@ -1,8 +1,12 @@ package main +import ( + "github.com/matttproud/prometheus/coding" +) + type MembershipIndex interface { - Has(key Encoder) (bool, error) - Put(key Encoder) error - Drop(key Encoder) error + Has(key coding.Encoder) (bool, error) + Put(key coding.Encoder) error + Drop(key coding.Encoder) error Close() error } diff --git a/levigo_index.go b/levigo_index.go index 6fe8a4da1..c2da77bd2 100644 --- a/levigo_index.go +++ b/levigo_index.go @@ -1,40 +1,41 @@ package main import ( + "github.com/matttproud/prometheus/coding" data "github.com/matttproud/prometheus/model/generated" ) +var ( + existenceValue = coding.NewProtocolBufferEncoder(&data.MembershipIndexValueDDO{}) +) + type LevigoMembershipIndex struct { - persistence *LevigoPersistence - existenceValue Encoder + persistence *LevigoPersistence } func (l *LevigoMembershipIndex) Close() error { return l.persistence.Close() } -func (l *LevigoMembershipIndex) Has(key Encoder) (bool, error) { +func (l *LevigoMembershipIndex) Has(key coding.Encoder) (bool, error) { return l.persistence.Has(key) } -func (l *LevigoMembershipIndex) Drop(key Encoder) error { +func (l *LevigoMembershipIndex) Drop(key coding.Encoder) error { return l.persistence.Drop(key) } -func (l *LevigoMembershipIndex) Put(key Encoder) error { - return l.persistence.Put(key, l.existenceValue) +func (l *LevigoMembershipIndex) Put(key coding.Encoder) error { + return l.persistence.Put(key, existenceValue) } func NewLevigoMembershipIndex(storageRoot string, cacheCapacity, bitsPerBloomFilterEncoded int) (*LevigoMembershipIndex, error) { var levigoPersistence *LevigoPersistence var levigoPersistenceError error - existenceValue := NewProtocolBufferEncoder(&data.MembershipIndexValueDDO{}) - if levigoPersistence, levigoPersistenceError = NewLevigoPersistence(storageRoot, cacheCapacity, bitsPerBloomFilterEncoded); levigoPersistenceError == nil { levigoMembershipIndex := &LevigoMembershipIndex{ - persistence: levigoPersistence, - existenceValue: existenceValue, + persistence: levigoPersistence, } return levigoMembershipIndex, nil } diff --git a/levigo_metric_persistence.go b/levigo_metric_persistence.go index d299cb68b..0833c91cc 100644 --- a/levigo_metric_persistence.go +++ b/levigo_metric_persistence.go @@ -4,6 +4,7 @@ import ( "code.google.com/p/goprotobuf/proto" "errors" "fmt" + "github.com/matttproud/prometheus/coding" "github.com/matttproud/prometheus/coding/indexable" data "github.com/matttproud/prometheus/model/generated" "github.com/matttproud/prometheus/utility" @@ -254,12 +255,12 @@ func fingerprintDDOFromByteArray(fingerprint []byte) *data.FingerprintDDO { } func (l *LevigoMetricPersistence) hasIndexMetric(ddo *data.MetricDDO) (bool, error) { - ddoKey := NewProtocolBufferEncoder(ddo) + ddoKey := coding.NewProtocolBufferEncoder(ddo) return l.metricMembershipIndex.Has(ddoKey) } func (l *LevigoMetricPersistence) indexMetric(ddo *data.MetricDDO) error { - ddoKey := NewProtocolBufferEncoder(ddo) + ddoKey := coding.NewProtocolBufferEncoder(ddo) return l.metricMembershipIndex.Put(ddoKey) } @@ -277,17 +278,17 @@ func fingerprintDDOForMessage(message proto.Message) (*data.FingerprintDDO, erro } func (l *LevigoMetricPersistence) HasLabelPair(ddo *data.LabelPairDDO) (bool, error) { - ddoKey := NewProtocolBufferEncoder(ddo) + ddoKey := coding.NewProtocolBufferEncoder(ddo) return l.labelPairFingerprints.Has(ddoKey) } func (l *LevigoMetricPersistence) HasLabelName(ddo *data.LabelNameDDO) (bool, error) { - ddoKey := NewProtocolBufferEncoder(ddo) + ddoKey := coding.NewProtocolBufferEncoder(ddo) return l.labelNameFingerprints.Has(ddoKey) } func (l *LevigoMetricPersistence) GetLabelPairFingerprints(ddo *data.LabelPairDDO) (*data.FingerprintCollectionDDO, error) { - ddoKey := NewProtocolBufferEncoder(ddo) + ddoKey := coding.NewProtocolBufferEncoder(ddo) if get, getError := l.labelPairFingerprints.Get(ddoKey); getError == nil { value := &data.FingerprintCollectionDDO{} if unmarshalError := proto.Unmarshal(get, value); unmarshalError == nil { @@ -302,7 +303,7 @@ func (l *LevigoMetricPersistence) GetLabelPairFingerprints(ddo *data.LabelPairDD } func (l *LevigoMetricPersistence) GetLabelNameFingerprints(ddo *data.LabelNameDDO) (*data.FingerprintCollectionDDO, error) { - ddoKey := NewProtocolBufferEncoder(ddo) + ddoKey := coding.NewProtocolBufferEncoder(ddo) if get, getError := l.labelNameFingerprints.Get(ddoKey); getError == nil { value := &data.FingerprintCollectionDDO{} if unmarshalError := proto.Unmarshal(get, value); unmarshalError == nil { @@ -318,14 +319,14 @@ func (l *LevigoMetricPersistence) GetLabelNameFingerprints(ddo *data.LabelNameDD } func (l *LevigoMetricPersistence) setLabelPairFingerprints(labelPair *data.LabelPairDDO, fingerprints *data.FingerprintCollectionDDO) error { - labelPairEncoded := NewProtocolBufferEncoder(labelPair) - fingerprintsEncoded := NewProtocolBufferEncoder(fingerprints) + labelPairEncoded := coding.NewProtocolBufferEncoder(labelPair) + fingerprintsEncoded := coding.NewProtocolBufferEncoder(fingerprints) return l.labelPairFingerprints.Put(labelPairEncoded, fingerprintsEncoded) } func (l *LevigoMetricPersistence) setLabelNameFingerprints(labelName *data.LabelNameDDO, fingerprints *data.FingerprintCollectionDDO) error { - labelNameEncoded := NewProtocolBufferEncoder(labelName) - fingerprintsEncoded := NewProtocolBufferEncoder(fingerprints) + labelNameEncoded := coding.NewProtocolBufferEncoder(labelName) + fingerprintsEncoded := coding.NewProtocolBufferEncoder(fingerprints) return l.labelNameFingerprints.Put(labelNameEncoded, fingerprintsEncoded) } @@ -384,8 +385,8 @@ func (l *LevigoMetricPersistence) appendFingerprints(ddo *data.MetricDDO) error labelPairCollectionDDO := &data.LabelPairCollectionDDO{ Member: ddo.LabelPair, } - fingerprintKey := NewProtocolBufferEncoder(fingerprintDDO) - labelPairCollectionDDOEncoder := NewProtocolBufferEncoder(labelPairCollectionDDO) + fingerprintKey := coding.NewProtocolBufferEncoder(fingerprintDDO) + labelPairCollectionDDOEncoder := coding.NewProtocolBufferEncoder(labelPairCollectionDDO) if putError := l.fingerprintLabelPairs.Put(fingerprintKey, labelPairCollectionDDOEncoder); putError == nil { labelCount := len(ddo.LabelPair) @@ -463,8 +464,8 @@ func (l *LevigoMetricPersistence) AppendSample(sample *Sample) error { Value: proto.Float32(float32(sample.Value)), } - sampleKeyEncoded := NewProtocolBufferEncoder(sampleKeyDDO) - sampleValueEncoded := NewProtocolBufferEncoder(sampleValueDDO) + sampleKeyEncoded := coding.NewProtocolBufferEncoder(sampleKeyDDO) + sampleValueEncoded := coding.NewProtocolBufferEncoder(sampleValueDDO) if putError := l.fingerprintSamples.Put(sampleKeyEncoded, sampleValueEncoded); putError != nil { log.Printf("Could not append metric sample: %q\n", putError) @@ -544,7 +545,7 @@ func (l *LevigoMetricPersistence) GetMetrics() ([]LabelPairs, error) { log.Printf("!Has: %q\n", member.Signature) fingerprints.Add(*member.Signature) log.Printf("fingerprints %q\n", fingerprints) - fingerprintEncoded := NewProtocolBufferEncoder(member) + fingerprintEncoded := coding.NewProtocolBufferEncoder(member) if labelPairCollectionRaw, labelPairCollectionRawError := l.fingerprintLabelPairs.Get(fingerprintEncoded); labelPairCollectionRawError == nil { log.Printf("labelPairCollectionRaw: %q\n", labelPairCollectionRaw) @@ -592,7 +593,7 @@ func (l *LevigoMetricPersistence) GetWatermarksForMetric(metric Metric) (*Interv Timestamp: indexable.EarliestTime, } - if encode, encodeErr := NewProtocolBufferEncoder(start).Encode(); encodeErr == nil { + if encode, encodeErr := coding.NewProtocolBufferEncoder(start).Encode(); encodeErr == nil { iterator.Seek(encode) if iterator.Valid() { @@ -668,7 +669,7 @@ func (l *LevigoMetricPersistence) GetSamplesForMetric(metric Metric, interval In emission := make([]Samples, 0) - if encode, encodeErr := NewProtocolBufferEncoder(start).Encode(); encodeErr == nil { + if encode, encodeErr := coding.NewProtocolBufferEncoder(start).Encode(); encodeErr == nil { iterator.Seek(encode) for iterator = iterator; iterator.Valid(); iterator.Next() { diff --git a/levigo_persistence.go b/levigo_persistence.go index 99eaae1f1..62c2a478e 100644 --- a/levigo_persistence.go +++ b/levigo_persistence.go @@ -2,6 +2,7 @@ package main import ( "github.com/jmhodges/levigo" + "github.com/matttproud/prometheus/coding" "io" ) @@ -85,7 +86,7 @@ func (l *LevigoPersistence) Close() error { return nil } -func (l *LevigoPersistence) Get(value Encoder) ([]byte, error) { +func (l *LevigoPersistence) Get(value coding.Encoder) ([]byte, error) { var key []byte var keyError error @@ -96,7 +97,7 @@ func (l *LevigoPersistence) Get(value Encoder) ([]byte, error) { return nil, keyError } -func (l *LevigoPersistence) Has(value Encoder) (bool, error) { +func (l *LevigoPersistence) Has(value coding.Encoder) (bool, error) { if value, getError := l.Get(value); getError != nil { return false, getError } else if value == nil { @@ -106,7 +107,7 @@ func (l *LevigoPersistence) Has(value Encoder) (bool, error) { return true, nil } -func (l *LevigoPersistence) Drop(value Encoder) error { +func (l *LevigoPersistence) Drop(value coding.Encoder) error { var key []byte var keyError error @@ -122,7 +123,7 @@ func (l *LevigoPersistence) Drop(value Encoder) error { return keyError } -func (l *LevigoPersistence) Put(key Encoder, value Encoder) error { +func (l *LevigoPersistence) Put(key, value coding.Encoder) error { var keyEncoded []byte var keyError error diff --git a/persistence.go b/persistence.go index b249ebb8d..0ec743124 100644 --- a/persistence.go +++ b/persistence.go @@ -1,15 +1,19 @@ package main +import ( + "github.com/matttproud/prometheus/coding" +) + type Pair struct { Left []byte Right []byte } type Persistence interface { - Has(key Encoder) (bool, error) - Get(key Encoder) ([]byte, error) + Has(key coding.Encoder) (bool, error) + Get(key coding.Encoder) ([]byte, error) GetAll() ([]Pair, error) - Drop(key Encoder) error - Put(key Encoder, value Encoder) error + Drop(key coding.Encoder) error + Put(key, value coding.Encoder) error Close() error }