Persist solely Protocol Buffers.

An design question was open for me in the beginning was whether to
serialize other types to disk, but Protocol Buffers quickly won out,
which allows us to drop support for other types.  This is a good
start to cleaning up a lot of cruft in the storage stack and
can let us eventually decouple the various moving parts into
separate subsystems for easier reasoning.

This commit is not strictly required, but it is a start to making
the rest a lot more enjoyable to interact with.

storage/metric/curator.go

@@ -206,10 +206,10 @@ func (w watermarkFilter) shouldStop() bool {
 	return len(w.stop) != 0
 }
 
-func getCurationRemark(states raw.Persistence, processor Processor, ignoreYoungerThan time.Duration, fingerprint *model.Fingerprint) (remark *model.CurationRemark, err error) {
+func getCurationRemark(states raw.Persistence, processor Processor, ignoreYoungerThan time.Duration, fingerprint *model.Fingerprint) (*model.CurationRemark, error) {
 	rawSignature, err := processor.Signature()
 	if err != nil {
-		return
+		return nil, err
 	}
 
 	curationKey := model.CurationKey{
@@ -220,30 +220,17 @@ func getCurationRemark(states raw.Persistence, processor Processor, ignoreYounge
 	}.ToDTO()
 	curationValue := &dto.CurationValue{}
 
-	rawKey := coding.NewPBEncoder(curationKey)
+	present, err := states.Get(curationKey, curationValue)
-
-	has, err := states.Has(rawKey)
 	if err != nil {
-		return
+		return nil, err
 	}
-	if !has {
+	if !present {
-		return
+		return nil, nil
 	}
 
-	rawCurationValue, err := states.Get(rawKey)
+	remark := model.NewCurationRemarkFromDTO(curationValue)
-	if err != nil {
-		return
-	}
 
-	err = proto.Unmarshal(rawCurationValue, curationValue)
+	return &remark, nil
-	if err != nil {
-		return
-	}
-
-	baseRemark := model.NewCurationRemarkFromDTO(curationValue)
-	remark = &baseRemark
-
-	return
 }
 
 func (w watermarkFilter) Filter(key, value interface{}) (r storage.FilterResult) {
@@ -386,7 +373,7 @@ func (w watermarkOperator) refreshCurationRemark(f *model.Fingerprint, finished
 		LastCompletionTimestamp: finished,
 	}.ToDTO()
 
-	err = w.curationState.Put(coding.NewPBEncoder(curationKey), coding.NewPBEncoder(curationValue))
+	err = w.curationState.Put(curationKey, curationValue)
 
 	return
 }

storage/metric/leveldb.go

@@ -14,25 +14,25 @@
 package metric
 
 import (
-	"code.google.com/p/goprotobuf/proto"
 	"flag"
 	"fmt"
-	"github.com/prometheus/prometheus/coding"
-	"github.com/prometheus/prometheus/model"
-	dto "github.com/prometheus/prometheus/model/generated"
-	"github.com/prometheus/prometheus/storage"
-	index "github.com/prometheus/prometheus/storage/raw/index/leveldb"
-	"github.com/prometheus/prometheus/storage/raw/leveldb"
-	"github.com/prometheus/prometheus/utility"
 	"log"
 	"sort"
 	"sync"
 	"time"
+
+	"code.google.com/p/goprotobuf/proto"
+
+	dto "github.com/prometheus/prometheus/model/generated"
+	index "github.com/prometheus/prometheus/storage/raw/index/leveldb"
+
+	"github.com/prometheus/prometheus/model"
+	"github.com/prometheus/prometheus/storage"
+	"github.com/prometheus/prometheus/storage/raw/leveldb"
+	"github.com/prometheus/prometheus/utility"
 )
 
-const (
+const sortConcurrency = 2
-	sortConcurrency = 2
-)
 
 type LevelDBMetricPersistence struct {
 	CurationRemarks         *leveldb.LevelDBPersistence
@@ -302,7 +302,7 @@ func (l *LevelDBMetricPersistence) indexLabelNames(metrics map[model.Fingerprint
 			value.Member = append(value.Member, fingerprint.ToDTO())
 		}
 
-		batch.Put(coding.NewPBEncoder(key), coding.NewPBEncoder(value))
+		batch.Put(key, value)
 	}
 
 	err = l.labelNameToFingerprints.Commit(batch)
@@ -375,7 +375,7 @@ func (l *LevelDBMetricPersistence) indexLabelPairs(metrics map[model.Fingerprint
 			value.Member = append(value.Member, fingerprint.ToDTO())
 		}
 
-		batch.Put(coding.NewPBEncoder(key), coding.NewPBEncoder(value))
+		batch.Put(key, value)
 	}
 
 	err = l.labelSetToFingerprints.Commit(batch)
@@ -401,9 +401,7 @@ func (l *LevelDBMetricPersistence) indexFingerprints(metrics map[model.Fingerpri
 	defer batch.Close()
 
 	for fingerprint, metric := range metrics {
-		key := coding.NewPBEncoder(fingerprint.ToDTO())
+		batch.Put(fingerprint.ToDTO(), model.MetricToDTO(metric))
-		value := coding.NewPBEncoder(model.MetricToDTO(metric))
-		batch.Put(key, value)
 	}
 
 	err = l.fingerprintToMetrics.Commit(batch)
@@ -414,6 +412,8 @@ func (l *LevelDBMetricPersistence) indexFingerprints(metrics map[model.Fingerpri
 	return
 }
 
+var existenceIdentity = &dto.MembershipIndexValue{}
+
 // indexMetrics takes groups of samples, determines which ones contain metrics
 // that are unknown to the storage stack, and then proceeds to update all
 // affected indices.
@@ -465,10 +465,8 @@ func (l *LevelDBMetricPersistence) indexMetrics(fingerprints map[model.Fingerpri
 	batch := leveldb.NewBatch()
 	defer batch.Close()
 
-	// WART: We should probably encode simple fingerprints.
 	for _, metric := range absentMetrics {
-		key := coding.NewPBEncoder(model.MetricToDTO(metric))
+		batch.Put(model.MetricToDTO(metric), existenceIdentity)
-		batch.Put(key, key)
 	}
 
 	err = l.metricMembershipIndex.Commit(batch)
@@ -492,27 +490,23 @@ func (l *LevelDBMetricPersistence) refreshHighWatermarks(groups map[model.Finger
 
 	mutationCount := 0
 	for fingerprint, samples := range groups {
-		keyEncoded := coding.NewPBEncoder(fingerprint.ToDTO())
 		value := &dto.MetricHighWatermark{}
 		newestSampleTimestamp := samples[len(samples)-1].Timestamp
-
+		present, err := l.MetricHighWatermarks.Get(fingerprint.ToDTO(), value)
-		raw, err := l.MetricHighWatermarks.Get(keyEncoded)
 		if err != nil {
 			return err
 		}
-
+		if !present {
-		if raw != nil {
+			continue
-			err = proto.Unmarshal(raw, value)
-			if err != nil {
-				return err
-			}
-
-			if newestSampleTimestamp.Before(time.Unix(*value.Timestamp, 0)) {
-				continue
-			}
 		}
+
+		// BUG(matt): Repace this with watermark management.
+		if newestSampleTimestamp.Before(time.Unix(*value.Timestamp, 0)) {
+			continue
+		}
+
 		value.Timestamp = proto.Int64(newestSampleTimestamp.Unix())
-		batch.Put(keyEncoded, coding.NewPBEncoder(value))
+		batch.Put(fingerprint.ToDTO(), value)
 		mutationCount++
 	}
 
@@ -583,7 +577,7 @@ func (l *LevelDBMetricPersistence) AppendSamples(samples model.Samples) (err err
 				})
 			}
 
-			samplesBatch.Put(coding.NewPBEncoder(key), coding.NewPBEncoder(value))
+			samplesBatch.Put(key, value)
 		}
 	}
 
@@ -656,8 +650,7 @@ func (l *LevelDBMetricPersistence) hasIndexMetric(dto *dto.Metric) (value bool,
 		recordOutcome(duration, err, map[string]string{operation: hasIndexMetric, result: success}, map[string]string{operation: hasIndexMetric, result: failure})
 	}(time.Now())
 
-	dtoKey := coding.NewPBEncoder(dto)
+	value, err = l.metricMembershipIndex.Has(dto)
-	value, err = l.metricMembershipIndex.Has(dtoKey)
 
 	return
 }
@@ -669,8 +662,7 @@ func (l *LevelDBMetricPersistence) HasLabelPair(dto *dto.LabelPair) (value bool,
 		recordOutcome(duration, err, map[string]string{operation: hasLabelPair, result: success}, map[string]string{operation: hasLabelPair, result: failure})
 	}(time.Now())
 
-	dtoKey := coding.NewPBEncoder(dto)
+	value, err = l.labelSetToFingerprints.Has(dto)
-	value, err = l.labelSetToFingerprints.Has(dtoKey)
 
 	return
 }
@@ -682,8 +674,7 @@ func (l *LevelDBMetricPersistence) HasLabelName(dto *dto.LabelName) (value bool,
 		recordOutcome(duration, err, map[string]string{operation: hasLabelName, result: success}, map[string]string{operation: hasLabelName, result: failure})
 	}(time.Now())
 
-	dtoKey := coding.NewPBEncoder(dto)
+	value, err = l.labelNameToFingerprints.Has(dto)
-	value, err = l.labelNameToFingerprints.Has(dtoKey)
 
 	return
 }
@@ -698,15 +689,13 @@ func (l *LevelDBMetricPersistence) GetFingerprintsForLabelSet(labelSet model.Lab
 	sets := []utility.Set{}
 
 	for _, labelSetDTO := range model.LabelSetToDTOs(&labelSet) {
-		f, err := l.labelSetToFingerprints.Get(coding.NewPBEncoder(labelSetDTO))
+		unmarshaled := &dto.FingerprintCollection{}
+		present, err := l.labelSetToFingerprints.Get(labelSetDTO, unmarshaled)
 		if err != nil {
 			return fps, err
 		}
-
+		if !present {
-		unmarshaled := &dto.FingerprintCollection{}
+			return nil, nil
-		err = proto.Unmarshal(f, unmarshaled)
-		if err != nil {
-			return fps, err
 		}
 
 		set := utility.Set{}
@@ -743,16 +732,13 @@ func (l *LevelDBMetricPersistence) GetFingerprintsForLabelName(labelName model.L
 		recordOutcome(duration, err, map[string]string{operation: getFingerprintsForLabelName, result: success}, map[string]string{operation: getFingerprintsForLabelName, result: failure})
 	}(time.Now())
 
-	raw, err := l.labelNameToFingerprints.Get(coding.NewPBEncoder(model.LabelNameToDTO(&labelName)))
-	if err != nil {
-		return
-	}
-
 	unmarshaled := &dto.FingerprintCollection{}
-
+	present, err := l.labelNameToFingerprints.Get(model.LabelNameToDTO(&labelName), unmarshaled)
-	err = proto.Unmarshal(raw, unmarshaled)
 	if err != nil {
-		return
+		return nil, err
+	}
+	if !present {
+		return nil, nil
 	}
 
 	for _, m := range unmarshaled.Member {
@@ -760,7 +746,7 @@ func (l *LevelDBMetricPersistence) GetFingerprintsForLabelName(labelName model.L
 		fps = append(fps, fp)
 	}
 
-	return
+	return fps, nil
 }
 
 func (l *LevelDBMetricPersistence) GetMetricForFingerprint(f *model.Fingerprint) (m model.Metric, err error) {
@@ -770,15 +756,13 @@ func (l *LevelDBMetricPersistence) GetMetricForFingerprint(f *model.Fingerprint)
 		recordOutcome(duration, err, map[string]string{operation: getMetricForFingerprint, result: success}, map[string]string{operation: getMetricForFingerprint, result: failure})
 	}(time.Now())
 
-	raw, err := l.fingerprintToMetrics.Get(coding.NewPBEncoder(model.FingerprintToDTO(f)))
-	if err != nil {
-		return
-	}
-
 	unmarshaled := &dto.Metric{}
-	err = proto.Unmarshal(raw, unmarshaled)
+	present, err := l.fingerprintToMetrics.Get(model.FingerprintToDTO(f), unmarshaled)
 	if err != nil {
-		return
+		return nil, err
+	}
+	if !present {
+		return nil, nil
 	}
 
 	m = model.Metric{}
@@ -787,7 +771,7 @@ func (l *LevelDBMetricPersistence) GetMetricForFingerprint(f *model.Fingerprint)
 		m[model.LabelName(*v.Name)] = model.LabelValue(*v.Value)
 	}
 
-	return
+	return m, nil
 }
 
 func (l LevelDBMetricPersistence) GetValueAtTime(f *model.Fingerprint, t time.Time) model.Values {

storage/metric/processor.go

@@ -14,14 +14,16 @@
 package metric
 
 import (
-	"code.google.com/p/goprotobuf/proto"
 	"fmt"
-	"github.com/prometheus/prometheus/coding"
+	"time"
-	"github.com/prometheus/prometheus/model"
+
+	"code.google.com/p/goprotobuf/proto"
+
 	dto "github.com/prometheus/prometheus/model/generated"
+
+	"github.com/prometheus/prometheus/model"
 	"github.com/prometheus/prometheus/storage/raw"
 	"github.com/prometheus/prometheus/storage/raw/leveldb"
-	"time"
 )
 
 // processor models a post-processing agent that performs work given a sample
@@ -153,8 +155,7 @@ func (p CompactionProcessor) Apply(sampleIterator leveldb.Iterator, samplesPersi
 
 		case len(pendingSamples)+len(unactedSamples) < p.MinimumGroupSize:
 			if !keyDropped {
-				key := coding.NewPBEncoder(sampleKey.ToDTO())
+				pendingBatch.Drop(sampleKey.ToDTO())
-				pendingBatch.Drop(key)
 				keyDropped = true
 			}
 			pendingSamples = append(pendingSamples, unactedSamples...)
@@ -165,15 +166,12 @@ func (p CompactionProcessor) Apply(sampleIterator leveldb.Iterator, samplesPersi
 		// If the number of pending writes equals the target group size
 		case len(pendingSamples) == p.MinimumGroupSize:
 			newSampleKey := pendingSamples.ToSampleKey(fingerprint)
-			key := coding.NewPBEncoder(newSampleKey.ToDTO())
+			pendingBatch.Put(newSampleKey.ToDTO(), pendingSamples.ToDTO())
-			value := coding.NewPBEncoder(pendingSamples.ToDTO())
-			pendingBatch.Put(key, value)
 			pendingMutations++
 			lastCurated = newSampleKey.FirstTimestamp.In(time.UTC)
 			if len(unactedSamples) > 0 {
 				if !keyDropped {
-					key := coding.NewPBEncoder(sampleKey.ToDTO())
+					pendingBatch.Drop(sampleKey.ToDTO())
-					pendingBatch.Drop(key)
 					keyDropped = true
 				}
 
@@ -190,8 +188,7 @@ func (p CompactionProcessor) Apply(sampleIterator leveldb.Iterator, samplesPersi
 
 		case len(pendingSamples)+len(unactedSamples) >= p.MinimumGroupSize:
 			if !keyDropped {
-				key := coding.NewPBEncoder(sampleKey.ToDTO())
+				pendingBatch.Drop(sampleKey.ToDTO())
-				pendingBatch.Drop(key)
 				keyDropped = true
 			}
 			remainder := p.MinimumGroupSize - len(pendingSamples)
@@ -211,9 +208,7 @@ func (p CompactionProcessor) Apply(sampleIterator leveldb.Iterator, samplesPersi
 	if len(unactedSamples) > 0 || len(pendingSamples) > 0 {
 		pendingSamples = append(pendingSamples, unactedSamples...)
 		newSampleKey := pendingSamples.ToSampleKey(fingerprint)
-		key := coding.NewPBEncoder(newSampleKey.ToDTO())
+		pendingBatch.Put(newSampleKey.ToDTO(), pendingSamples.ToDTO())
-		value := coding.NewPBEncoder(pendingSamples.ToDTO())
-		pendingBatch.Put(key, value)
 		pendingSamples = model.Values{}
 		pendingMutations++
 		lastCurated = newSampleKey.FirstTimestamp.In(time.UTC)
@@ -320,24 +315,20 @@ func (p DeletionProcessor) Apply(sampleIterator leveldb.Iterator, samplesPersist
 			pendingBatch = nil
 
 		case !sampleKey.MayContain(stopAt):
-			key := coding.NewPBEncoder(sampleKey.ToDTO())
+			pendingBatch.Drop(sampleKey.ToDTO())
-			pendingBatch.Drop(key)
 			lastCurated = sampleKey.LastTimestamp
 			sampleValues = model.Values{}
 			pendingMutations++
 
 		case sampleKey.MayContain(stopAt):
-			key := coding.NewPBEncoder(sampleKey.ToDTO())
+			pendingBatch.Drop(sampleKey.ToDTO())
-			pendingBatch.Drop(key)
 			pendingMutations++
 
 			sampleValues = sampleValues.TruncateBefore(stopAt)
 			if len(sampleValues) > 0 {
 				sampleKey = sampleValues.ToSampleKey(fingerprint)
 				lastCurated = sampleKey.FirstTimestamp
-				newKey := coding.NewPBEncoder(sampleKey.ToDTO())
+				pendingBatch.Put(sampleKey.ToDTO(), sampleValues.ToDTO())
-				newValue := coding.NewPBEncoder(sampleValues.ToDTO())
-				pendingBatch.Put(newKey, newValue)
 				pendingMutations++
 			} else {
 				lastCurated = sampleKey.LastTimestamp

storage/metric/processor_test.go

@@ -14,15 +14,17 @@
 package metric
 
 import (
-	"code.google.com/p/goprotobuf/proto"
 	"fmt"
-	"github.com/prometheus/prometheus/coding"
-	"github.com/prometheus/prometheus/model"
-	dto "github.com/prometheus/prometheus/model/generated"
-	"github.com/prometheus/prometheus/storage/raw/leveldb"
-	fixture "github.com/prometheus/prometheus/storage/raw/leveldb/test"
 	"testing"
 	"time"
+
+	"code.google.com/p/goprotobuf/proto"
+
+	dto "github.com/prometheus/prometheus/model/generated"
+	fixture "github.com/prometheus/prometheus/storage/raw/leveldb/test"
+
+	"github.com/prometheus/prometheus/model"
+	"github.com/prometheus/prometheus/storage/raw/leveldb"
 )
 
 type curationState struct {
@@ -56,40 +58,40 @@ type out struct {
 	sampleGroups   []sampleGroup
 }
 
-func (c curationState) Get() (key, value coding.Encoder) {
+func (c curationState) Get() (key, value proto.Message) {
 	signature, err := c.processor.Signature()
 	if err != nil {
 		panic(err)
 	}
-	key = coding.NewPBEncoder(model.CurationKey{
+	key = model.CurationKey{
 		Fingerprint:              model.NewFingerprintFromRowKey(c.fingerprint),
 		ProcessorMessageRaw:      signature,
 		ProcessorMessageTypeName: c.processor.Name(),
 		IgnoreYoungerThan:        c.ignoreYoungerThan,
-	}.ToDTO())
+	}.ToDTO()
 
-	value = coding.NewPBEncoder(model.CurationRemark{
+	value = model.CurationRemark{
 		LastCompletionTimestamp: c.lastCurated,
-	}.ToDTO())
+	}.ToDTO()
 
 	return
 }
 
-func (w watermarkState) Get() (key, value coding.Encoder) {
+func (w watermarkState) Get() (key, value proto.Message) {
-	key = coding.NewPBEncoder(model.NewFingerprintFromRowKey(w.fingerprint).ToDTO())
+	key = model.NewFingerprintFromRowKey(w.fingerprint).ToDTO()
-	value = coding.NewPBEncoder(model.NewWatermarkFromTime(w.lastAppended).ToMetricHighWatermarkDTO())
+	value = model.NewWatermarkFromTime(w.lastAppended).ToMetricHighWatermarkDTO()
 	return
 }
 
-func (s sampleGroup) Get() (key, value coding.Encoder) {
+func (s sampleGroup) Get() (key, value proto.Message) {
-	key = coding.NewPBEncoder(model.SampleKey{
+	key = model.SampleKey{
 		Fingerprint:    model.NewFingerprintFromRowKey(s.fingerprint),
 		FirstTimestamp: s.values[0].Timestamp,
 		LastTimestamp:  s.values[len(s.values)-1].Timestamp,
 		SampleCount:    uint32(len(s.values)),
-	}.ToDTO())
+	}.ToDTO()
 
-	value = coding.NewPBEncoder(s.values.ToDTO())
+	value = s.values.ToDTO()
 
 	return
 }

storage/metric/tiered.go

@@ -19,8 +19,6 @@ import (
 	"sort"
 	"time"
 
-	"code.google.com/p/goprotobuf/proto"
-
 	dto "github.com/prometheus/prometheus/model/generated"
 
 	"github.com/prometheus/prometheus/coding"
@@ -341,18 +339,12 @@ func (t *TieredStorage) seriesTooOld(f *model.Fingerprint, i time.Time) (bool, e
 
 	wm, ok := t.wmCache.Get(f)
 	if !ok {
-		rowKey := coding.NewPBEncoder(f.ToDTO())
+		value := &dto.MetricHighWatermark{}
-		raw, err := t.DiskStorage.MetricHighWatermarks.Get(rowKey)
+		present, err := t.DiskStorage.MetricHighWatermarks.Get(f.ToDTO(), value)
 		if err != nil {
 			return false, err
 		}
-		if raw != nil {
+		if present {
-			value := &dto.MetricHighWatermark{}
-			err = proto.Unmarshal(raw, value)
-			if err != nil {
-				return false, err
-			}
-
 			wmTime := time.Unix(*value.Timestamp, 0).UTC()
 			t.wmCache.Set(f, &Watermarks{High: wmTime})
 			return wmTime.Before(i), nil

storage/raw/index/interface.go

@@ -13,13 +13,11 @@
 
 package index
 
-import (
+import "code.google.com/p/goprotobuf/proto"
-	"github.com/prometheus/prometheus/coding"
-)
 
 type MembershipIndex interface {
-	Has(key coding.Encoder) (bool, error)
+	Has(key proto.Message) (bool, error)
-	Put(key coding.Encoder) error
+	Put(key proto.Message) error
-	Drop(key coding.Encoder) error
+	Drop(key proto.Message) error
 	Close()
 }

storage/raw/index/leveldb/leveldb.go

@@ -14,20 +14,15 @@
 package leveldb
 
 import (
-	"github.com/prometheus/prometheus/coding"
+	"code.google.com/p/goprotobuf/proto"
+
+	dto "github.com/prometheus/prometheus/model/generated"
+
 	"github.com/prometheus/prometheus/storage/raw"
 	"github.com/prometheus/prometheus/storage/raw/leveldb"
 )
 
-type indexValue struct{}
+var existenceValue = &dto.MembershipIndexValue{}
-
-func (i *indexValue) MustEncode() []byte {
-	return []byte{}
-}
-
-var (
-	existenceValue = &indexValue{}
-)
 
 type LevelDBMembershipIndex struct {
 	persistence *leveldb.LevelDBPersistence
@@ -37,16 +32,16 @@ func (l *LevelDBMembershipIndex) Close() {
 	l.persistence.Close()
 }
 
-func (l *LevelDBMembershipIndex) Has(key coding.Encoder) (bool, error) {
+func (l *LevelDBMembershipIndex) Has(k proto.Message) (bool, error) {
-	return l.persistence.Has(key)
+	return l.persistence.Has(k)
 }
 
-func (l *LevelDBMembershipIndex) Drop(key coding.Encoder) error {
+func (l *LevelDBMembershipIndex) Drop(k proto.Message) error {
-	return l.persistence.Drop(key)
+	return l.persistence.Drop(k)
 }
 
-func (l *LevelDBMembershipIndex) Put(key coding.Encoder) error {
+func (l *LevelDBMembershipIndex) Put(k proto.Message) error {
-	return l.persistence.Put(key, existenceValue)
+	return l.persistence.Put(k, existenceValue)
 }
 
 func NewLevelDBMembershipIndex(storageRoot string, cacheCapacity, bitsPerBloomFilterEncoded int) (i *LevelDBMembershipIndex, err error) {

storage/raw/interface.go

@@ -14,7 +14,8 @@
 package raw
 
 import (
-	"github.com/prometheus/prometheus/coding"
+	"code.google.com/p/goprotobuf/proto"
+
 	"github.com/prometheus/prometheus/storage"
 )
 
@@ -25,14 +26,14 @@ type Persistence interface {
 	// persistence.
 	Close()
 	// Has informs the user whether a given key exists in the database.
-	Has(key coding.Encoder) (bool, error)
+	Has(key proto.Message) (bool, error)
 	// Get retrieves the key from the database if it exists or returns nil if
 	// it is absent.
-	Get(key coding.Encoder) ([]byte, error)
+	Get(key, value proto.Message) (present bool, err error)
 	// Drop removes the key from the database.
-	Drop(key coding.Encoder) error
+	Drop(key proto.Message) error
 	// Put sets the key to a given value.
-	Put(key, value coding.Encoder) error
+	Put(key, value proto.Message) error
 	// ForEach is responsible for iterating through all records in the database
 	// until one of the following conditions are met:
 	//
@@ -41,7 +42,7 @@ type Persistence interface {
 	// 3.) A FilterResult of STOP is emitted by the Filter.
 	//
 	// Decoding errors for an entity cause that entity to be skipped.
-	ForEach(decoder storage.RecordDecoder, filter storage.RecordFilter, operator storage.RecordOperator) (scannedEntireCorpus bool, err error)
+	ForEach(storage.RecordDecoder, storage.RecordFilter, storage.RecordOperator) (scannedEntireCorpus bool, err error)
 	// Commit applies the Batch operations to the database.
 	Commit(Batch) error
 }
@@ -54,7 +55,7 @@ type Batch interface {
 	// batch mutation.
 	Close()
 	// Put follows the same protocol as Persistence.Put.
-	Put(key, value coding.Encoder)
+	Put(key, value proto.Message)
 	// Drop follows the same protocol as Persistence.Drop.
-	Drop(key coding.Encoder)
+	Drop(key proto.Message)
 }

storage/raw/leveldb/batch.go

@@ -15,7 +15,10 @@ package leveldb
 
 import (
 	"fmt"
+
+	"code.google.com/p/goprotobuf/proto"
 	"github.com/jmhodges/levigo"
+
 	"github.com/prometheus/prometheus/coding"
 )
 
@@ -31,25 +34,23 @@ func NewBatch() *batch {
 	}
 }
 
-func (b *batch) Drop(key coding.Encoder) {
+func (b *batch) Drop(key proto.Message) {
-	keyEncoded := key.MustEncode()
+	b.batch.Delete(coding.NewPBEncoder(key).MustEncode())
-	b.drops++
 
-	b.batch.Delete(keyEncoded)
+	b.drops++
 }
 
-func (b *batch) Put(key, value coding.Encoder) {
+func (b *batch) Put(key, value proto.Message) {
-	keyEncoded := key.MustEncode()
+	b.batch.Put(coding.NewPBEncoder(key).MustEncode(), coding.NewPBEncoder(value).MustEncode())
-	valueEncoded := value.MustEncode()
+
 	b.puts++
 
-	b.batch.Put(keyEncoded, valueEncoded)
 }
 
-func (b batch) Close() {
+func (b *batch) Close() {
 	b.batch.Close()
 }
 
-func (b batch) String() string {
+func (b *batch) String() string {
 	return fmt.Sprintf("LevelDB batch with %d puts and %d drops.", b.puts, b.drops)
 }

storage/raw/leveldb/leveldb.go

@@ -16,11 +16,14 @@ package leveldb
 import (
 	"flag"
 	"fmt"
+	"time"
+
+	"code.google.com/p/goprotobuf/proto"
 	"github.com/jmhodges/levigo"
+
 	"github.com/prometheus/prometheus/coding"
 	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/storage/raw"
-	"time"
 )
 
 var (
@@ -250,38 +253,37 @@ func (l *LevelDBPersistence) Close() {
 	return
 }
 
-func (l *LevelDBPersistence) Get(value coding.Encoder) (b []byte, err error) {
+func (l *LevelDBPersistence) Get(k, v proto.Message) (bool, error) {
-	key := value.MustEncode()
+	raw, err := l.storage.Get(l.readOptions, coding.NewPBEncoder(k).MustEncode())
-
-	return l.storage.Get(l.readOptions, key)
-}
-
-func (l *LevelDBPersistence) Has(value coding.Encoder) (h bool, err error) {
-	raw, err := l.Get(value)
 	if err != nil {
-		return
+		return false, err
+	}
+	if raw == nil {
+		return false, nil
 	}
 
-	h = raw != nil
+	if v == nil {
+		return true, nil
+	}
 
-	return
+	err = proto.Unmarshal(raw, v)
+	if err != nil {
+		return true, err
+	}
+
+	return true, nil
 }
 
-func (l *LevelDBPersistence) Drop(value coding.Encoder) (err error) {
+func (l *LevelDBPersistence) Has(k proto.Message) (has bool, err error) {
-	key := value.MustEncode()
+	return l.Get(k, nil)
-	err = l.storage.Delete(l.writeOptions, key)
-
-	return
 }
 
-func (l *LevelDBPersistence) Put(key, value coding.Encoder) (err error) {
+func (l *LevelDBPersistence) Drop(k proto.Message) error {
-	keyEncoded := key.MustEncode()
+	return l.storage.Delete(l.writeOptions, coding.NewPBEncoder(k).MustEncode())
+}
 
-	valueEncoded := value.MustEncode()
+func (l *LevelDBPersistence) Put(key, value proto.Message) error {
-
+	return l.storage.Put(l.writeOptions, coding.NewPBEncoder(key).MustEncode(), coding.NewPBEncoder(value).MustEncode())
-	err = l.storage.Put(l.writeOptions, keyEncoded, valueEncoded)
-
-	return
 }
 
 func (l *LevelDBPersistence) Commit(b raw.Batch) (err error) {

storage/raw/leveldb/test/fixtures.go

@@ -14,7 +14,8 @@
 package test
 
 import (
-	"github.com/prometheus/prometheus/coding"
+	"code.google.com/p/goprotobuf/proto"
+
 	"github.com/prometheus/prometheus/storage/raw/leveldb"
 	"github.com/prometheus/prometheus/utility/test"
 )
@@ -28,7 +29,7 @@ type (
 	// Pair models a prospective (key, value) double that will be committed to
 	// a database.
 	Pair interface {
-		Get() (key, value coding.Encoder)
+		Get() (key, value proto.Message)
 	}
 
 	// Pairs models a list of Pair for disk committing.
@@ -47,7 +48,7 @@ type (
 		// data to build.
 		HasNext() (has bool)
 		// Next emits the next (key, value) double for storage.
-		Next() (key coding.Encoder, value coding.Encoder)
+		Next() (key, value proto.Message)
 	}
 
 	preparer struct {
@@ -88,7 +89,7 @@ func (f cassetteFactory) HasNext() bool {
 	return f.index < f.count
 }
 
-func (f *cassetteFactory) Next() (key, value coding.Encoder) {
+func (f *cassetteFactory) Next() (key, value proto.Message) {
 	key, value = f.pairs[f.index].Get()
 
 	f.index++

tools/dumper/main.go

@@ -19,17 +19,20 @@
 package main
 
 import (
-	"code.google.com/p/goprotobuf/proto"
 	"encoding/csv"
 	"flag"
 	"fmt"
-	"github.com/prometheus/prometheus/model"
-	dto "github.com/prometheus/prometheus/model/generated"
-	"github.com/prometheus/prometheus/storage"
-	"github.com/prometheus/prometheus/storage/metric"
 	"log"
 	"os"
 	"strconv"
+
+	"code.google.com/p/goprotobuf/proto"
+
+	dto "github.com/prometheus/prometheus/model/generated"
+
+	"github.com/prometheus/prometheus/model"
+	"github.com/prometheus/prometheus/storage"
+	"github.com/prometheus/prometheus/storage/metric"
 )
 
 var (

web/web.go

@@ -102,7 +102,6 @@ func getEmbeddedTemplate(name string) (*template.Template, error) {
 	return t, nil
 }
 
-
 func getTemplate(name string) (t *template.Template, err error) {
 	if *useLocalAssets {
 		t, err = getLocalTemplate(name)