Merge pull request #61 from prometheus/feature/bulk-iteration-metric-names

Bulk iteration interface and UI metrics selector

api/api.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"code.google.com/p/gorest"
+	"github.com/prometheus/prometheus/storage/metric"
 	"github.com/prometheus/prometheus/utility"
 )
 
@@ -10,5 +11,14 @@ type MetricsService struct {
 
 	query      gorest.EndPoint `method:"GET" path:"/query?{expr:string}&{json:string}" output:"string"`
 	queryRange gorest.EndPoint `method:"GET" path:"/query_range?{expr:string}&{end:int64}&{range:int64}&{step:int64}" output:"string"`
+	metrics    gorest.EndPoint `method:"GET" path:"/metrics" output:"string"`
+
+	persistence metric.MetricPersistence
 	time        utility.Time
 }
+
+func NewMetricsService(p metric.MetricPersistence) *MetricsService {
+	return &MetricsService{
+		persistence: p,
+	}
+}

api/query.go

@@ -2,9 +2,11 @@ package api
 
 import (
 	"code.google.com/p/gorest"
+	"encoding/json"
 	"errors"
 	"github.com/prometheus/prometheus/rules"
 	"github.com/prometheus/prometheus/rules/ast"
+	"log"
 	"sort"
 	"time"
 )
@@ -65,3 +67,22 @@ func (serv MetricsService) QueryRange(Expr string, End int64, Range int64, Step
 	sort.Sort(matrix)
 	return ast.TypedValueToJSON(matrix, "matrix")
 }
+
+func (serv MetricsService) Metrics() string {
+	metricNames, err := serv.persistence.GetAllMetricNames()
+	rb := serv.ResponseBuilder()
+	rb.SetContentType(gorest.Application_Json)
+	if err != nil {
+		log.Printf("Error loading metric names: %v", err)
+		rb.SetResponseCode(500)
+		return err.Error()
+	}
+	sort.Strings(metricNames)
+	resultBytes, err := json.Marshal(metricNames)
+	if err != nil {
+		log.Printf("Error marshalling metric names: %v", err)
+		rb.SetResponseCode(500)
+		return err.Error()
+	}
+	return string(resultBytes)
+}

main.go

@@ -49,8 +49,7 @@ func main() {
 
 	persistence, err := leveldb.NewLevelDBMetricPersistence(*metricsStoragePath)
 	if err != nil {
-		log.Print(err)
-		os.Exit(1)
+		log.Fatalf("Error opening storage: %v", err)
 	}
 
 	go func() {
@@ -79,7 +78,7 @@ func main() {
 	}
 
 	go func() {
-		gorest.RegisterService(new(api.MetricsService))
+		gorest.RegisterService(api.NewMetricsService(persistence))
 		exporter := registry.DefaultRegistry.YieldExporter()
 
 		http.Handle("/", gorest.Handle())

static/graph.html

@@ -22,7 +22,12 @@
         <div class="grouping_box">
           <form action="/api/query_range" method="GET" class="query_form">
             <label for="expr{{id}}">Expression:</label>
-            <input type="text" name="expr" id="expr{{id}}" size="80" value="{{expr}}"><br>
+            <input type="text" name="expr" id="expr{{id}}" size="80" value="{{expr}}">
+            <select name="insert_metric">
+                <option value="">- Insert Metric -</option>
+            </select>
+
+            <br>
 
             <label for="range_input{{id}}">Range:</label>
             <input type="button" value="-" name="dec_range">

static/js/graph.js

@@ -69,6 +69,7 @@ Prometheus.Graph.prototype.initialize = function() {
   self.expr = graphWrapper.find("input[name=expr]");
   self.rangeInput = self.queryForm.find("input[name=range_input]");
   self.stacked = self.queryForm.find("input[name=stacked]");
+  self.insertMetric = self.queryForm.find("select[name=insert_metric]");
 
   self.graph = graphWrapper.find(".graph");
   self.legend = graphWrapper.find(".legend");
@@ -81,18 +82,40 @@ Prometheus.Graph.prototype.initialize = function() {
 
   self.queryForm.find("input[name=inc_range]").click(function() { self.increaseRange(); });
   self.queryForm.find("input[name=dec_range]").click(function() { self.decreaseRange(); });
+  self.insertMetric.change(function() {
+      self.expr.val(self.expr.val() + self.insertMetric.val());
+  });
   self.expr.focus(); // TODO: move to external Graph method.
 
+  self.populateInsertableMetrics();
+
   if (self.expr.val()) {
     self.submitQuery();
   }
 };
 
+Prometheus.Graph.prototype.populateInsertableMetrics = function() {
+  var self = this;
+  $.ajax({
+      method: "GET",
+      url: "/api/metrics",
+      dataType: "json",
+      success: function(json, textStatus) {
+        for (var i = 0; i < json.length; i++) {
+          self.insertMetric[0].options.add(new Option(json[i], json[i]));
+        }
+      },
+      error: function() {
+        alert("Error loading available metrics!");
+      },
+  });
+};
+
 Prometheus.Graph.prototype.onChange = function(handler) {
   this.changeHandler = handler;
 };
 
-Prometheus.Graph.prototype.getOptions = function(handler) {
+Prometheus.Graph.prototype.getOptions = function() {
   var self = this;
   var options = {};
 
@@ -307,8 +330,6 @@ function storeGraphOptionsInUrl(options) {
   var allGraphsOptions = [];
   for (var i = 0; i < graphs.length; i++) {
     allGraphsOptions.push(graphs[i].getOptions());
-    console.log(graphs[i].id);
-    console.log(graphs[i].getOptions());
   }
   var optionsJSON = JSON.stringify(allGraphsOptions);
   window.location.hash = encodeURIComponent(optionsJSON);

storage/interface.go

@@ -0,0 +1,63 @@
+// Copyright 2012 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+// RecordDecoder decodes each key-value pair in the database. The protocol
+// around it makes the assumption that the underlying implementation is
+// concurrency safe.
+type RecordDecoder interface {
+	DecodeKey(in interface{}) (out interface{}, err error)
+	DecodeValue(in interface{}) (out interface{}, err error)
+}
+
+// FilterResult describes the record matching and scanning behavior for the
+// database.
+type FilterResult int
+
+const (
+	// Stop scanning the database.
+	STOP FilterResult = iota
+	// Skip this record but continue scanning.
+	SKIP
+	// Accept this record for the Operator.
+	ACCEPT
+)
+
+type OperatorErrorType int
+
+type OperatorError struct {
+	error
+	Continuable bool
+}
+
+// Filter is responsible for controlling the behavior of the database scan
+// process and determines the disposition of various records.
+//
+// The protocol around it makes the assumption that the underlying
+// implementation is concurrency safe.
+type RecordFilter interface {
+	// Filter receives the key and value as decoded from the RecordDecoder type.
+	Filter(key, value interface{}) (filterResult FilterResult)
+}
+
+// RecordOperator is responsible for taking action upon each entity that is
+// passed to it.
+//
+// The protocol around it makes the assumption that the underlying
+// implementation is concurrency safe.
+type RecordOperator interface {
+	// Take action on a given record. If the action returns an error, the entire
+	// scan process stops.
+	Operate(key, value interface{}) (err *OperatorError)
+}

storage/metric/interface.go

@@ -46,6 +46,8 @@ type MetricPersistence interface {
 	GetBoundaryValues(*model.Metric, *model.Interval, *StalenessPolicy) (*model.Sample, *model.Sample, error)
 	GetRangeValues(*model.Metric, *model.Interval, *StalenessPolicy) (*model.SampleSet, error)
 
+	GetAllMetricNames() ([]string, error)
+
 	// DIAGNOSTIC FUNCTIONS PENDING DELETION BELOW HERE
 
 	GetAllLabelNames() ([]string, error)

storage/metric/leveldb/reading.go

@@ -15,10 +15,12 @@ package leveldb
 
 import (
 	"code.google.com/p/goprotobuf/proto"
+	"errors"
 	"github.com/prometheus/prometheus/coding"
 	"github.com/prometheus/prometheus/coding/indexable"
 	"github.com/prometheus/prometheus/model"
 	dto "github.com/prometheus/prometheus/model/generated"
+	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/storage/metric"
 	"github.com/prometheus/prometheus/utility"
 	"time"
@@ -626,3 +628,58 @@ func (l *LevelDBMetricPersistence) GetRangeValues(m *model.Metric, i *model.Inte
 
 	return
 }
+
+type MetricKeyDecoder struct{}
+
+func (d *MetricKeyDecoder) DecodeKey(in interface{}) (out interface{}, err error) {
+	unmarshaled := &dto.LabelPair{}
+	err = proto.Unmarshal(in.([]byte), unmarshaled)
+	if err != nil {
+		return
+	}
+	return unmarshaled, nil
+}
+
+func (d *MetricKeyDecoder) DecodeValue(in interface{}) (out interface{}, err error) {
+	return
+}
+
+type AcceptAllFilter struct{}
+
+func (f *AcceptAllFilter) Filter(key, value interface{}) (filterResult storage.FilterResult) {
+	return storage.ACCEPT
+}
+
+type CollectMetricNamesOp struct {
+	MetricNameMap map[string]bool
+}
+
+func (op *CollectMetricNamesOp) Operate(key, value interface{}) (err *storage.OperatorError) {
+	unmarshaled, ok := key.(*dto.LabelPair)
+	if !ok {
+		return &storage.OperatorError{
+			error:       errors.New("Key is not of correct type"),
+			Continuable: true,
+		}
+	}
+	if *unmarshaled.Name == "name" {
+		op.MetricNameMap[*unmarshaled.Value] = true
+	}
+	return
+}
+
+func (l *LevelDBMetricPersistence) GetAllMetricNames() (metricNames []string, err error) {
+	metricNamesOp := &CollectMetricNamesOp{
+		MetricNameMap: map[string]bool{},
+	}
+
+	_, err = l.labelSetToFingerprints.ForEach(&MetricKeyDecoder{}, &AcceptAllFilter{}, metricNamesOp)
+	if err != nil {
+		return
+	}
+
+	for labelName := range metricNamesOp.MetricNameMap {
+		metricNames = append(metricNames, labelName)
+	}
+	return
+}

storage/raw/interface.go

@@ -15,6 +15,7 @@ package raw
 
 import (
 	"github.com/prometheus/prometheus/coding"
+	"github.com/prometheus/prometheus/storage"
 )
 
 type Pair struct {
@@ -22,11 +23,25 @@ type Pair struct {
 	Right []byte
 }
 
+type EachFunc func(pair *Pair)
+
 type Persistence interface {
 	Has(key coding.Encoder) (bool, error)
 	Get(key coding.Encoder) ([]byte, error)
-	GetAll() ([]Pair, error)
 	Drop(key coding.Encoder) error
 	Put(key, value coding.Encoder) error
 	Close() error
+
+	// ForEach is responsible for iterating through all records in the database
+	// until one of the following conditions are met:
+	//
+	// 1.) A system anomaly in the database scan.
+	// 2.) The last record in the database is reached.
+	// 3.) A FilterResult of STOP is emitted by the Filter.
+	//
+	// Decoding errors for an entity cause that entity to be skipped.
+	ForEach(decoder storage.RecordDecoder, filter storage.RecordFilter, operator storage.RecordOperator) (scannedEntireCorpus bool, err error)
+
+	// Pending removal.
+	GetAll() ([]Pair, error)
 }

storage/raw/leveldb/leveldb.go

@@ -17,6 +17,7 @@ import (
 	"flag"
 	"github.com/jmhodges/levigo"
 	"github.com/prometheus/prometheus/coding"
+	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/storage/raw"
 	"io"
 )
@@ -230,3 +231,44 @@ func (l *LevelDBPersistence) GetIterator() (i *levigo.Iterator, c io.Closer, err
 
 	return
 }
+
+func (l *LevelDBPersistence) ForEach(decoder storage.RecordDecoder, filter storage.RecordFilter, operator storage.RecordOperator) (scannedEntireCorpus bool, err error) {
+	iterator, closer, err := l.GetIterator()
+	if err != nil {
+		return
+	}
+	defer closer.Close()
+
+	for iterator.SeekToFirst(); iterator.Valid(); iterator.Next() {
+		err = iterator.GetError()
+		if err != nil {
+			return
+		}
+
+		decodedKey, decodeErr := decoder.DecodeKey(iterator.Key())
+		if decodeErr != nil {
+			continue
+		}
+		decodedValue, decodeErr := decoder.DecodeValue(iterator.Value())
+		if decodeErr != nil {
+			continue
+		}
+
+		switch filter.Filter(decodedKey, decodedValue) {
+		case storage.STOP:
+			return
+		case storage.SKIP:
+			continue
+		case storage.ACCEPT:
+			opErr := operator.Operate(decodedKey, decodedValue)
+			if opErr != nil {
+				if opErr.Continuable {
+					continue
+				}
+				break
+			}
+		}
+	}
+	scannedEntireCorpus = true
+	return
+}