Hook OpenMetrics parser into scraping.

Extend metadata api to support units.

Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>

docs/querying/api.md

@@ -499,7 +499,8 @@ curl -G http://localhost:9091/api/v1/targets/metadata \
         "job": "prometheus"
       },
       "type": "gauge",
-      "help": "Number of goroutines that currently exist."
+      "help": "Number of goroutines that currently exist.",
+      "unit": ""
     },
     {
       "target": {
@@ -507,7 +508,8 @@ curl -G http://localhost:9091/api/v1/targets/metadata \
         "job": "prometheus"
       },
       "type": "gauge",
-      "help": "Number of goroutines that currently exist."
+      "help": "Number of goroutines that currently exist.",
+      "unit": ""
     }
   ]
 }
@@ -530,7 +532,8 @@ curl -G http://localhost:9091/api/v1/targets/metadata \
       },
       "metric": "prometheus_treecache_zookeeper_failures_total",
       "type": "counter",
-      "help": "The total number of ZooKeeper failures."
+      "help": "The total number of ZooKeeper failures.",
+      "unit": ""
     },
     {
       "target": {
@@ -539,7 +542,8 @@ curl -G http://localhost:9091/api/v1/targets/metadata \
       },
       "metric": "prometheus_tsdb_reloads_total",
       "type": "counter",
-      "help": "Number of times the database reloaded block data from disk."
+      "help": "Number of times the database reloaded block data from disk.",
+      "unit": ""
     },
     // ...
   ]

pkg/textparse/interface.go

@@ -14,6 +14,8 @@
 package textparse
 
 import (
+	"mime"
+
 	"github.com/prometheus/prometheus/pkg/labels"
 )
 
@@ -55,6 +57,10 @@ type Parser interface {
 
 // New returns a new parser of the byte slice.
 func New(b []byte, contentType string) Parser {
+	mediaType, _, err := mime.ParseMediaType(contentType)
+	if err == nil && mediaType == "application/openmetrics-text" {
+		return NewOMParser(b)
+	}
 	return NewPromParser(b)
 }
 

scrape/scrape.go

@@ -451,7 +451,7 @@ type targetScraper struct {
 	buf   *bufio.Reader
 }
 
-const acceptHeader = `text/plain;version=0.0.4;q=1,*/*;q=0.1`
+const acceptHeader = `application/openmetrics-text; version=0.0.1,text/plain;version=0.0.4;q=0.5,*/*;q=0.1`
 
 var userAgentHeader = fmt.Sprintf("Prometheus/%s", version.Version)
 
@@ -565,6 +565,7 @@ type metaEntry struct {
 	lastIter uint64 // Last scrape iteration the entry was observed at.
 	typ      textparse.MetricType
 	help     string
+	unit     string
 }
 
 func newScrapeCache() *scrapeCache {
@@ -659,7 +660,7 @@ func (c *scrapeCache) setType(metric []byte, t textparse.MetricType) {
 
 	e, ok := c.metadata[yoloString(metric)]
 	if !ok {
-		e = &metaEntry{typ: textparse.MetricTypeUntyped}
+		e = &metaEntry{typ: textparse.MetricTypeUnknown}
 		c.metadata[string(metric)] = e
 	}
 	e.typ = t
@@ -673,7 +674,7 @@ func (c *scrapeCache) setHelp(metric, help []byte) {
 
 	e, ok := c.metadata[yoloString(metric)]
 	if !ok {
-		e = &metaEntry{typ: textparse.MetricTypeUntyped}
+		e = &metaEntry{typ: textparse.MetricTypeUnknown}
 		c.metadata[string(metric)] = e
 	}
 	if e.help != yoloString(help) {
@@ -684,6 +685,22 @@ func (c *scrapeCache) setHelp(metric, help []byte) {
 	c.metaMtx.Unlock()
 }
 
+func (c *scrapeCache) setUnit(metric, unit []byte) {
+	c.metaMtx.Lock()
+
+	e, ok := c.metadata[yoloString(metric)]
+	if !ok {
+		e = &metaEntry{typ: textparse.MetricTypeUnknown}
+		c.metadata[string(metric)] = e
+	}
+	if e.unit != yoloString(unit) {
+		e.unit = string(unit)
+	}
+	e.lastIter = c.iter
+
+	c.metaMtx.Unlock()
+}
+
 func (c *scrapeCache) getMetadata(metric string) (MetricMetadata, bool) {
 	c.metaMtx.Lock()
 	defer c.metaMtx.Unlock()
@@ -696,6 +713,7 @@ func (c *scrapeCache) getMetadata(metric string) (MetricMetadata, bool) {
 		Metric: metric,
 		Type:   m.typ,
 		Help:   m.help,
+		Unit:   m.unit,
 	}, true
 }
 
@@ -710,6 +728,7 @@ func (c *scrapeCache) listMetadata() []MetricMetadata {
 			Metric: m,
 			Type:   e.typ,
 			Help:   e.help,
+			Unit:   e.unit,
 		})
 	}
 	return res
@@ -951,6 +970,9 @@ loop:
 		case textparse.EntryHelp:
 			sl.cache.setHelp(p.Help())
 			continue
+		case textparse.EntryUnit:
+			sl.cache.setUnit(p.Unit())
+			continue
 		case textparse.EntryComment:
 			continue
 		default:

scrape/scrape_test.go

@@ -615,13 +615,13 @@ func TestScrapeLoopMetadata(t *testing.T) {
 	)
 	defer cancel()
 
-	total, _, err := sl.append([]byte(`
+	total, _, err := sl.append([]byte(`# TYPE test_metric counter
-# TYPE test_metric counter
 # HELP test_metric some help text
-# other comment
+# UNIT test_metric metric
 test_metric 1
 # TYPE test_metric_no_help gauge
-# HELP test_metric_no_type other help text`), "", time.Now())
+# HELP test_metric_no_type other help text
+# EOF`), "application/openmetrics-text", time.Now())
 	testutil.Ok(t, err)
 	testutil.Equals(t, 1, total)
 
@@ -629,16 +629,19 @@ test_metric 1
 	testutil.Assert(t, ok, "expected metadata to be present")
 	testutil.Assert(t, textparse.MetricTypeCounter == md.Type, "unexpected metric type")
 	testutil.Equals(t, "some help text", md.Help)
+	testutil.Equals(t, "metric", md.Unit)
 
 	md, ok = cache.getMetadata("test_metric_no_help")
 	testutil.Assert(t, ok, "expected metadata to be present")
 	testutil.Assert(t, textparse.MetricTypeGauge == md.Type, "unexpected metric type")
 	testutil.Equals(t, "", md.Help)
+	testutil.Equals(t, "", md.Unit)
 
 	md, ok = cache.getMetadata("test_metric_no_type")
 	testutil.Assert(t, ok, "expected metadata to be present")
-	testutil.Assert(t, textparse.MetricTypeUntyped == md.Type, "unexpected metric type")
+	testutil.Assert(t, textparse.MetricTypeUnknown == md.Type, "unexpected metric type")
 	testutil.Equals(t, "other help text", md.Help)
+	testutil.Equals(t, "", md.Unit)
 }
 
 func TestScrapeLoopRunCreatesStaleMarkersOnFailedScrape(t *testing.T) {
@@ -1177,8 +1180,8 @@ func TestTargetScraperScrapeOK(t *testing.T) {
 	server := httptest.NewServer(
 		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			accept := r.Header.Get("Accept")
-			if !strings.HasPrefix(accept, "text/plain;") {
+			if !strings.HasPrefix(accept, "application/openmetrics-text;") {
-				t.Errorf("Expected Accept header to prefer text/plain, got %q", accept)
+				t.Errorf("Expected Accept header to prefer application/openmetrics-text, got %q", accept)
 			}
 
 			timeout := r.Header.Get("X-Prometheus-Scrape-Timeout-Seconds")

scrape/target.go

@@ -85,6 +85,7 @@ type MetricMetadata struct {
 	Metric string
 	Type   textparse.MetricType
 	Help   string
+	Unit   string
 }
 
 func (t *Target) MetadataList() []MetricMetadata {

web/api/v1/api.go

@@ -573,6 +573,7 @@ Outer:
 						Metric: md.Metric,
 						Type:   md.Type,
 						Help:   md.Help,
+						Unit:   md.Unit,
 					})
 				}
 				continue
@@ -583,6 +584,7 @@ Outer:
 					Target: t.Labels(),
 					Type:   md.Type,
 					Help:   md.Help,
+					Unit:   md.Unit,
 				})
 			}
 		}
@@ -598,6 +600,7 @@ type metricMetadata struct {
 	Metric string               `json:"metric,omitempty"`
 	Type   textparse.MetricType `json:"type"`
 	Help   string               `json:"help"`
+	Unit   string               `json:"unit"`
 }
 
 // AlertmanagerDiscovery has all the active Alertmanagers.