Added the remote read histogram (#7334)

change remote read queries total metric to a histogram and add read requests counter with status code

Signed-off-by: njingco <jingco.nicole@gmail.com>

storage/remote/client.go

@@ -21,6 +21,7 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"strconv"
 	"strings"
 	"time"
 
@@ -28,6 +29,7 @@ import (
 	"github.com/golang/snappy"
 	"github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 	config_util "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/version"
@@ -40,6 +42,16 @@ const maxErrMsgLen = 256
 
 var userAgent = fmt.Sprintf("Prometheus/%s", version.Version)
 
+var remoteReadQueriesTotal = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Namespace: namespace,
+		Subsystem: subsystem,
+		Name:      "read_queries_total",
+		Help:      "The total number of remote read queries.",
+	},
+	[]string{remoteName, endpoint, "code"},
+)
+
 // Client allows reading and writing from/to a remote HTTP endpoint.
 type Client struct {
 	remoteName string // Used to differentiate clients in metrics.
@@ -55,6 +67,10 @@ type ClientConfig struct {
 	HTTPClientConfig config_util.HTTPClientConfig
 }
 
+func init() {
+	prometheus.MustRegister(remoteReadQueriesTotal)
+}
+
 // NewClient creates a new Client.
 func NewClient(remoteName string, conf *ClientConfig) (*Client, error) {
 	httpClient, err := config_util.NewClientFromConfig(conf.HTTPClientConfig, "remote_storage", false)
@@ -193,6 +209,9 @@ func (c *Client) Read(ctx context.Context, query *prompb.Query) (*prompb.QueryRe
 		httpResp.Body.Close()
 	}()
 
+	remoteReadTotalCounter := remoteReadQueriesTotal.WithLabelValues(c.remoteName, c.url.String(), strconv.Itoa(httpResp.StatusCode))
+	remoteReadTotalCounter.Inc()
+
 	compressed, err = ioutil.ReadAll(httpResp.Body)
 	if err != nil {
 		return nil, errors.Wrap(err, fmt.Sprintf("error reading response. HTTP status code: %s", httpResp.Status))

storage/remote/read.go

@@ -16,6 +16,7 @@ package remote
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/prometheus/pkg/labels"
@@ -32,26 +33,26 @@ var remoteReadQueries = prometheus.NewGaugeVec(
 	[]string{remoteName, endpoint},
 )
 
-var remoteReadQueriesTotal = prometheus.NewCounterVec(
+var remoteReadQueriesHistogram = prometheus.NewHistogramVec(
-	prometheus.CounterOpts{
+	prometheus.HistogramOpts{
 		Namespace: namespace,
 		Subsystem: subsystem,
-		Name:      "remote_read_queries_total",
+		Name:      "read_request_duration_seconds",
-		Help:      "The total number of remote read queries.",
+		Help:      "Histogram of the latency for remote read requests.",
+		Buckets:   append(prometheus.DefBuckets, 25, 60),
 	},
 	[]string{remoteName, endpoint},
 )
 
 func init() {
-	prometheus.MustRegister(remoteReadQueries)
+	prometheus.MustRegister(remoteReadQueries, remoteReadQueriesHistogram)
-	prometheus.MustRegister(remoteReadQueriesTotal)
 }
 
 // QueryableClient returns a storage.Queryable which queries the given
 // Client to select series sets.
 func QueryableClient(c *Client) storage.Queryable {
 	remoteReadQueries.WithLabelValues(c.remoteName, c.url.String())
-	remoteReadQueriesTotal.WithLabelValues(c.remoteName, c.url.String())
+	remoteReadQueriesHistogram.WithLabelValues(c.remoteName, c.url.String())
 
 	return storage.QueryableFunc(func(ctx context.Context, mint, maxt int64) (storage.Querier, error) {
 		return &querier{
@@ -81,8 +82,8 @@ func (q *querier) Select(sortSeries bool, hints *storage.SelectHints, matchers .
 	remoteReadGauge.Inc()
 	defer remoteReadGauge.Dec()
 
-	remoteReadTotalCounter := remoteReadQueriesTotal.WithLabelValues(q.client.remoteName, q.client.url.String())
+	remoteReadQueriesHistogram := remoteReadQueriesHistogram.WithLabelValues(q.client.remoteName, q.client.url.String())
-	remoteReadTotalCounter.Inc()
+	remoteReadQueriesHistogram.Observe(time.Since(time.Now()).Seconds())
 
 	res, err := q.client.Read(q.ctx, query)
 	if err != nil {