mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2024-11-09 23:24:05 -08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Added the remote read histogram (#7334)

Browse source 
change remote read queries total metric to a histogram and add read requests counter with status code

Signed-off-by: njingco <jingco.nicole@gmail.com>
This commit is contained in:
Nicole J 2020-06-16 07:11:41 -07:00 committed by GitHub
parent 3c753aba5f
commit d5a8f2afc4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 29 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
storage/remote/client.go
View file

@ -21,6 +21,7 @@ import (
"io"
"io/ioutil"
"net/http"
"strconv"
"strings"
"time"
@ -28,6 +29,7 @@ import (
"github.com/golang/snappy"
"github.com/opentracing/opentracing-go"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
config_util "github.com/prometheus/common/config"
"github.com/prometheus/common/model"
"github.com/prometheus/common/version"
@ -40,6 +42,16 @@ const maxErrMsgLen = 256
var userAgent = fmt.Sprintf("Prometheus/%s", version.Version)
var remoteReadQueriesTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "read_queries_total",
Help: "The total number of remote read queries.",
},
[]string{remoteName, endpoint, "code"},
)
// Client allows reading and writing from/to a remote HTTP endpoint.
type Client struct {
remoteName string // Used to differentiate clients in metrics.
@ -55,6 +67,10 @@ type ClientConfig struct {
HTTPClientConfig config_util.HTTPClientConfig
}
func init() {
prometheus.MustRegister(remoteReadQueriesTotal)
}
// NewClient creates a new Client.
func NewClient(remoteName string, conf *ClientConfig) (*Client, error) {
httpClient, err := config_util.NewClientFromConfig(conf.HTTPClientConfig, "remote_storage", false)
@ -193,6 +209,9 @@ func (c *Client) Read(ctx context.Context, query *prompb.Query) (*prompb.QueryRe
httpResp.Body.Close()
}()
remoteReadTotalCounter := remoteReadQueriesTotal.WithLabelValues(c.remoteName, c.url.String(), strconv.Itoa(httpResp.StatusCode))
remoteReadTotalCounter.Inc()
compressed, err = ioutil.ReadAll(httpResp.Body)
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf("error reading response. HTTP status code: %s", httpResp.Status))

19
storage/remote/read.go
View file

@ -16,6 +16,7 @@ package remote
import (
"context"
"fmt"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/prometheus/pkg/labels"
@ -32,26 +33,26 @@ var remoteReadQueries = prometheus.NewGaugeVec(
[]string{remoteName, endpoint},
)
var remoteReadQueriesTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{
var remoteReadQueriesHistogram = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "remote_read_queries_total",
Help: "The total number of remote read queries.",
Name: "read_request_duration_seconds",
Help: "Histogram of the latency for remote read requests.",
Buckets: append(prometheus.DefBuckets, 25, 60),
},
[]string{remoteName, endpoint},
)
func init() {
prometheus.MustRegister(remoteReadQueries)
prometheus.MustRegister(remoteReadQueriesTotal)
prometheus.MustRegister(remoteReadQueries, remoteReadQueriesHistogram)
}
// QueryableClient returns a storage.Queryable which queries the given
// Client to select series sets.
func QueryableClient(c *Client) storage.Queryable {
remoteReadQueries.WithLabelValues(c.remoteName, c.url.String())
remoteReadQueriesTotal.WithLabelValues(c.remoteName, c.url.String())
remoteReadQueriesHistogram.WithLabelValues(c.remoteName, c.url.String())
return storage.QueryableFunc(func(ctx context.Context, mint, maxt int64) (storage.Querier, error) {
return &querier{
@ -81,8 +82,8 @@ func (q *querier) Select(sortSeries bool, hints *storage.SelectHints, matchers .
remoteReadGauge.Inc()
defer remoteReadGauge.Dec()
remoteReadTotalCounter := remoteReadQueriesTotal.WithLabelValues(q.client.remoteName, q.client.url.String())
remoteReadTotalCounter.Inc()
remoteReadQueriesHistogram := remoteReadQueriesHistogram.WithLabelValues(q.client.remoteName, q.client.url.String())
remoteReadQueriesHistogram.Observe(time.Since(time.Now()).Seconds())
res, err := q.client.Read(q.ctx, query)
if err != nil {