diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index 5529f0dcb..f6d936acc 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -123,6 +123,10 @@ func main() { notifier: notifier.Options{ Registerer: prometheus.DefaultRegisterer, }, + web: web.Options{ + Registerer: prometheus.DefaultRegisterer, + Gatherer: prometheus.DefaultGatherer, + }, promlogConfig: promlog.Config{}, } diff --git a/web/web.go b/web/web.go index ba2af44de..a7ee559f0 100644 --- a/web/web.go +++ b/web/web.go @@ -89,40 +89,71 @@ func withStackTracer(h http.Handler, l log.Logger) http.Handler { }) } -var ( - requestCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "prometheus_http_requests_total", - Help: "Counter of HTTP requests.", - }, - []string{"handler", "code"}, - ) - requestDuration = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "prometheus_http_request_duration_seconds", - Help: "Histogram of latencies for HTTP requests.", - Buckets: []float64{.1, .2, .4, 1, 3, 8, 20, 60, 120}, - }, - []string{"handler"}, - ) - responseSize = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "prometheus_http_response_size_bytes", - Help: "Histogram of response size for HTTP requests.", - Buckets: prometheus.ExponentialBuckets(100, 10, 8), - }, - []string{"handler"}, - ) -) +type metrics struct { + requestCounter *prometheus.CounterVec + requestDuration *prometheus.HistogramVec + responseSize *prometheus.HistogramVec +} -func init() { - prometheus.MustRegister(requestCounter, requestDuration, responseSize) +func newMetrics(r prometheus.Registerer) *metrics { + m := &metrics{ + requestCounter: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "prometheus_http_requests_total", + Help: "Counter of HTTP requests.", + }, + []string{"handler", "code"}, + ), + requestDuration: prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "prometheus_http_request_duration_seconds", + Help: "Histogram of latencies for HTTP requests.", + Buckets: []float64{.1, .2, .4, 1, 3, 8, 20, 60, 120}, + }, + []string{"handler"}, + ), + responseSize: prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "prometheus_http_response_size_bytes", + Help: "Histogram of response size for HTTP requests.", + Buckets: prometheus.ExponentialBuckets(100, 10, 8), + }, + []string{"handler"}, + ), + } + + if r != nil { + r.MustRegister(m.requestCounter, m.requestDuration, m.responseSize) + } + return m +} + +func (m *metrics) instrumentHandlerWithPrefix(prefix string) func(handlerName string, handler http.HandlerFunc) http.HandlerFunc { + return func(handlerName string, handler http.HandlerFunc) http.HandlerFunc { + return m.instrumentHandler(prefix+handlerName, handler) + } +} + +func (m *metrics) instrumentHandler(handlerName string, handler http.HandlerFunc) http.HandlerFunc { + return promhttp.InstrumentHandlerCounter( + m.requestCounter.MustCurryWith(prometheus.Labels{"handler": handlerName}), + promhttp.InstrumentHandlerDuration( + m.requestDuration.MustCurryWith(prometheus.Labels{"handler": handlerName}), + promhttp.InstrumentHandlerResponseSize( + m.responseSize.MustCurryWith(prometheus.Labels{"handler": handlerName}), + handler, + ), + ), + ) } // Handler serves various HTTP endpoints of the Prometheus server type Handler struct { logger log.Logger + gatherer prometheus.Gatherer + metrics *metrics + scrapeManager *scrape.Manager ruleManager *rules.Manager queryEngine *promql.Engine @@ -197,41 +228,31 @@ type Options struct { PageTitle string RemoteReadSampleLimit int RemoteReadConcurrencyLimit int -} -func instrumentHandlerWithPrefix(prefix string) func(handlerName string, handler http.HandlerFunc) http.HandlerFunc { - return func(handlerName string, handler http.HandlerFunc) http.HandlerFunc { - return instrumentHandler(prefix+handlerName, handler) - } -} - -func instrumentHandler(handlerName string, handler http.HandlerFunc) http.HandlerFunc { - return promhttp.InstrumentHandlerCounter( - requestCounter.MustCurryWith(prometheus.Labels{"handler": handlerName}), - promhttp.InstrumentHandlerDuration( - requestDuration.MustCurryWith(prometheus.Labels{"handler": handlerName}), - promhttp.InstrumentHandlerResponseSize( - responseSize.MustCurryWith(prometheus.Labels{"handler": handlerName}), - handler, - ), - ), - ) + Gatherer prometheus.Gatherer + Registerer prometheus.Registerer } // New initializes a new web Handler. func New(logger log.Logger, o *Options) *Handler { - router := route.New().WithInstrumentation(instrumentHandler) - cwd, err := os.Getwd() - - if err != nil { - cwd = "" - } if logger == nil { logger = log.NewNopLogger() } + m := newMetrics(o.Registerer) + router := route.New().WithInstrumentation(m.instrumentHandler) + + cwd, err := os.Getwd() + if err != nil { + cwd = "" + } + h := &Handler{ - logger: logger, + logger: logger, + + gatherer: o.Gatherer, + metrics: m, + router: router, quitCh: make(chan struct{}), reloadCh: make(chan chan error), @@ -463,7 +484,7 @@ func (h *Handler) Run(ctx context.Context) error { mux := http.NewServeMux() mux.Handle("/", h.router) - av1 := route.New().WithInstrumentation(instrumentHandlerWithPrefix("/api/v1")) + av1 := route.New().WithInstrumentation(h.metrics.instrumentHandlerWithPrefix("/api/v1")) h.apiV1.Register(av1) apiPath := "/api" if h.options.RoutePrefix != "/" { @@ -649,7 +670,7 @@ func (h *Handler) status(w http.ResponseWriter, r *http.Request) { status.StorageRetention = status.StorageRetention + h.options.TSDBCfg.MaxBytes.String() } - metrics, err := prometheus.DefaultGatherer.Gather() + metrics, err := h.gatherer.Gather() if err != nil { http.Error(w, fmt.Sprintf("error gathering runtime status: %s", err), http.StatusInternalServerError) return diff --git a/web/web_test.go b/web/web_test.go index f6975bc07..d93ec058d 100644 --- a/web/web_test.go +++ b/web/web_test.go @@ -26,6 +26,7 @@ import ( "testing" "time" + "github.com/prometheus/client_golang/prometheus" prom_testutil "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/prometheus/config" @@ -119,7 +120,8 @@ func TestReadyAndHealthy(t *testing.T) { Host: "localhost:9090", Path: "/", }, - Version: &PrometheusVersion{}, + Version: &PrometheusVersion{}, + Gatherer: prometheus.DefaultGatherer, } opts.Flags = map[string]string{} @@ -423,13 +425,14 @@ func TestHTTPMetrics(t *testing.T) { code := getReady() testutil.Equals(t, http.StatusServiceUnavailable, code) - testutil.Equals(t, 1, int(prom_testutil.ToFloat64(requestCounter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusServiceUnavailable))))) + counter := handler.metrics.requestCounter + testutil.Equals(t, 1, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusServiceUnavailable))))) handler.Ready() for range [2]int{} { code = getReady() testutil.Equals(t, http.StatusOK, code) } - testutil.Equals(t, 2, int(prom_testutil.ToFloat64(requestCounter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusOK))))) - testutil.Equals(t, 1, int(prom_testutil.ToFloat64(requestCounter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusServiceUnavailable))))) + testutil.Equals(t, 2, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusOK))))) + testutil.Equals(t, 1, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusServiceUnavailable))))) }