mirror of
https://github.com/prometheus/prometheus.git
synced 2024-11-09 23:24:05 -08:00
Merge pull request #4458 from FUSAKLA/k8s-sd-add-metrics
feat: added more k8s SD metrics
This commit is contained in:
Frederic Branczyk
GitHub
commit
b75ec7e6ef
246
discovery/kubernetes/client_metrics.go
Normal file
246
discovery/kubernetes/client_metrics.go
Normal file
|
|
@ -0,0 +1,246 @@
|
|||
// Copyright 2018 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package kubernetes
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
"k8s.io/client-go/tools/metrics"
|
||||
"k8s.io/client-go/util/workqueue"
|
||||
)
|
||||
|
||||
const (
|
||||
cacheMetricsNamespace = metricsNamespace + "_cache"
|
||||
workqueueMetricsNamespace = metricsNamespace + "_workqueue"
|
||||
)
|
||||
|
||||
var (
|
||||
// Metrics for client-go's HTTP requests.
|
||||
clientGoRequestResultMetricVec = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: metricsNamespace,
|
||||
Name: "http_request_total",
|
||||
Help: "Total number of HTTP requests to the Kubernetes API by status code.",
|
||||
},
|
||||
[]string{"status_code"},
|
||||
)
|
||||
clientGoRequestLatencyMetricVec = prometheus.NewSummaryVec(
|
||||
prometheus.SummaryOpts{
|
||||
Namespace: metricsNamespace,
|
||||
Name: "http_request_duration_seconds",
|
||||
Help: "Summary of latencies for HTTP requests to the Kubernetes API by endpoint.",
|
||||
Objectives: map[float64]float64{},
|
||||
},
|
||||
[]string{"endpoint"},
|
||||
)
|
||||
|
||||
// Definition of metrics for client-go cache metrics provider.
|
||||
clientGoCacheListTotalMetric = prometheus.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: cacheMetricsNamespace,
|
||||
Name: "list_total",
|
||||
Help: "Total number of list operations.",
|
||||
},
|
||||
)
|
||||
clientGoCacheListDurationMetric = prometheus.NewSummary(
|
||||
prometheus.SummaryOpts{
|
||||
Namespace: cacheMetricsNamespace,
|
||||
Name: "list_duration_seconds",
|
||||
Help: "Duration of a Kubernetes API call in seconds.",
|
||||
Objectives: map[float64]float64{},
|
||||
},
|
||||
)
|
||||
clientGoCacheItemsInListCountMetric = prometheus.NewSummary(
|
||||
prometheus.SummaryOpts{
|
||||
Namespace: cacheMetricsNamespace,
|
||||
Name: "list_items",
|
||||
Help: "Count of items in a list from the Kubernetes API.",
|
||||
Objectives: map[float64]float64{},
|
||||
},
|
||||
)
|
||||
clientGoCacheWatchesCountMetric = prometheus.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: cacheMetricsNamespace,
|
||||
Name: "watches_total",
|
||||
Help: "Total number of watch operations.",
|
||||
},
|
||||
)
|
||||
clientGoCacheShortWatchesCountMetric = prometheus.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: cacheMetricsNamespace,
|
||||
Name: "short_watches_total",
|
||||
Help: "Total number of short watch operations.",
|
||||
},
|
||||
)
|
||||
clientGoCacheWatchesDurationMetric = prometheus.NewSummary(
|
||||
prometheus.SummaryOpts{
|
||||
Namespace: cacheMetricsNamespace,
|
||||
Name: "watch_duration_seconds",
|
||||
Help: "Duration of watches on the Kubernetes API.",
|
||||
Objectives: map[float64]float64{},
|
||||
},
|
||||
)
|
||||
clientGoCacheItemsInWatchesCountMetric = prometheus.NewSummary(
|
||||
prometheus.SummaryOpts{
|
||||
Namespace: cacheMetricsNamespace,
|
||||
Name: "watch_events",
|
||||
Help: "Number of items in watches on the Kubernetes API.",
|
||||
Objectives: map[float64]float64{},
|
||||
},
|
||||
)
|
||||
clientGoCacheLastResourceVersionMetric = prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: cacheMetricsNamespace,
|
||||
Name: "last_resource_version",
|
||||
Help: "Last resource version from the Kubernetes API.",
|
||||
},
|
||||
)
|
||||
|
||||
// Definition of metrics for client-go workflow metrics provider
|
||||
clientGoWorkqueueDepthMetricVec = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: workqueueMetricsNamespace,
|
||||
Name: "depth",
|
||||
Help: "Current depth of the work queue.",
|
||||
},
|
||||
[]string{"queue_name"},
|
||||
)
|
||||
clientGoWorkqueueAddsMetricVec = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: workqueueMetricsNamespace,
|
||||
Name: "items_total",
|
||||
Help: "Total number of items added to the work queue.",
|
||||
},
|
||||
[]string{"queue_name"},
|
||||
)
|
||||
clientGoWorkqueueLatencyMetricVec = prometheus.NewSummaryVec(
|
||||
prometheus.SummaryOpts{
|
||||
Namespace: workqueueMetricsNamespace,
|
||||
Name: "latency_seconds",
|
||||
Help: "How long an item stays in the work queue.",
|
||||
Objectives: map[float64]float64{},
|
||||
},
|
||||
[]string{"queue_name"},
|
||||
)
|
||||
clientGoWorkqueueWorkDurationMetricVec = prometheus.NewSummaryVec(
|
||||
prometheus.SummaryOpts{
|
||||
Namespace: workqueueMetricsNamespace,
|
||||
Name: "work_duration_seconds",
|
||||
Help: "How long processing an item from the work queue takes.",
|
||||
Objectives: map[float64]float64{},
|
||||
},
|
||||
[]string{"queue_name"},
|
||||
)
|
||||
)
|
||||
|
||||
// Definition of dummy metric used as a placeholder if we don't want to observe some data.
|
||||
type noopMetric struct{}
|
||||
|
||||
func (noopMetric) Inc() {}
|
||||
func (noopMetric) Dec() {}
|
||||
func (noopMetric) Observe(float64) {}
|
||||
|
||||
// Definition of client-go metrics adapters for HTTP requests observation
|
||||
type clientGoRequestMetricAdapter struct{}
|
||||
|
||||
func (f *clientGoRequestMetricAdapter) Register(registerer prometheus.Registerer) {
|
||||
metrics.Register(f, f)
|
||||
registerer.MustRegister(clientGoRequestResultMetricVec)
|
||||
registerer.MustRegister(clientGoRequestLatencyMetricVec)
|
||||
}
|
||||
func (clientGoRequestMetricAdapter) Increment(code string, method string, host string) {
|
||||
clientGoRequestResultMetricVec.WithLabelValues(code).Inc()
|
||||
}
|
||||
func (clientGoRequestMetricAdapter) Observe(verb string, u url.URL, latency time.Duration) {
|
||||
clientGoRequestLatencyMetricVec.WithLabelValues(u.EscapedPath()).Observe(latency.Seconds())
|
||||
}
|
||||
|
||||
// Definition of client-go cache metrics provider definition
|
||||
type clientGoCacheMetricsProvider struct{}
|
||||
|
||||
func (f *clientGoCacheMetricsProvider) Register(registerer prometheus.Registerer) {
|
||||
cache.SetReflectorMetricsProvider(f)
|
||||
registerer.MustRegister(clientGoCacheWatchesDurationMetric)
|
||||
registerer.MustRegister(clientGoCacheWatchesCountMetric)
|
||||
registerer.MustRegister(clientGoCacheListDurationMetric)
|
||||
registerer.MustRegister(clientGoCacheListTotalMetric)
|
||||
registerer.MustRegister(clientGoCacheLastResourceVersionMetric)
|
||||
registerer.MustRegister(clientGoCacheShortWatchesCountMetric)
|
||||
registerer.MustRegister(clientGoCacheItemsInWatchesCountMetric)
|
||||
registerer.MustRegister(clientGoCacheItemsInListCountMetric)
|
||||
|
||||
}
|
||||
func (clientGoCacheMetricsProvider) NewListsMetric(name string) cache.CounterMetric {
|
||||
return clientGoCacheListTotalMetric
|
||||
}
|
||||
func (clientGoCacheMetricsProvider) NewListDurationMetric(name string) cache.SummaryMetric {
|
||||
return clientGoCacheListDurationMetric
|
||||
}
|
||||
func (clientGoCacheMetricsProvider) NewItemsInListMetric(name string) cache.SummaryMetric {
|
||||
return clientGoCacheItemsInListCountMetric
|
||||
}
|
||||
func (clientGoCacheMetricsProvider) NewWatchesMetric(name string) cache.CounterMetric {
|
||||
return clientGoCacheWatchesCountMetric
|
||||
}
|
||||
func (clientGoCacheMetricsProvider) NewShortWatchesMetric(name string) cache.CounterMetric {
|
||||
return clientGoCacheShortWatchesCountMetric
|
||||
}
|
||||
func (clientGoCacheMetricsProvider) NewWatchDurationMetric(name string) cache.SummaryMetric {
|
||||
return clientGoCacheWatchesDurationMetric
|
||||
}
|
||||
func (clientGoCacheMetricsProvider) NewItemsInWatchMetric(name string) cache.SummaryMetric {
|
||||
return clientGoCacheItemsInWatchesCountMetric
|
||||
}
|
||||
func (clientGoCacheMetricsProvider) NewLastResourceVersionMetric(name string) cache.GaugeMetric {
|
||||
return clientGoCacheLastResourceVersionMetric
|
||||
}
|
||||
|
||||
// Definition of client-go workqueue metrics provider definition
|
||||
type clientGoWorkqueueMetricsProvider struct{}
|
||||
|
||||
func (f *clientGoWorkqueueMetricsProvider) Register(registerer prometheus.Registerer) {
|
||||
workqueue.SetProvider(f)
|
||||
registerer.MustRegister(clientGoWorkqueueDepthMetricVec)
|
||||
registerer.MustRegister(clientGoWorkqueueAddsMetricVec)
|
||||
registerer.MustRegister(clientGoWorkqueueLatencyMetricVec)
|
||||
registerer.MustRegister(clientGoWorkqueueWorkDurationMetricVec)
|
||||
}
|
||||
|
||||
func (f *clientGoWorkqueueMetricsProvider) NewDepthMetric(name string) workqueue.GaugeMetric {
|
||||
return clientGoWorkqueueDepthMetricVec.WithLabelValues(name)
|
||||
}
|
||||
func (f *clientGoWorkqueueMetricsProvider) NewAddsMetric(name string) workqueue.CounterMetric {
|
||||
return clientGoWorkqueueAddsMetricVec.WithLabelValues(name)
|
||||
}
|
||||
func (f *clientGoWorkqueueMetricsProvider) NewLatencyMetric(name string) workqueue.SummaryMetric {
|
||||
metric := clientGoWorkqueueLatencyMetricVec.WithLabelValues(name)
|
||||
// Convert microseconds to seconds for consistency across metrics.
|
||||
return prometheus.ObserverFunc(func(v float64) {
|
||||
metric.Observe(v / 1e6)
|
||||
})
|
||||
}
|
||||
func (f *clientGoWorkqueueMetricsProvider) NewWorkDurationMetric(name string) workqueue.SummaryMetric {
|
||||
metric := clientGoWorkqueueWorkDurationMetricVec.WithLabelValues(name)
|
||||
// Convert microseconds to seconds for consistency across metrics.
|
||||
return prometheus.ObserverFunc(func(v float64) {
|
||||
metric.Observe(v / 1e6)
|
||||
})
|
||||
}
|
||||
func (clientGoWorkqueueMetricsProvider) NewRetriesMetric(name string) workqueue.CounterMetric {
|
||||
// Retries are not used so the metric is ommited.
|
||||
return noopMetric{}
|
||||
}
|
||||
|
|
@ -40,15 +40,18 @@ import (
|
|||
const (
|
||||
// kubernetesMetaLabelPrefix is the meta prefix used for all meta labels.
|
||||
// in this discovery.
|
||||
metaLabelPrefix = model.MetaLabelPrefix + "kubernetes_"
|
||||
namespaceLabel = metaLabelPrefix + "namespace"
|
||||
metaLabelPrefix = model.MetaLabelPrefix + "kubernetes_"
|
||||
namespaceLabel = metaLabelPrefix + "namespace"
|
||||
metricsNamespace = "prometheus_sd_kubernetes"
|
||||
)
|
||||
|
||||
var (
|
||||
// Custom events metric
|
||||
eventCount = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "prometheus_sd_kubernetes_events_total",
|
||||
Help: "The number of Kubernetes events handled.",
|
||||
Namespace: metricsNamespace,
|
||||
Name: "events_total",
|
||||
Help: "The number of Kubernetes events handled.",
|
||||
},
|
||||
[]string{"role", "event"},
|
||||
)
|
||||
|
|
@ -139,6 +142,17 @@ func init() {
|
|||
eventCount.WithLabelValues(role, evt)
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
clientGoRequestMetricAdapterInstance = clientGoRequestMetricAdapter{}
|
||||
clientGoCacheMetricsProviderInstance = clientGoCacheMetricsProvider{}
|
||||
clientGoWorkqueueMetricsProviderInstance = clientGoWorkqueueMetricsProvider{}
|
||||
)
|
||||
|
||||
clientGoRequestMetricAdapterInstance.Register(prometheus.DefaultRegisterer)
|
||||
clientGoCacheMetricsProviderInstance.Register(prometheus.DefaultRegisterer)
|
||||
clientGoWorkqueueMetricsProviderInstance.Register(prometheus.DefaultRegisterer)
|
||||
|
||||
}
|
||||
|
||||
// This is only for internal use.
|
||||
|
|
|
|||
Loading…
Reference in a new issue