From 02ee897c03ce15970f76e5404f35d85fea9210f9 Mon Sep 17 00:00:00 2001 From: Kozlov Alexander Date: Wed, 23 Jun 2021 12:52:43 +0300 Subject: [PATCH] Added conntrack statistics metrics (#1155) * Added conntrack statistics metrics Signed-off-by: Aleksandr Kozlov Co-authored-by: Aleksandr Kozlov Co-authored-by: Ben Kochie --- collector/conntrack_linux.go | 114 +++++++++++++++++- collector/fixtures/e2e-output.txt | 31 ++++- collector/fixtures/proc/1/stat | 1 + collector/fixtures/proc/11/stat | 1 + collector/fixtures/proc/net/stat/nf_conntrack | 5 + 5 files changed, 146 insertions(+), 6 deletions(-) create mode 100644 collector/fixtures/proc/1/stat create mode 100644 collector/fixtures/proc/11/stat create mode 100644 collector/fixtures/proc/net/stat/nf_conntrack diff --git a/collector/conntrack_linux.go b/collector/conntrack_linux.go index 67298f62..902024fd 100644 --- a/collector/conntrack_linux.go +++ b/collector/conntrack_linux.go @@ -23,12 +23,32 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/procfs" ) type conntrackCollector struct { - current *prometheus.Desc - limit *prometheus.Desc - logger log.Logger + current *prometheus.Desc + limit *prometheus.Desc + found *prometheus.Desc + invalid *prometheus.Desc + ignore *prometheus.Desc + insert *prometheus.Desc + insertFailed *prometheus.Desc + drop *prometheus.Desc + earlyDrop *prometheus.Desc + searchRestart *prometheus.Desc + logger log.Logger +} + +type conntrackStatistics struct { + found uint64 // Number of searched entries which were successful + invalid uint64 // Number of packets seen which can not be tracked + ignore uint64 // Number of packets seen which are already connected to a conntrack entry + insert uint64 // Number of entries inserted into the list + insertFailed uint64 // Number of entries for which list insertion was attempted but failed (happens if the same entry is already present) + drop uint64 // Number of packets dropped due to conntrack failure. Either new conntrack entry allocation failed, or protocol helper dropped the packet + earlyDrop uint64 // Number of dropped conntrack entries to make room for new ones, if maximum table size was reached + searchRestart uint64 // Number of conntrack table lookups which had to be restarted due to hashtable resizes } func init() { @@ -48,6 +68,46 @@ func NewConntrackCollector(logger log.Logger) (Collector, error) { "Maximum size of connection tracking table.", nil, nil, ), + found: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "nf_conntrack_stat_found"), + "Number of searched entries which were successful.", + nil, nil, + ), + invalid: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "nf_conntrack_stat_invalid"), + "Number of packets seen which can not be tracked.", + nil, nil, + ), + ignore: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "nf_conntrack_stat_ignore"), + "Number of packets seen which are already connected to a conntrack entry.", + nil, nil, + ), + insert: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "nf_conntrack_stat_insert"), + "Number of entries inserted into the list.", + nil, nil, + ), + insertFailed: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "nf_conntrack_stat_insert_failed"), + "Number of entries for which list insertion was attempted but failed.", + nil, nil, + ), + drop: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "nf_conntrack_stat_drop"), + "Number of packets dropped due to conntrack failure.", + nil, nil, + ), + earlyDrop: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "nf_conntrack_stat_early_drop"), + "Number of dropped conntrack entries to make room for new ones, if maximum table size was reached.", + nil, nil, + ), + searchRestart: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "nf_conntrack_stat_search_restart"), + "Number of conntrack table lookups which had to be restarted due to hashtable resizes.", + nil, nil, + ), logger: logger, }, nil } @@ -67,6 +127,27 @@ func (c *conntrackCollector) Update(ch chan<- prometheus.Metric) error { ch <- prometheus.MustNewConstMetric( c.limit, prometheus.GaugeValue, float64(value)) + conntrackStats, err := getConntrackStatistics() + if err != nil { + return err + } + + ch <- prometheus.MustNewConstMetric( + c.found, prometheus.GaugeValue, float64(conntrackStats.found)) + ch <- prometheus.MustNewConstMetric( + c.invalid, prometheus.GaugeValue, float64(conntrackStats.invalid)) + ch <- prometheus.MustNewConstMetric( + c.ignore, prometheus.GaugeValue, float64(conntrackStats.ignore)) + ch <- prometheus.MustNewConstMetric( + c.insert, prometheus.GaugeValue, float64(conntrackStats.insert)) + ch <- prometheus.MustNewConstMetric( + c.insertFailed, prometheus.GaugeValue, float64(conntrackStats.insertFailed)) + ch <- prometheus.MustNewConstMetric( + c.drop, prometheus.GaugeValue, float64(conntrackStats.drop)) + ch <- prometheus.MustNewConstMetric( + c.earlyDrop, prometheus.GaugeValue, float64(conntrackStats.earlyDrop)) + ch <- prometheus.MustNewConstMetric( + c.searchRestart, prometheus.GaugeValue, float64(conntrackStats.searchRestart)) return nil } @@ -77,3 +158,30 @@ func (c *conntrackCollector) handleErr(err error) error { } return fmt.Errorf("failed to retrieve conntrack stats: %w", err) } + +func getConntrackStatistics() (*conntrackStatistics, error) { + c := conntrackStatistics{} + + fs, err := procfs.NewFS(*procPath) + if err != nil { + return nil, fmt.Errorf("failed to open procfs: %w", err) + } + + connStats, err := fs.ConntrackStat() + if err != nil { + return nil, err + } + + for _, connStat := range connStats { + c.found += connStat.Found + c.invalid += connStat.Invalid + c.ignore += connStat.Ignore + c.insert += connStat.Insert + c.insertFailed += connStat.InsertFailed + c.drop += connStat.Drop + c.earlyDrop += connStat.EarlyDrop + c.searchRestart += connStat.SearchRestart + } + + return &c, nil +} \ No newline at end of file diff --git a/collector/fixtures/e2e-output.txt b/collector/fixtures/e2e-output.txt index e04cb0cc..19f40d2a 100644 --- a/collector/fixtures/e2e-output.txt +++ b/collector/fixtures/e2e-output.txt @@ -2381,6 +2381,30 @@ node_nf_conntrack_entries 123 # HELP node_nf_conntrack_entries_limit Maximum size of connection tracking table. # TYPE node_nf_conntrack_entries_limit gauge node_nf_conntrack_entries_limit 65536 +# HELP node_nf_conntrack_stat_drop Number of packets dropped due to conntrack failure. +# TYPE node_nf_conntrack_stat_drop gauge +node_nf_conntrack_stat_drop 0 +# HELP node_nf_conntrack_stat_early_drop Number of dropped conntrack entries to make room for new ones, if maximum table size was reached. +# TYPE node_nf_conntrack_stat_early_drop gauge +node_nf_conntrack_stat_early_drop 0 +# HELP node_nf_conntrack_stat_found Number of searched entries which were successful. +# TYPE node_nf_conntrack_stat_found gauge +node_nf_conntrack_stat_found 0 +# HELP node_nf_conntrack_stat_ignore Number of packets seen which are already connected to a conntrack entry. +# TYPE node_nf_conntrack_stat_ignore gauge +node_nf_conntrack_stat_ignore 89738 +# HELP node_nf_conntrack_stat_insert Number of entries inserted into the list. +# TYPE node_nf_conntrack_stat_insert gauge +node_nf_conntrack_stat_insert 0 +# HELP node_nf_conntrack_stat_insert_failed Number of entries for which list insertion was attempted but failed. +# TYPE node_nf_conntrack_stat_insert_failed gauge +node_nf_conntrack_stat_insert_failed 0 +# HELP node_nf_conntrack_stat_invalid Number of packets seen which can not be tracked. +# TYPE node_nf_conntrack_stat_invalid gauge +node_nf_conntrack_stat_invalid 53 +# HELP node_nf_conntrack_stat_search_restart Number of conntrack table lookups which had to be restarted due to hashtable resizes. +# TYPE node_nf_conntrack_stat_search_restart gauge +node_nf_conntrack_stat_search_restart 7 # HELP node_nfs_connections_total Total number of NFSd TCP connections. # TYPE node_nfs_connections_total counter node_nfs_connections_total 45 @@ -2670,13 +2694,14 @@ node_processes_max_processes 123 node_processes_max_threads 7801 # HELP node_processes_pids Number of PIDs # TYPE node_processes_pids gauge -node_processes_pids 1 +node_processes_pids 3 # HELP node_processes_state Number of processes in each state. # TYPE node_processes_state gauge -node_processes_state{state="S"} 1 +node_processes_state{state="I"} 1 +node_processes_state{state="S"} 2 # HELP node_processes_threads Allocated threads in system # TYPE node_processes_threads gauge -node_processes_threads 1 +node_processes_threads 3 # HELP node_procs_blocked Number of processes blocked waiting for I/O to complete. # TYPE node_procs_blocked gauge node_procs_blocked 0 diff --git a/collector/fixtures/proc/1/stat b/collector/fixtures/proc/1/stat new file mode 100644 index 00000000..b6bc64d6 --- /dev/null +++ b/collector/fixtures/proc/1/stat @@ -0,0 +1 @@ +1 (systemd) S 0 1 1 0 -1 4194560 9061 9416027 94 2620 36 98 54406 13885 20 0 1 0 29 109604864 2507 18446744073709551615 1 1 0 0 0 0 671173123 4096 1260 0 0 0 17 0 0 0 19 0 0 0 0 0 0 0 0 0 0 diff --git a/collector/fixtures/proc/11/stat b/collector/fixtures/proc/11/stat new file mode 100644 index 00000000..1d91e2c1 --- /dev/null +++ b/collector/fixtures/proc/11/stat @@ -0,0 +1 @@ +11 (rcu_preempt) I 2 0 0 0 -1 2129984 0 0 0 0 0 346 0 0 -2 0 1 0 32 0 0 18446744073709551615 0 0 0 0 0 0 0 2147483647 0 0 0 0 17 2 1 1 0 0 0 0 0 0 0 0 0 0 0 diff --git a/collector/fixtures/proc/net/stat/nf_conntrack b/collector/fixtures/proc/net/stat/nf_conntrack new file mode 100644 index 00000000..c833c40e --- /dev/null +++ b/collector/fixtures/proc/net/stat/nf_conntrack @@ -0,0 +1,5 @@ +entries searched found new invalid ignore delete delete_list insert insert_failed drop early_drop icmp_error expect_new expect_create expect_delete search_restart +00000021 00000000 00000000 00000000 00000003 0000588a 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 +00000021 00000000 00000000 00000000 00000002 000056a4 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000002 +00000021 00000000 00000000 00000000 00000001 000058d4 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000001 +00000021 00000000 00000000 00000000 0000002f 00005688 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000004