Merge pull request #67 from matthiasr/ipvs-metrics

Add IPVS stats collector.

README.md

@@ -50,6 +50,7 @@ Name     | Description
 bonding | Exposes the number of configured and active slaves of Linux bonding interfaces.
 gmond | Exposes statistics from Ganglia.
 interrupts | Exposes detailed interrupts statistics from `/proc/interrupts`.
+ipvs | Exposes IPVS status from `/proc/net/ip_vs` and stats from `/proc/net/ip_vs_stats`.
 lastlogin | Exposes the last time there was a login.
 megacli | Exposes RAID statistics from MegaCLI.
 ntp | Exposes time drift from an NTP server.

collector/fixtures/net/ip_vs

@@ -0,0 +1,14 @@
+IP Virtual Server version 1.2.1 (size=4096)
+Prot LocalAddress:Port Scheduler Flags
+  -> RemoteAddress:Port Forward Weight ActiveConn InActConn
+TCP  C0A80016:0CEA wlc  
+  -> C0A85216:0CEA      Tunnel  100    248        2         
+  -> C0A85318:0CEA      Tunnel  100    248        2         
+  -> C0A85315:0CEA      Tunnel  100    248        1         
+TCP  C0A80039:0CEA wlc  
+  -> C0A85416:0CEA      Tunnel  0      0          0         
+  -> C0A85215:0CEA      Tunnel  100    1499       0         
+  -> C0A83215:0CEA      Tunnel  100    1498       0         
+TCP  C0A80037:0CEA wlc  
+  -> C0A8321A:0CEA      Tunnel  0      0          0         
+  -> C0A83120:0CEA      Tunnel  100    0          0         

collector/fixtures/net/ip_vs_result.txt

@@ -0,0 +1,45 @@
+# HELP node_ipvs_backend_connections_active The current active connections by local and remote address.
+# TYPE node_ipvs_backend_connections_active gauge
+node_ipvs_backend_connections_active{local_address="192.168.0.22",local_port="3306",proto="TCP",remote_address="192.168.82.22",remote_port="3306"} 248
+node_ipvs_backend_connections_active{local_address="192.168.0.22",local_port="3306",proto="TCP",remote_address="192.168.83.21",remote_port="3306"} 248
+node_ipvs_backend_connections_active{local_address="192.168.0.22",local_port="3306",proto="TCP",remote_address="192.168.83.24",remote_port="3306"} 248
+node_ipvs_backend_connections_active{local_address="192.168.0.55",local_port="3306",proto="TCP",remote_address="192.168.49.32",remote_port="3306"} 0
+node_ipvs_backend_connections_active{local_address="192.168.0.55",local_port="3306",proto="TCP",remote_address="192.168.50.26",remote_port="3306"} 0
+node_ipvs_backend_connections_active{local_address="192.168.0.57",local_port="3306",proto="TCP",remote_address="192.168.50.21",remote_port="3306"} 1498
+node_ipvs_backend_connections_active{local_address="192.168.0.57",local_port="3306",proto="TCP",remote_address="192.168.82.21",remote_port="3306"} 1499
+node_ipvs_backend_connections_active{local_address="192.168.0.57",local_port="3306",proto="TCP",remote_address="192.168.84.22",remote_port="3306"} 0
+# HELP node_ipvs_backend_connections_inactive The current inactive connections by local and remote address.
+# TYPE node_ipvs_backend_connections_inactive gauge
+node_ipvs_backend_connections_inactive{local_address="192.168.0.22",local_port="3306",proto="TCP",remote_address="192.168.82.22",remote_port="3306"} 2
+node_ipvs_backend_connections_inactive{local_address="192.168.0.22",local_port="3306",proto="TCP",remote_address="192.168.83.21",remote_port="3306"} 1
+node_ipvs_backend_connections_inactive{local_address="192.168.0.22",local_port="3306",proto="TCP",remote_address="192.168.83.24",remote_port="3306"} 2
+node_ipvs_backend_connections_inactive{local_address="192.168.0.55",local_port="3306",proto="TCP",remote_address="192.168.49.32",remote_port="3306"} 0
+node_ipvs_backend_connections_inactive{local_address="192.168.0.55",local_port="3306",proto="TCP",remote_address="192.168.50.26",remote_port="3306"} 0
+node_ipvs_backend_connections_inactive{local_address="192.168.0.57",local_port="3306",proto="TCP",remote_address="192.168.50.21",remote_port="3306"} 0
+node_ipvs_backend_connections_inactive{local_address="192.168.0.57",local_port="3306",proto="TCP",remote_address="192.168.82.21",remote_port="3306"} 0
+node_ipvs_backend_connections_inactive{local_address="192.168.0.57",local_port="3306",proto="TCP",remote_address="192.168.84.22",remote_port="3306"} 0
+# HELP node_ipvs_backend_weight The current backend weight by local and remote address.
+# TYPE node_ipvs_backend_weight gauge
+node_ipvs_backend_weight{local_address="192.168.0.22",local_port="3306",proto="TCP",remote_address="192.168.82.22",remote_port="3306"} 100
+node_ipvs_backend_weight{local_address="192.168.0.22",local_port="3306",proto="TCP",remote_address="192.168.83.21",remote_port="3306"} 100
+node_ipvs_backend_weight{local_address="192.168.0.22",local_port="3306",proto="TCP",remote_address="192.168.83.24",remote_port="3306"} 100
+node_ipvs_backend_weight{local_address="192.168.0.55",local_port="3306",proto="TCP",remote_address="192.168.49.32",remote_port="3306"} 100
+node_ipvs_backend_weight{local_address="192.168.0.55",local_port="3306",proto="TCP",remote_address="192.168.50.26",remote_port="3306"} 0
+node_ipvs_backend_weight{local_address="192.168.0.57",local_port="3306",proto="TCP",remote_address="192.168.50.21",remote_port="3306"} 100
+node_ipvs_backend_weight{local_address="192.168.0.57",local_port="3306",proto="TCP",remote_address="192.168.82.21",remote_port="3306"} 100
+node_ipvs_backend_weight{local_address="192.168.0.57",local_port="3306",proto="TCP",remote_address="192.168.84.22",remote_port="3306"} 0
+# HELP node_ipvs_connections_total The total number of connections made.
+# TYPE node_ipvs_connections_total counter
+node_ipvs_connections_total 2.3765872e+07
+# HELP node_ipvs_incoming_bytes_total The total amount of incoming data.
+# TYPE node_ipvs_incoming_bytes_total counter
+node_ipvs_incoming_bytes_total 8.9991519156915e+13
+# HELP node_ipvs_incoming_packets_total The total number of incoming packets.
+# TYPE node_ipvs_incoming_packets_total counter
+node_ipvs_incoming_packets_total 3.811989221e+09
+# HELP node_ipvs_outgoing_bytes_total The total amount of outgoing data.
+# TYPE node_ipvs_outgoing_bytes_total counter
+node_ipvs_outgoing_bytes_total 0
+# HELP node_ipvs_outgoing_packets_total The total number of outgoing packets.
+# TYPE node_ipvs_outgoing_packets_total counter
+node_ipvs_outgoing_packets_total 0

collector/fixtures/net/ip_vs_stats

@@ -0,0 +1,6 @@
+   Total Incoming Outgoing         Incoming         Outgoing
+   Conns  Packets  Packets            Bytes            Bytes
+ 16AA370 E33656E5        0     51D8C8883AB3                0
+
+ Conns/s   Pkts/s   Pkts/s          Bytes/s          Bytes/s
+       4    1FB3C        0          1282A8F                0

collector/ipvs.go

@@ -0,0 +1,171 @@
+// +build !noipvs
+
+package collector
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/procfs"
+)
+
+type ipvsCollector struct {
+	Collector
+	fs                                                                          procfs.FS
+	backendConnectionsActive, backendConnectionsInact, backendWeight            *prometheus.GaugeVec
+	connections, incomingPackets, outgoingPackets, incomingBytes, outgoingBytes prometheus.Counter
+}
+
+func init() {
+	Factories["ipvs"] = NewIPVSCollector
+}
+
+// NewIPVSCollector sets up a new collector for IPVS metrics. It accepts the
+// "procfs" config parameter to override the default proc location (/proc).
+func NewIPVSCollector(config Config) (Collector, error) {
+	return newIPVSCollector(config)
+}
+
+func newIPVSCollector(config Config) (*ipvsCollector, error) {
+	var (
+		ipvsBackendLabelNames = []string{
+			"local_address",
+			"local_port",
+			"remote_address",
+			"remote_port",
+			"proto",
+		}
+		c         ipvsCollector
+		subsystem string
+		err       error
+	)
+
+	if p, ok := config.Config["procfs"]; !ok {
+		c.fs, err = procfs.NewFS(procfs.DefaultMountPoint)
+	} else {
+		c.fs, err = procfs.NewFS(p)
+	}
+	if err != nil {
+		return nil, err
+	}
+	if s, ok := config.Config["ipvs_subsystem"]; ok {
+		subsystem = s
+	} else {
+		subsystem = "ipvs"
+	}
+
+	c.connections = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: Namespace,
+			Subsystem: subsystem,
+			Name:      "connections_total",
+			Help:      "The total number of connections made.",
+		},
+	)
+	c.incomingPackets = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: Namespace,
+			Subsystem: subsystem,
+			Name:      "incoming_packets_total",
+			Help:      "The total number of incoming packets.",
+		},
+	)
+	c.outgoingPackets = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: Namespace,
+			Subsystem: subsystem,
+			Name:      "outgoing_packets_total",
+			Help:      "The total number of outgoing packets.",
+		},
+	)
+	c.incomingBytes = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: Namespace,
+			Subsystem: subsystem,
+			Name:      "incoming_bytes_total",
+			Help:      "The total amount of incoming data.",
+		},
+	)
+	c.outgoingBytes = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: Namespace,
+			Subsystem: subsystem,
+			Name:      "outgoing_bytes_total",
+			Help:      "The total amount of outgoing data.",
+		},
+	)
+
+	c.backendConnectionsActive = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: Namespace,
+			Subsystem: subsystem,
+			Name:      "backend_connections_active",
+			Help:      "The current active connections by local and remote address.",
+		},
+		ipvsBackendLabelNames,
+	)
+	c.backendConnectionsInact = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: Namespace,
+			Subsystem: subsystem,
+			Name:      "backend_connections_inactive",
+			Help:      "The current inactive connections by local and remote address.",
+		},
+		ipvsBackendLabelNames,
+	)
+	c.backendWeight = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: Namespace,
+			Subsystem: subsystem,
+			Name:      "backend_weight",
+			Help:      "The current backend weight by local and remote address.",
+		},
+		ipvsBackendLabelNames,
+	)
+
+	return &c, nil
+}
+
+func (c *ipvsCollector) Update(ch chan<- prometheus.Metric) error {
+	ipvsStats, err := c.fs.NewIPVSStats()
+	if err != nil {
+		return fmt.Errorf("could not get IPVS stats: %s", err)
+	}
+
+	c.connections.Set(float64(ipvsStats.Connections))
+	c.incomingPackets.Set(float64(ipvsStats.IncomingPackets))
+	c.outgoingPackets.Set(float64(ipvsStats.OutgoingPackets))
+	c.incomingBytes.Set(float64(ipvsStats.IncomingBytes))
+	c.outgoingBytes.Set(float64(ipvsStats.OutgoingBytes))
+
+	c.connections.Collect(ch)
+	c.incomingPackets.Collect(ch)
+	c.outgoingPackets.Collect(ch)
+	c.incomingBytes.Collect(ch)
+	c.outgoingBytes.Collect(ch)
+
+	backendStats, err := c.fs.NewIPVSBackendStatus()
+	if err != nil {
+		return fmt.Errorf("could not get backend status: %s", err)
+	}
+
+	for _, backend := range backendStats {
+		labelValues := []string{
+			backend.LocalAddress.String(),
+			strconv.FormatUint(uint64(backend.LocalPort), 10),
+			backend.RemoteAddress.String(),
+			strconv.FormatUint(uint64(backend.RemotePort), 10),
+			backend.Proto,
+		}
+		c.backendConnectionsActive.WithLabelValues(labelValues...).Set(float64(backend.ActiveConn))
+		c.backendConnectionsInact.WithLabelValues(labelValues...).Set(float64(backend.InactConn))
+		c.backendWeight.WithLabelValues(labelValues...).Set(float64(backend.Weight))
+	}
+
+	c.backendConnectionsActive.Collect(ch)
+	c.backendConnectionsInact.Collect(ch)
+	c.backendWeight.Collect(ch)
+
+	return nil
+}

collector/ipvs_test.go

@@ -0,0 +1,205 @@
+package collector
+
+import (
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/procfs"
+)
+
+var (
+	expectedIPVSStats = procfs.IPVSStats{
+		Connections:     23765872,
+		IncomingPackets: 3811989221,
+		OutgoingPackets: 0,
+		IncomingBytes:   89991519156915,
+		OutgoingBytes:   0,
+	}
+	expectedIPVSBackendStatuses = []procfs.IPVSBackendStatus{
+		procfs.IPVSBackendStatus{
+			LocalAddress:  net.ParseIP("192.168.0.22"),
+			LocalPort:     3306,
+			RemoteAddress: net.ParseIP("192.168.82.22"),
+			RemotePort:    3306,
+			Proto:         "TCP",
+			Weight:        100,
+			ActiveConn:    248,
+			InactConn:     2,
+		},
+		procfs.IPVSBackendStatus{
+			LocalAddress:  net.ParseIP("192.168.0.22"),
+			LocalPort:     3306,
+			RemoteAddress: net.ParseIP("192.168.83.24"),
+			RemotePort:    3306,
+			Proto:         "TCP",
+			Weight:        100,
+			ActiveConn:    248,
+			InactConn:     2,
+		},
+		procfs.IPVSBackendStatus{
+			LocalAddress:  net.ParseIP("192.168.0.22"),
+			LocalPort:     3306,
+			RemoteAddress: net.ParseIP("192.168.83.21"),
+			RemotePort:    3306,
+			Proto:         "TCP",
+			Weight:        100,
+			ActiveConn:    248,
+			InactConn:     1,
+		},
+		procfs.IPVSBackendStatus{
+			LocalAddress:  net.ParseIP("192.168.0.57"),
+			LocalPort:     3306,
+			RemoteAddress: net.ParseIP("192.168.84.22"),
+			RemotePort:    3306,
+			Proto:         "TCP",
+			Weight:        0,
+			ActiveConn:    0,
+			InactConn:     0,
+		},
+		procfs.IPVSBackendStatus{
+			LocalAddress:  net.ParseIP("192.168.0.57"),
+			LocalPort:     3306,
+			RemoteAddress: net.ParseIP("192.168.82.21"),
+			RemotePort:    3306,
+			Proto:         "TCP",
+			Weight:        100,
+			ActiveConn:    1499,
+			InactConn:     0,
+		},
+		procfs.IPVSBackendStatus{
+			LocalAddress:  net.ParseIP("192.168.0.57"),
+			LocalPort:     3306,
+			RemoteAddress: net.ParseIP("192.168.50.21"),
+			RemotePort:    3306,
+			Proto:         "TCP",
+			Weight:        100,
+			ActiveConn:    1498,
+			InactConn:     0,
+		},
+		procfs.IPVSBackendStatus{
+			LocalAddress:  net.ParseIP("192.168.0.55"),
+			LocalPort:     3306,
+			RemoteAddress: net.ParseIP("192.168.50.26"),
+			RemotePort:    3306,
+			Proto:         "TCP",
+			Weight:        0,
+			ActiveConn:    0,
+			InactConn:     0,
+		},
+		procfs.IPVSBackendStatus{
+			LocalAddress:  net.ParseIP("192.168.0.55"),
+			LocalPort:     3306,
+			RemoteAddress: net.ParseIP("192.168.49.32"),
+			RemotePort:    3306,
+			Proto:         "TCP",
+			Weight:        100,
+			ActiveConn:    0,
+			InactConn:     0,
+		},
+	}
+)
+
+func TestIPVSCollector(t *testing.T) {
+	collector, err := newIPVSCollector(Config{Config: map[string]string{"procfs": "fixtures"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	sink := make(chan prometheus.Metric)
+	go func() {
+		for {
+			<-sink
+		}
+	}()
+
+	err = collector.Update(sink)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, expect := range expectedIPVSBackendStatuses {
+		labels := prometheus.Labels{
+			"local_address":  expect.LocalAddress.String(),
+			"local_port":     strconv.FormatUint(uint64(expect.LocalPort), 10),
+			"remote_address": expect.RemoteAddress.String(),
+			"remote_port":    strconv.FormatUint(uint64(expect.RemotePort), 10),
+			"proto":          expect.Proto,
+		}
+		// TODO: Pending prometheus/client_golang#58, check the actual numbers
+		_, err = collector.backendConnectionsActive.GetMetricWith(labels)
+		if err != nil {
+			t.Errorf("Missing active connections metric for label combination: %+v", labels)
+		}
+		_, err = collector.backendConnectionsInact.GetMetricWith(labels)
+		if err != nil {
+			t.Errorf("Missing inactive connections metric for label combination: %+v", labels)
+		}
+		_, err = collector.backendWeight.GetMetricWith(labels)
+		if err != nil {
+			t.Errorf("Missing weight metric for label combination: %+v", labels)
+		}
+	}
+}
+
+// mock collector
+type miniCollector struct {
+	c Collector
+}
+
+func (c miniCollector) Collect(ch chan<- prometheus.Metric) {
+	c.c.Update(ch)
+}
+
+func (c miniCollector) Describe(ch chan<- *prometheus.Desc) {
+	prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "fake",
+		Subsystem: "fake",
+		Name:      "fake",
+		Help:      "fake",
+	}).Describe(ch)
+}
+
+func TestIPVSCollectorResponse(t *testing.T) {
+	collector, err := NewIPVSCollector(Config{Config: map[string]string{"procfs": "fixtures"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	prometheus.MustRegister(miniCollector{c: collector})
+
+	rw := httptest.NewRecorder()
+	prometheus.Handler().ServeHTTP(rw, &http.Request{})
+
+	metricsFile := "fixtures/net/ip_vs_result.txt"
+	wantMetrics, err := ioutil.ReadFile(metricsFile)
+	if err != nil {
+		t.Fatalf("unable to read input test file %s: %s", metricsFile, err)
+	}
+
+	wantLines := strings.Split(string(wantMetrics), "\n")
+	gotLines := strings.Split(string(rw.Body.String()), "\n")
+	gotLinesIdx := 0
+
+	// Until the Prometheus Go client library offers better testability
+	// (https://github.com/prometheus/client_golang/issues/58), we simply compare
+	// verbatim text-format metrics outputs, but ignore any lines we don't have
+	// in the fixture. Put differently, we are only testing that each line from
+	// the fixture is present, in the order given.
+wantLoop:
+	for _, want := range wantLines {
+		for _, got := range gotLines[gotLinesIdx:] {
+			if want == got {
+				// this is a line we are interested in, and it is correct
+				continue wantLoop
+			} else {
+				gotLinesIdx++
+			}
+		}
+		// if this point is reached, the line we want was missing
+		t.Fatalf("Missing expected output line(s), first missing line is %s", want)
+	}
+}