Merge pull request #1832 from ventifus/master

Add a new ethtool stats collector

README.md

@@ -190,6 +190,7 @@ Name     | Description | OS
 buddyinfo | Exposes statistics of memory fragments as reported by /proc/buddyinfo. | Linux
 devstat | Exposes device statistics | Dragonfly, FreeBSD
 drbd | Exposes Distributed Replicated Block Device statistics (to version 8.4) | Linux
+ethtool | Exposes network interface and network driver statistics equivalent to `ethtool -S`. | Linux
 interrupts | Exposes detailed interrupts statistics. | Linux, OpenBSD
 ksmd | Exposes kernel and system statistics from `/sys/kernel/mm/ksm`. | Linux
 logind | Exposes session counts from [logind](http://www.freedesktop.org/wiki/Software/systemd/logind/). | Linux

collector/ethtool_linux.go

@@ -0,0 +1,192 @@
+// Copyright 2021 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !noethtool
+
+// The hard work of collecting data from the kernel via the ethtool interfaces is done by
+// https://github.com/safchain/ethtool/
+// by Sylvain Afchain. Used under the Apache license.
+
+package collector
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"regexp"
+	"sort"
+	"syscall"
+
+	"github.com/go-kit/kit/log"
+	"github.com/go-kit/kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/procfs/sysfs"
+	"github.com/safchain/ethtool"
+	"golang.org/x/sys/unix"
+)
+
+var (
+	receivedRegex    = regexp.MustCompile(`_rx_`)
+	transmittedRegex = regexp.MustCompile(`_tx_`)
+)
+
+type EthtoolStats interface {
+	Stats(string) (map[string]uint64, error)
+}
+
+type ethtoolStats struct {
+}
+
+func (e *ethtoolStats) Stats(intf string) (map[string]uint64, error) {
+	return ethtool.Stats(intf)
+}
+
+type ethtoolCollector struct {
+	fs      sysfs.FS
+	entries map[string]*prometheus.Desc
+	logger  log.Logger
+	stats   EthtoolStats
+}
+
+// makeEthtoolCollector is the internal constructor for EthtoolCollector.
+// This allows NewEthtoolTestCollector to override its .stats interface
+// for testing.
+func makeEthtoolCollector(logger log.Logger) (*ethtoolCollector, error) {
+	fs, err := sysfs.NewFS(*sysPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open sysfs: %w", err)
+	}
+
+	// Pre-populate some common ethtool metrics.
+	return &ethtoolCollector{
+		fs:    fs,
+		stats: &ethtoolStats{},
+		entries: map[string]*prometheus.Desc{
+			"rx_bytes": prometheus.NewDesc(
+				"node_ethtool_received_bytes_total",
+				"Network interface bytes received",
+				[]string{"device"}, nil,
+			),
+			"rx_dropped": prometheus.NewDesc(
+				"node_ethtool_received_dropped_total",
+				"Number of received frames dropped",
+				[]string{"device"}, nil,
+			),
+			"rx_errors": prometheus.NewDesc(
+				"node_ethtool_received_errors_total",
+				"Number of received frames with errors",
+				[]string{"device"}, nil,
+			),
+			"rx_packets": prometheus.NewDesc(
+				"node_ethtool_received_packets_total",
+				"Network interface packets received",
+				[]string{"device"}, nil,
+			),
+			"tx_bytes": prometheus.NewDesc(
+				"node_ethtool_transmitted_bytes_total",
+				"Network interface bytes sent",
+				[]string{"device"}, nil,
+			),
+			"tx_errors": prometheus.NewDesc(
+				"node_ethtool_transmitted_errors_total",
+				"Number of sent frames with errors",
+				[]string{"device"}, nil,
+			),
+			"tx_packets": prometheus.NewDesc(
+				"node_ethtool_transmitted_packets_total",
+				"Network interface packets sent",
+				[]string{"device"}, nil,
+			),
+		},
+		logger: logger,
+	}, nil
+}
+
+func init() {
+	registerCollector("ethtool", defaultDisabled, NewEthtoolCollector)
+}
+
+// NewEthtoolCollector returns a new Collector exposing ethtool stats.
+func NewEthtoolCollector(logger log.Logger) (Collector, error) {
+	return makeEthtoolCollector(logger)
+}
+
+func (c *ethtoolCollector) Update(ch chan<- prometheus.Metric) error {
+	netClass, err := c.fs.NetClass()
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) || errors.Is(err, os.ErrPermission) {
+			level.Debug(c.logger).Log("msg", "Could not read netclass file", "err", err)
+			return ErrNoData
+		}
+		return fmt.Errorf("could not get net class info: %w", err)
+	}
+
+	if len(netClass) == 0 {
+		return fmt.Errorf("no network devices found")
+	}
+
+	for device := range netClass {
+		var stats map[string]uint64
+		var err error
+
+		stats, err = c.stats.Stats(device)
+
+		// If Stats() returns EOPNOTSUPP it doesn't support ethtool stats. Log that only at Debug level.
+		// Otherwise log it at Error level.
+		if err != nil {
+			if errno, ok := err.(syscall.Errno); ok {
+				if err == unix.EOPNOTSUPP {
+					level.Debug(c.logger).Log("msg", "ethtool stats error", "err", err, "device", device, "errno", uint(errno))
+				} else if errno != 0 {
+					level.Error(c.logger).Log("msg", "ethtool stats error", "err", err, "device", device, "errno", uint(errno))
+				}
+			} else {
+				level.Error(c.logger).Log("msg", "ethtool stats error", "err", err, "device", device)
+			}
+		}
+
+		if stats == nil || len(stats) < 1 {
+			// No stats returned; device does not support ethtool stats.
+			continue
+		}
+
+		// Sort metric names so that the test fixtures will match up
+		keys := make([]string, 0, len(stats))
+		for k := range stats {
+			keys = append(keys, k)
+		}
+		sort.Strings(keys)
+
+		for _, metric := range keys {
+			val := stats[metric]
+			metricFQName := prometheus.BuildFQName(namespace, "ethtool", metric)
+			metricFQName = receivedRegex.ReplaceAllString(metricFQName, "_received_")
+			metricFQName = transmittedRegex.ReplaceAllString(metricFQName, "_transmitted_")
+
+			// Check to see if this metric exists; if not then create it and store it in c.entries.
+			entry, exists := c.entries[metric]
+			if !exists {
+				entry = prometheus.NewDesc(
+					metricFQName,
+					fmt.Sprintf("Network interface %s", metric),
+					[]string{"device"}, nil,
+				)
+				c.entries[metric] = entry
+			}
+			ch <- prometheus.MustNewConstMetric(
+				entry, prometheus.UntypedValue, float64(val), device)
+		}
+	}
+
+	return nil
+}

collector/ethtool_linux_test.go

@@ -0,0 +1,129 @@
+// Copyright 2021 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collector
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+	"testing"
+
+	"github.com/go-kit/kit/log"
+	"github.com/prometheus/client_golang/prometheus"
+	"golang.org/x/sys/unix"
+)
+
+type EthtoolFixture struct {
+	fixturePath string
+}
+
+func (e *EthtoolFixture) Stats(intf string) (map[string]uint64, error) {
+	res := make(map[string]uint64)
+
+	fixtureFile, err := os.Open(filepath.Join(e.fixturePath, intf))
+	if e, ok := err.(*os.PathError); ok && e.Err == syscall.ENOENT {
+		// The fixture for this interface doesn't exist. Translate that to unix.EOPNOTSUPP
+		// to replicate an interface that doesn't support ethtool stats
+		return res, unix.EOPNOTSUPP
+	}
+	if err != nil {
+		return res, err
+	}
+	defer fixtureFile.Close()
+
+	scanner := bufio.NewScanner(fixtureFile)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasPrefix(line, "#") {
+			continue
+		}
+		if strings.HasPrefix(line, "NIC statistics:") {
+			continue
+		}
+		line = strings.Trim(line, " ")
+		items := strings.Split(line, ": ")
+		val, err := strconv.ParseUint(items[1], 10, 64)
+		if err != nil {
+			return res, err
+		}
+		if items[0] == "ERROR" {
+			return res, unix.Errno(val)
+		}
+		res[items[0]] = val
+	}
+
+	return res, err
+}
+
+func NewEthtoolTestCollector(logger log.Logger) (Collector, error) {
+	collector, err := makeEthtoolCollector(logger)
+	collector.stats = &EthtoolFixture{
+		fixturePath: "fixtures/ethtool/",
+	}
+	if err != nil {
+		return nil, err
+	}
+	return collector, nil
+}
+
+func TestEthtoolCollector(t *testing.T) {
+	testcases := []string{
+		prometheus.NewDesc("node_ethtool_align_errors", "Network interface align_errors", []string{"device"}, nil).String(),
+		prometheus.NewDesc("node_ethtool_received_broadcast", "Network interface rx_broadcast", []string{"device"}, nil).String(),
+		prometheus.NewDesc("node_ethtool_received_errors_total", "Number of received frames with errors", []string{"device"}, nil).String(),
+		prometheus.NewDesc("node_ethtool_received_missed", "Network interface rx_missed", []string{"device"}, nil).String(),
+		prometheus.NewDesc("node_ethtool_received_multicast", "Network interface rx_multicast", []string{"device"}, nil).String(),
+		prometheus.NewDesc("node_ethtool_received_packets_total", "Network interface packets received", []string{"device"}, nil).String(),
+		prometheus.NewDesc("node_ethtool_received_unicast", "Network interface rx_unicast", []string{"device"}, nil).String(),
+		prometheus.NewDesc("node_ethtool_transmitted_aborted", "Network interface tx_aborted", []string{"device"}, nil).String(),
+		prometheus.NewDesc("node_ethtool_transmitted_errors_total", "Number of sent frames with errors", []string{"device"}, nil).String(),
+		prometheus.NewDesc("node_ethtool_transmitted_multi_collisions", "Network interface tx_multi_collisions", []string{"device"}, nil).String(),
+		prometheus.NewDesc("node_ethtool_transmitted_packets_total", "Network interface packets sent", []string{"device"}, nil).String(),
+	}
+
+	*sysPath = "fixtures/sys"
+
+	collector, err := NewEthtoolTestCollector(log.NewNopLogger())
+	if err != nil {
+		panic(err)
+	}
+
+	sink := make(chan prometheus.Metric)
+	go func() {
+		err = collector.Update(sink)
+		if err != nil {
+			panic(fmt.Errorf("failed to update collector: %s", err))
+		}
+		close(sink)
+	}()
+
+	for _, expected := range testcases {
+		metric := (<-sink)
+		if metric == nil {
+			t.Fatalf("Expected '%s' but got nothing (nil).", expected)
+		}
+
+		got := metric.Desc().String()
+		metric.Desc()
+		if expected != got {
+			t.Errorf("Expected '%s' but got '%s'", expected, got)
+		} else {
+			t.Logf("Successfully got '%s'", got)
+		}
+	}
+}

collector/fixtures/ethtool/bond0

@@ -0,0 +1 @@
+ERROR: 1

collector/fixtures/ethtool/eth0

@@ -0,0 +1,15 @@
+# ethtool -S eth0
+NIC statistics:
+     tx_packets: 961500
+     rx_packets: 1260062
+     tx_errors: 0
+     rx_errors: 0
+     rx_missed: 401
+     align_errors: 0
+     tx_single_collisions: 0
+     tx_multi_collisions: 0
+     rx_unicast: 1230297
+     rx_broadcast: 5792
+     rx_multicast: 23973
+     tx_aborted: 0
+     tx_underrun: 0

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/prometheus/common v0.26.0
 	github.com/prometheus/exporter-toolkit v0.5.1
 	github.com/prometheus/procfs v0.6.0
+	github.com/safchain/ethtool v0.0.0-20200804214954-8f958a28363a
 	github.com/siebenmann/go-kstat v0.0.0-20200303194639-4e8294f9e9d5
 	github.com/soundcloud/go-runit v0.0.0-20150630195641-06ad41a06c4a
 	golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c

go.sum

@@ -288,6 +288,8 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/safchain/ethtool v0.0.0-20200804214954-8f958a28363a h1:TXFp1qmI50hk8dfGl7o8PKT9Lxo84T7RlpKCOB1DndI=
+github.com/safchain/ethtool v0.0.0-20200804214954-8f958a28363a/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=