Add node_softirqs_total metric (#2221)

This adds a new Linux metric, node_softirqs_total, which corresponds
to the 'softirq' line in /proc/stat. This metric is disabled by
default and it can be enabled with '--collector.stat.softirq'.

Signed-off-by: Jacob Vosmaer <jacob@gitlab.com>

collector/fixtures/e2e-output.txt

@@ -3080,6 +3080,18 @@ node_sockstat_UDP_mem_bytes 0
 # HELP node_sockstat_sockets_used Number of IPv4 sockets in use.
 # TYPE node_sockstat_sockets_used gauge
 node_sockstat_sockets_used 229
+# HELP node_softirqs_total Number of softirq calls.
+# TYPE node_softirqs_total counter
+node_softirqs_total{vector="block"} 186066
+node_softirqs_total{vector="block_iopoll"} 0
+node_softirqs_total{vector="hi"} 250191
+node_softirqs_total{vector="hrtimer"} 12499
+node_softirqs_total{vector="net_rx"} 211099
+node_softirqs_total{vector="net_tx"} 1647
+node_softirqs_total{vector="rcu"} 508444
+node_softirqs_total{vector="sched"} 622196
+node_softirqs_total{vector="tasklet"} 1.783454e+06
+node_softirqs_total{vector="timer"} 1.481983e+06
 # HELP node_softnet_dropped_total Number of dropped packets
 # TYPE node_softnet_dropped_total counter
 node_softnet_dropped_total{cpu="0"} 0

collector/stat_linux.go

@@ -22,6 +22,7 @@ import (
 	"github.com/go-kit/log"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/procfs"
+	"gopkg.in/alecthomas/kingpin.v2"
 )
 
 type statCollector struct {
@@ -32,9 +33,12 @@ type statCollector struct {
 	btime        *prometheus.Desc
 	procsRunning *prometheus.Desc
 	procsBlocked *prometheus.Desc
+	softIRQ      *prometheus.Desc
 	logger       log.Logger
 }
 
+var statSoftirqFlag = kingpin.Flag("collector.stat.softirq", "Export softirq calls per vector").Default("false").Bool()
+
 func init() {
 	registerCollector("stat", defaultEnabled, NewStatCollector)
 }
@@ -77,6 +81,11 @@ func NewStatCollector(logger log.Logger) (Collector, error) {
 			"Number of processes blocked waiting for I/O to complete.",
 			nil, nil,
 		),
+		softIRQ: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, "", "softirqs_total"),
+			"Number of softirq calls.",
+			[]string{"vector"}, nil,
+		),
 		logger: logger,
 	}, nil
 }
@@ -97,5 +106,27 @@ func (c *statCollector) Update(ch chan<- prometheus.Metric) error {
 	ch <- prometheus.MustNewConstMetric(c.procsRunning, prometheus.GaugeValue, float64(stats.ProcessesRunning))
 	ch <- prometheus.MustNewConstMetric(c.procsBlocked, prometheus.GaugeValue, float64(stats.ProcessesBlocked))
 
+	if *statSoftirqFlag {
+		si := stats.SoftIRQ
+
+		for _, vec := range []struct {
+			name  string
+			value uint64
+		}{
+			{name: "hi", value: si.Hi},
+			{name: "timer", value: si.Timer},
+			{name: "net_tx", value: si.NetTx},
+			{name: "net_rx", value: si.NetRx},
+			{name: "block", value: si.Block},
+			{name: "block_iopoll", value: si.BlockIoPoll},
+			{name: "tasklet", value: si.Tasklet},
+			{name: "sched", value: si.Sched},
+			{name: "hrtimer", value: si.Hrtimer},
+			{name: "rcu", value: si.Rcu},
+		} {
+			ch <- prometheus.MustNewConstMetric(c.softIRQ, prometheus.CounterValue, float64(vec.value), vec.name)
+		}
+	}
+
 	return nil
 }

end-to-end-test.sh

@@ -114,6 +114,7 @@ fi
   --collector.cpu.info \
   --collector.cpu.info.flags-include="^(aes|avx.?|constant_tsc)$" \
   --collector.cpu.info.bugs-include="^(cpu_meltdown|spectre_.*|mds)$" \
+  --collector.stat.softirq \
   --web.listen-address "127.0.0.1:${port}" \
   --log.level="debug" > "${tmpdir}/node_exporter.log" 2>&1 &