Merge pull request #367 from mdlayher/mountstats

Add mountstats collector for detailed NFS statistics

README.md

@@ -49,6 +49,7 @@ ksmd | Exposes kernel and system statistics from `/sys/kernel/mm/ksm`. | Linux
 logind | Exposes session counts from [logind](http://www.freedesktop.org/wiki/Software/systemd/logind/). | Linux
 megacli | Exposes RAID statistics from MegaCLI. | Linux
 meminfo_numa | Exposes memory statistics from `/proc/meminfo_numa`. | Linux
+mountstats | Exposes filesystem statistics from `/proc/self/mountstats`. Exposes detailed NFS client statistics. | Linux
 nfs | Exposes NFS client statistics from `/proc/net/rpc/nfs`. This is the same information as `nfsstat -c`. | Linux
 ntp | Exposes time drift from an NTP server. | _any_
 runit | Exposes service status from [runit](http://smarden.org/runit/). | _any_

collector/fixtures/e2e-output.txt

@@ -443,6 +443,11 @@ node_exporter_scrape_duration_seconds{collector="meminfo_numa",result="success",
 node_exporter_scrape_duration_seconds{collector="meminfo_numa",result="success",quantile="0.99"} 0.0017954980000000002
 node_exporter_scrape_duration_seconds_sum{collector="meminfo_numa",result="success"} 0.0017954980000000002
 node_exporter_scrape_duration_seconds_count{collector="meminfo_numa",result="success"} 1
+node_exporter_scrape_duration_seconds{collector="mountstats",result="success",quantile="0.5"} 0.000512374
+node_exporter_scrape_duration_seconds{collector="mountstats",result="success",quantile="0.9"} 0.000512374
+node_exporter_scrape_duration_seconds{collector="mountstats",result="success",quantile="0.99"} 0.000512374
+node_exporter_scrape_duration_seconds_sum{collector="mountstats",result="success"} 0.000512374
+node_exporter_scrape_duration_seconds_count{collector="mountstats",result="success"} 1
 node_exporter_scrape_duration_seconds{collector="netdev",result="success",quantile="0.5"} 0.000495389
 node_exporter_scrape_duration_seconds{collector="netdev",result="success",quantile="0.9"} 0.000495389
 node_exporter_scrape_duration_seconds{collector="netdev",result="success",quantile="0.99"} 0.000495389
@@ -1070,6 +1075,103 @@ node_memory_numa_numa_miss_total{node="1"} 5.9858626709e+10
 # TYPE node_memory_numa_other_node_total counter
 node_memory_numa_other_node_total{node="0"} 1.8179487e+07
 node_memory_numa_other_node_total{node="1"} 5.986052692e+10
+# HELP node_mountstats_nfs_age_seconds_total The age of the NFS mount in seconds.
+# TYPE node_mountstats_nfs_age_seconds_total counter
+node_mountstats_nfs_age_seconds_total{export="192.168.1.1:/srv/test"} 13968
+# HELP node_mountstats_nfs_direct_read_bytes_total Number of bytes read using the read() syscall in O_DIRECT mode.
+# TYPE node_mountstats_nfs_direct_read_bytes_total counter
+node_mountstats_nfs_direct_read_bytes_total{export="192.168.1.1:/srv/test"} 0
+# HELP node_mountstats_nfs_direct_write_bytes_total Number of bytes written using the write() syscall in O_DIRECT mode.
+# TYPE node_mountstats_nfs_direct_write_bytes_total counter
+node_mountstats_nfs_direct_write_bytes_total{export="192.168.1.1:/srv/test"} 0
+# HELP node_mountstats_nfs_operations_major_timeouts_total Number of times a request has had a major timeout for a given operation.
+# TYPE node_mountstats_nfs_operations_major_timeouts_total counter
+node_mountstats_nfs_operations_major_timeouts_total{export="192.168.1.1:/srv/test",operation="NULL"} 0
+node_mountstats_nfs_operations_major_timeouts_total{export="192.168.1.1:/srv/test",operation="READ"} 0
+node_mountstats_nfs_operations_major_timeouts_total{export="192.168.1.1:/srv/test",operation="WRITE"} 0
+# HELP node_mountstats_nfs_operations_queue_time_seconds_total Duration all requests spent queued for transmission for a given operation before they were sent, in seconds.
+# TYPE node_mountstats_nfs_operations_queue_time_seconds_total counter
+node_mountstats_nfs_operations_queue_time_seconds_total{export="192.168.1.1:/srv/test",operation="NULL"} 0
+node_mountstats_nfs_operations_queue_time_seconds_total{export="192.168.1.1:/srv/test",operation="READ"} 0.006
+node_mountstats_nfs_operations_queue_time_seconds_total{export="192.168.1.1:/srv/test",operation="WRITE"} 0
+# HELP node_mountstats_nfs_operations_received_bytes_total Number of bytes received for a given operation, including RPC headers and payload.
+# TYPE node_mountstats_nfs_operations_received_bytes_total counter
+node_mountstats_nfs_operations_received_bytes_total{export="192.168.1.1:/srv/test",operation="NULL"} 0
+node_mountstats_nfs_operations_received_bytes_total{export="192.168.1.1:/srv/test",operation="READ"} 1.210292152e+09
+node_mountstats_nfs_operations_received_bytes_total{export="192.168.1.1:/srv/test",operation="WRITE"} 0
+# HELP node_mountstats_nfs_operations_request_time_seconds_total Duration all requests took from when a request was enqueued to when it was completely handled for a given operation, in seconds.
+# TYPE node_mountstats_nfs_operations_request_time_seconds_total counter
+node_mountstats_nfs_operations_request_time_seconds_total{export="192.168.1.1:/srv/test",operation="NULL"} 0
+node_mountstats_nfs_operations_request_time_seconds_total{export="192.168.1.1:/srv/test",operation="READ"} 79.407
+node_mountstats_nfs_operations_request_time_seconds_total{export="192.168.1.1:/srv/test",operation="WRITE"} 0
+# HELP node_mountstats_nfs_operations_requests_total Number of requests performed for a given operation.
+# TYPE node_mountstats_nfs_operations_requests_total counter
+node_mountstats_nfs_operations_requests_total{export="192.168.1.1:/srv/test",operation="NULL"} 0
+node_mountstats_nfs_operations_requests_total{export="192.168.1.1:/srv/test",operation="READ"} 1298
+node_mountstats_nfs_operations_requests_total{export="192.168.1.1:/srv/test",operation="WRITE"} 0
+# HELP node_mountstats_nfs_operations_response_time_seconds_total Duration all requests took to get a reply back after a request for a given operation was transmitted, in seconds.
+# TYPE node_mountstats_nfs_operations_response_time_seconds_total counter
+node_mountstats_nfs_operations_response_time_seconds_total{export="192.168.1.1:/srv/test",operation="NULL"} 0
+node_mountstats_nfs_operations_response_time_seconds_total{export="192.168.1.1:/srv/test",operation="READ"} 79.386
+node_mountstats_nfs_operations_response_time_seconds_total{export="192.168.1.1:/srv/test",operation="WRITE"} 0
+# HELP node_mountstats_nfs_operations_sent_bytes_total Number of bytes sent for a given operation, including RPC headers and payload.
+# TYPE node_mountstats_nfs_operations_sent_bytes_total counter
+node_mountstats_nfs_operations_sent_bytes_total{export="192.168.1.1:/srv/test",operation="NULL"} 0
+node_mountstats_nfs_operations_sent_bytes_total{export="192.168.1.1:/srv/test",operation="READ"} 207680
+node_mountstats_nfs_operations_sent_bytes_total{export="192.168.1.1:/srv/test",operation="WRITE"} 0
+# HELP node_mountstats_nfs_operations_transmissions_total Number of times an actual RPC request has been transmitted for a given operation.
+# TYPE node_mountstats_nfs_operations_transmissions_total counter
+node_mountstats_nfs_operations_transmissions_total{export="192.168.1.1:/srv/test",operation="NULL"} 0
+node_mountstats_nfs_operations_transmissions_total{export="192.168.1.1:/srv/test",operation="READ"} 1298
+node_mountstats_nfs_operations_transmissions_total{export="192.168.1.1:/srv/test",operation="WRITE"} 0
+# HELP node_mountstats_nfs_read_bytes_total Number of bytes read using the read() syscall.
+# TYPE node_mountstats_nfs_read_bytes_total counter
+node_mountstats_nfs_read_bytes_total{export="192.168.1.1:/srv/test"} 1.20764023e+09
+# HELP node_mountstats_nfs_read_pages_total Number of pages read directly via mmap()'d files.
+# TYPE node_mountstats_nfs_read_pages_total counter
+node_mountstats_nfs_read_pages_total{export="192.168.1.1:/srv/test"} 295483
+# HELP node_mountstats_nfs_total_read_bytes_total Number of bytes read from the NFS server, in total.
+# TYPE node_mountstats_nfs_total_read_bytes_total counter
+node_mountstats_nfs_total_read_bytes_total{export="192.168.1.1:/srv/test"} 1.210214218e+09
+# HELP node_mountstats_nfs_total_write_bytes_total Number of bytes written to the NFS server, in total.
+# TYPE node_mountstats_nfs_total_write_bytes_total counter
+node_mountstats_nfs_total_write_bytes_total{export="192.168.1.1:/srv/test"} 0
+# HELP node_mountstats_nfs_transport_backlog_queue_total Total number of items added to the RPC backlog queue.
+# TYPE node_mountstats_nfs_transport_backlog_queue_total counter
+node_mountstats_nfs_transport_backlog_queue_total{export="192.168.1.1:/srv/test"} 0
+# HELP node_mountstats_nfs_transport_bad_transaction_ids_total Number of times the NFS server sent a response with a transaction ID unknown to this client.
+# TYPE node_mountstats_nfs_transport_bad_transaction_ids_total counter
+node_mountstats_nfs_transport_bad_transaction_ids_total{export="192.168.1.1:/srv/test"} 0
+# HELP node_mountstats_nfs_transport_bind_total Number of times the client has had to establish a connection from scratch to the NFS server.
+# TYPE node_mountstats_nfs_transport_bind_total counter
+node_mountstats_nfs_transport_bind_total{export="192.168.1.1:/srv/test"} 0
+# HELP node_mountstats_nfs_transport_connect_total Number of times the client has made a TCP connection to the NFS server.
+# TYPE node_mountstats_nfs_transport_connect_total counter
+node_mountstats_nfs_transport_connect_total{export="192.168.1.1:/srv/test"} 1
+# HELP node_mountstats_nfs_transport_idle_time_seconds Duration since the NFS mount last saw any RPC traffic, in seconds.
+# TYPE node_mountstats_nfs_transport_idle_time_seconds gauge
+node_mountstats_nfs_transport_idle_time_seconds{export="192.168.1.1:/srv/test"} 11
+# HELP node_mountstats_nfs_transport_maximum_rpc_slots Maximum number of simultaneously active RPC requests ever used.
+# TYPE node_mountstats_nfs_transport_maximum_rpc_slots gauge
+node_mountstats_nfs_transport_maximum_rpc_slots{export="192.168.1.1:/srv/test"} 24
+# HELP node_mountstats_nfs_transport_pending_queue_total Total number of items added to the RPC transmission pending queue.
+# TYPE node_mountstats_nfs_transport_pending_queue_total counter
+node_mountstats_nfs_transport_pending_queue_total{export="192.168.1.1:/srv/test"} 5726
+# HELP node_mountstats_nfs_transport_receives_total Number of RPC responses for this mount received from the NFS server.
+# TYPE node_mountstats_nfs_transport_receives_total counter
+node_mountstats_nfs_transport_receives_total{export="192.168.1.1:/srv/test"} 6428
+# HELP node_mountstats_nfs_transport_sending_queue_total Total number of items added to the RPC transmission sending queue.
+# TYPE node_mountstats_nfs_transport_sending_queue_total counter
+node_mountstats_nfs_transport_sending_queue_total{export="192.168.1.1:/srv/test"} 26
+# HELP node_mountstats_nfs_transport_sends_total Number of RPC requests for this mount sent to the NFS server.
+# TYPE node_mountstats_nfs_transport_sends_total counter
+node_mountstats_nfs_transport_sends_total{export="192.168.1.1:/srv/test"} 6428
+# HELP node_mountstats_nfs_write_bytes_total Number of bytes written using the write() syscall.
+# TYPE node_mountstats_nfs_write_bytes_total counter
+node_mountstats_nfs_write_bytes_total{export="192.168.1.1:/srv/test"} 0
+# HELP node_mountstats_nfs_write_pages_total Number of pages written directly via mmap()'d files.
+# TYPE node_mountstats_nfs_write_pages_total counter
+node_mountstats_nfs_write_pages_total{export="192.168.1.1:/srv/test"} 0
 # HELP node_net_bonding_slaves Number of configured slaves per bonding interface.
 # TYPE node_net_bonding_slaves gauge
 node_net_bonding_slaves{master="bond0"} 0

collector/fixtures/proc/10/mountstats

@@ -0,0 +1,19 @@
+device rootfs mounted on / with fstype rootfs
+device sysfs mounted on /sys with fstype sysfs
+device proc mounted on /proc with fstype proc
+device /dev/sda1 mounted on / with fstype ext4
+device 192.168.1.1:/srv/test mounted on /mnt/nfs/test with fstype nfs4 statvers=1.1
+	opts:	rw,vers=4.0,rsize=1048576,wsize=1048576,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,port=0,timeo=600,retrans=2,sec=sys,clientaddr=192.168.1.5,local_lock=none
+	age:	13968
+	caps:	caps=0xfff7,wtmult=512,dtsize=32768,bsize=0,namlen=255
+	nfsv4:	bm0=0xfdffafff,bm1=0xf9be3e,bm2=0x0,acl=0x0,pnfs=not configured
+	sec:	flavor=1,pseudoflavor=1
+	events:	52 226 0 0 1 13 398 0 0 331 0 47 0 0 77 0 0 77 0 0 0 0 0 0 0 0 0
+	bytes:	1207640230 0 0 0 1210214218 0 295483 0
+	RPC iostats version: 1.0  p/v: 100003/4 (nfs)
+	xprt:	tcp 832 0 1 0 11 6428 6428 0 12154 0 24 26 5726
+	per-op statistics
+	        NULL: 0 0 0 0 0 0 0 0
+	        READ: 1298 1298 0 207680 1210292152 6 79386 79407
+	       WRITE: 0 0 0 0 0 0 0 0
+

collector/fixtures/proc/self

@@ -0,0 +1 @@
+10

collector/mountstats_linux.go

@@ -0,0 +1,499 @@
+// Copyright 2016 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collector
+
+import (
+	"fmt"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/procfs"
+)
+
+type mountStatsCollector struct {
+	// General statistics
+	NFSAgeSecondsTotal *prometheus.Desc
+
+	// Byte statistics
+	NFSReadBytesTotal        *prometheus.Desc
+	NFSWriteBytesTotal       *prometheus.Desc
+	NFSDirectReadBytesTotal  *prometheus.Desc
+	NFSDirectWriteBytesTotal *prometheus.Desc
+	NFSTotalReadBytesTotal   *prometheus.Desc
+	NFSTotalWriteBytesTotal  *prometheus.Desc
+	NFSReadPagesTotal        *prometheus.Desc
+	NFSWritePagesTotal       *prometheus.Desc
+
+	// Per-operation statistics
+	NFSOperationsRequestsTotal            *prometheus.Desc
+	NFSOperationsTransmissionsTotal       *prometheus.Desc
+	NFSOperationsMajorTimeoutsTotal       *prometheus.Desc
+	NFSOperationsSentBytesTotal           *prometheus.Desc
+	NFSOperationsReceivedBytesTotal       *prometheus.Desc
+	NFSOperationsQueueTimeSecondsTotal    *prometheus.Desc
+	NFSOperationsResponseTimeSecondsTotal *prometheus.Desc
+	NFSOperationsRequestTimeSecondsTotal  *prometheus.Desc
+
+	// Transport statistics
+	NFSTransportBindTotal              *prometheus.Desc
+	NFSTransportConnectTotal           *prometheus.Desc
+	NFSTransportIdleTimeSeconds        *prometheus.Desc
+	NFSTransportSendsTotal             *prometheus.Desc
+	NFSTransportReceivesTotal          *prometheus.Desc
+	NFSTransportBadTransactionIDsTotal *prometheus.Desc
+	NFSTransportBacklogQueueTotal      *prometheus.Desc
+	NFSTransportMaximumRPCSlots        *prometheus.Desc
+	NFSTransportSendingQueueTotal      *prometheus.Desc
+	NFSTransportPendingQueueTotal      *prometheus.Desc
+
+	proc procfs.Proc
+}
+
+func init() {
+	Factories["mountstats"] = NewMountStatsCollector
+}
+
+func NewMountStatsCollector() (Collector, error) {
+	fs, err := procfs.NewFS(*procPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open procfs: %v", err)
+	}
+
+	proc, err := fs.Self()
+	if err != nil {
+		return nil, fmt.Errorf("failed to open /proc/self: %v", err)
+	}
+
+	const (
+		// For the time being, only NFS statistics are available via this mechanism
+		subsystem = "mountstats_nfs"
+	)
+
+	var (
+		labels   = []string{"export"}
+		opLabels = []string{"export", "operation"}
+	)
+
+	return &mountStatsCollector{
+		NFSAgeSecondsTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "age_seconds_total"),
+			"The age of the NFS mount in seconds.",
+			labels,
+			nil,
+		),
+
+		NFSReadBytesTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "read_bytes_total"),
+			"Number of bytes read using the read() syscall.",
+			labels,
+			nil,
+		),
+
+		NFSWriteBytesTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "write_bytes_total"),
+			"Number of bytes written using the write() syscall.",
+			labels,
+			nil,
+		),
+
+		NFSDirectReadBytesTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "direct_read_bytes_total"),
+			"Number of bytes read using the read() syscall in O_DIRECT mode.",
+			labels,
+			nil,
+		),
+
+		NFSDirectWriteBytesTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "direct_write_bytes_total"),
+			"Number of bytes written using the write() syscall in O_DIRECT mode.",
+			labels,
+			nil,
+		),
+
+		NFSTotalReadBytesTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "total_read_bytes_total"),
+			"Number of bytes read from the NFS server, in total.",
+			labels,
+			nil,
+		),
+
+		NFSTotalWriteBytesTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "total_write_bytes_total"),
+			"Number of bytes written to the NFS server, in total.",
+			labels,
+			nil,
+		),
+
+		NFSReadPagesTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "read_pages_total"),
+			"Number of pages read directly via mmap()'d files.",
+			labels,
+			nil,
+		),
+
+		NFSWritePagesTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "write_pages_total"),
+			"Number of pages written directly via mmap()'d files.",
+			labels,
+			nil,
+		),
+
+		NFSTransportBindTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "transport_bind_total"),
+			"Number of times the client has had to establish a connection from scratch to the NFS server.",
+			labels,
+			nil,
+		),
+
+		NFSTransportConnectTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "transport_connect_total"),
+			"Number of times the client has made a TCP connection to the NFS server.",
+			labels,
+			nil,
+		),
+
+		NFSTransportIdleTimeSeconds: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "transport_idle_time_seconds"),
+			"Duration since the NFS mount last saw any RPC traffic, in seconds.",
+			labels,
+			nil,
+		),
+
+		NFSTransportSendsTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "transport_sends_total"),
+			"Number of RPC requests for this mount sent to the NFS server.",
+			labels,
+			nil,
+		),
+
+		NFSTransportReceivesTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "transport_receives_total"),
+			"Number of RPC responses for this mount received from the NFS server.",
+			labels,
+			nil,
+		),
+
+		NFSTransportBadTransactionIDsTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "transport_bad_transaction_ids_total"),
+			"Number of times the NFS server sent a response with a transaction ID unknown to this client.",
+			labels,
+			nil,
+		),
+
+		NFSTransportBacklogQueueTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "transport_backlog_queue_total"),
+			"Total number of items added to the RPC backlog queue.",
+			labels,
+			nil,
+		),
+
+		NFSTransportMaximumRPCSlots: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "transport_maximum_rpc_slots"),
+			"Maximum number of simultaneously active RPC requests ever used.",
+			labels,
+			nil,
+		),
+
+		NFSTransportSendingQueueTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "transport_sending_queue_total"),
+			"Total number of items added to the RPC transmission sending queue.",
+			labels,
+			nil,
+		),
+
+		NFSTransportPendingQueueTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "transport_pending_queue_total"),
+			"Total number of items added to the RPC transmission pending queue.",
+			labels,
+			nil,
+		),
+
+		NFSOperationsRequestsTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "operations_requests_total"),
+			"Number of requests performed for a given operation.",
+			opLabels,
+			nil,
+		),
+
+		NFSOperationsTransmissionsTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "operations_transmissions_total"),
+			"Number of times an actual RPC request has been transmitted for a given operation.",
+			opLabels,
+			nil,
+		),
+
+		NFSOperationsMajorTimeoutsTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "operations_major_timeouts_total"),
+			"Number of times a request has had a major timeout for a given operation.",
+			opLabels,
+			nil,
+		),
+
+		NFSOperationsSentBytesTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "operations_sent_bytes_total"),
+			"Number of bytes sent for a given operation, including RPC headers and payload.",
+			opLabels,
+			nil,
+		),
+
+		NFSOperationsReceivedBytesTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "operations_received_bytes_total"),
+			"Number of bytes received for a given operation, including RPC headers and payload.",
+			opLabels,
+			nil,
+		),
+
+		NFSOperationsQueueTimeSecondsTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "operations_queue_time_seconds_total"),
+			"Duration all requests spent queued for transmission for a given operation before they were sent, in seconds.",
+			opLabels,
+			nil,
+		),
+
+		NFSOperationsResponseTimeSecondsTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "operations_response_time_seconds_total"),
+			"Duration all requests took to get a reply back after a request for a given operation was transmitted, in seconds.",
+			opLabels,
+			nil,
+		),
+
+		NFSOperationsRequestTimeSecondsTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "operations_request_time_seconds_total"),
+			"Duration all requests took from when a request was enqueued to when it was completely handled for a given operation, in seconds.",
+			opLabels,
+			nil,
+		),
+
+		proc: proc,
+	}, nil
+}
+
+func (c *mountStatsCollector) Update(ch chan<- prometheus.Metric) error {
+	mounts, err := c.proc.MountStats()
+	if err != nil {
+		return fmt.Errorf("failed to parse mountstats: %v", err)
+	}
+
+	for _, m := range mounts {
+		// For the time being, only NFS statistics are available via this mechanism
+		stats, ok := m.Stats.(*procfs.MountStatsNFS)
+		if !ok {
+			continue
+		}
+
+		c.updateNFSStats(ch, m.Device, stats)
+	}
+
+	return nil
+}
+
+func (c *mountStatsCollector) updateNFSStats(ch chan<- prometheus.Metric, export string, s *procfs.MountStatsNFS) {
+	ch <- prometheus.MustNewConstMetric(
+		c.NFSAgeSecondsTotal,
+		prometheus.CounterValue,
+		s.Age.Seconds(),
+		export,
+	)
+
+	ch <- prometheus.MustNewConstMetric(
+		c.NFSReadBytesTotal,
+		prometheus.CounterValue,
+		float64(s.Bytes.Read),
+		export,
+	)
+
+	ch <- prometheus.MustNewConstMetric(
+		c.NFSWriteBytesTotal,
+		prometheus.CounterValue,
+		float64(s.Bytes.Write),
+		export,
+	)
+
+	ch <- prometheus.MustNewConstMetric(
+		c.NFSDirectReadBytesTotal,
+		prometheus.CounterValue,
+		float64(s.Bytes.DirectRead),
+		export,
+	)
+
+	ch <- prometheus.MustNewConstMetric(
+		c.NFSDirectWriteBytesTotal,
+		prometheus.CounterValue,
+		float64(s.Bytes.DirectWrite),
+		export,
+	)
+
+	ch <- prometheus.MustNewConstMetric(
+		c.NFSTotalReadBytesTotal,
+		prometheus.CounterValue,
+		float64(s.Bytes.ReadTotal),
+		export,
+	)
+
+	ch <- prometheus.MustNewConstMetric(
+		c.NFSTotalWriteBytesTotal,
+		prometheus.CounterValue,
+		float64(s.Bytes.WriteTotal),
+		export,
+	)
+
+	ch <- prometheus.MustNewConstMetric(
+		c.NFSReadPagesTotal,
+		prometheus.CounterValue,
+		float64(s.Bytes.ReadPages),
+		export,
+	)
+
+	ch <- prometheus.MustNewConstMetric(
+		c.NFSWritePagesTotal,
+		prometheus.CounterValue,
+		float64(s.Bytes.WritePages),
+		export,
+	)
+
+	ch <- prometheus.MustNewConstMetric(
+		c.NFSTransportBindTotal,
+		prometheus.CounterValue,
+		float64(s.Transport.Bind),
+		export,
+	)
+
+	ch <- prometheus.MustNewConstMetric(
+		c.NFSTransportConnectTotal,
+		prometheus.CounterValue,
+		float64(s.Transport.Connect),
+		export,
+	)
+
+	ch <- prometheus.MustNewConstMetric(
+		c.NFSTransportIdleTimeSeconds,
+		prometheus.GaugeValue,
+		s.Transport.IdleTime.Seconds(),
+		export,
+	)
+
+	ch <- prometheus.MustNewConstMetric(
+		c.NFSTransportSendsTotal,
+		prometheus.CounterValue,
+		float64(s.Transport.Sends),
+		export,
+	)
+
+	ch <- prometheus.MustNewConstMetric(
+		c.NFSTransportReceivesTotal,
+		prometheus.CounterValue,
+		float64(s.Transport.Receives),
+		export,
+	)
+
+	ch <- prometheus.MustNewConstMetric(
+		c.NFSTransportBadTransactionIDsTotal,
+		prometheus.CounterValue,
+		float64(s.Transport.BadTransactionIDs),
+		export,
+	)
+
+	ch <- prometheus.MustNewConstMetric(
+		c.NFSTransportBacklogQueueTotal,
+		prometheus.CounterValue,
+		float64(s.Transport.CumulativeBacklog),
+		export,
+	)
+
+	ch <- prometheus.MustNewConstMetric(
+		c.NFSTransportMaximumRPCSlots,
+		prometheus.GaugeValue,
+		float64(s.Transport.MaximumRPCSlotsUsed),
+		export,
+	)
+
+	ch <- prometheus.MustNewConstMetric(
+		c.NFSTransportSendingQueueTotal,
+		prometheus.CounterValue,
+		float64(s.Transport.CumulativeSendingQueue),
+		export,
+	)
+
+	ch <- prometheus.MustNewConstMetric(
+		c.NFSTransportPendingQueueTotal,
+		prometheus.CounterValue,
+		float64(s.Transport.CumulativePendingQueue),
+		export,
+	)
+
+	for _, op := range s.Operations {
+		ch <- prometheus.MustNewConstMetric(
+			c.NFSOperationsRequestsTotal,
+			prometheus.CounterValue,
+			float64(op.Requests),
+			export,
+			op.Operation,
+		)
+
+		ch <- prometheus.MustNewConstMetric(
+			c.NFSOperationsTransmissionsTotal,
+			prometheus.CounterValue,
+			float64(op.Transmissions),
+			export,
+			op.Operation,
+		)
+
+		ch <- prometheus.MustNewConstMetric(
+			c.NFSOperationsMajorTimeoutsTotal,
+			prometheus.CounterValue,
+			float64(op.MajorTimeouts),
+			export,
+			op.Operation,
+		)
+
+		ch <- prometheus.MustNewConstMetric(
+			c.NFSOperationsSentBytesTotal,
+			prometheus.CounterValue,
+			float64(op.BytesSent),
+			export,
+			op.Operation,
+		)
+
+		ch <- prometheus.MustNewConstMetric(
+			c.NFSOperationsReceivedBytesTotal,
+			prometheus.CounterValue,
+			float64(op.BytesReceived),
+			export,
+			op.Operation,
+		)
+
+		ch <- prometheus.MustNewConstMetric(
+			c.NFSOperationsQueueTimeSecondsTotal,
+			prometheus.CounterValue,
+			op.CumulativeQueueTime.Seconds(),
+			export,
+			op.Operation,
+		)
+
+		ch <- prometheus.MustNewConstMetric(
+			c.NFSOperationsResponseTimeSecondsTotal,
+			prometheus.CounterValue,
+			op.CumulativeTotalResponseTime.Seconds(),
+			export,
+			op.Operation,
+		)
+
+		ch <- prometheus.MustNewConstMetric(
+			c.NFSOperationsRequestTimeSecondsTotal,
+			prometheus.CounterValue,
+			op.CumulativeTotalRequestTime.Seconds(),
+			export,
+			op.Operation,
+		)
+	}
+}

end-to-end-test.sh

@@ -13,6 +13,7 @@ collectors=$(cat << COLLECTORS
   mdadm
   meminfo
   meminfo_numa
+  mountstats
   netdev
   netstat
   nfs