From 01bd99fb1a3a7fca02deaa5cf5ba7f667ea33ea7 Mon Sep 17 00:00:00 2001 From: Ben Kochie Date: Thu, 15 Feb 2018 13:40:38 +0100 Subject: [PATCH] Refactor NFS client collector (#816) * Update vendor github.com/prometheus/procfs/... * Refactor NFS collector Use new procfs library to parse NFS client stats. * Ignore nfs proc file not existing. * Refactor with reflection to walk the structs. --- collector/nfs_linux.go | 303 ++++++++++-------- vendor/github.com/prometheus/procfs/fs.go | 4 +- .../github.com/prometheus/procfs/nfs/parse.go | 13 +- .../prometheus/procfs/nfs/parse_nfs.go | 10 +- vendor/vendor.json | 28 +- 5 files changed, 202 insertions(+), 156 deletions(-) diff --git a/collector/nfs_linux.go b/collector/nfs_linux.go index 8241a44f..ff3d529c 100644 --- a/collector/nfs_linux.go +++ b/collector/nfs_linux.go @@ -14,98 +14,30 @@ package collector import ( - "errors" - "io/ioutil" + "fmt" "os" - "regexp" - "strconv" + "reflect" "strings" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/log" + "github.com/prometheus/procfs" + "github.com/prometheus/procfs/nfs" ) -var ( - netLineRE = regexp.MustCompile(`^net \d+ (\d+) (\d+) (\d+)$`) - rpcLineRE = regexp.MustCompile(`^rpc (\d+) (\d+) (\d+)$`) - procLineRE = regexp.MustCompile(`^proc(\d+) \d+ (\d+( \d+)*)$`) - - nfsProcedures = map[string][]string{ - "2": { - "null", "getattr", "setattr", "root", "lookup", - "readlink", "read", "writecache", "write", "create", - "remove", "rename", "link", "symlink", "mkdir", - "rmdir", "readdir", "statfs", - }, - "3": { - "null", "getattr", "setattr", "lookup", "access", - "readlink", "read", "write", "create", "mkdir", - "symlink", "mknod", "remove", "rmdir", "rename", - "link", "readdir", "readdirplus", "fsstat", "fsinfo", - "pathconf", "commit", - }, - "4": { - "null", "read", "write", "commit", "open", - "open_confirm", "open_noattr", "open_downgrade", - "close", "setattr", "fsinfo", "renew", "setclientid", - "setclientid_confirm", "lock", "lockt", "locku", - "access", "getattr", "lookup", "lookup_root", "remove", - "rename", "link", "symlink", "create", "pathconf", - "statfs", "readlink", "readdir", "server_caps", - "delegreturn", "getacl", "setacl", "fs_locations", - "release_lockowner", "secinfo", "fsid_present", - "exchange_id", "create_session", "destroy_session", - "sequence", "get_lease_time", "reclaim_complete", - "layoutget", "getdeviceinfo", "layoutcommit", - "layoutreturn", "secinfo_no_name", "test_stateid", - "free_stateid", "getdevicelist", - "bind_conn_to_session", "destroy_clientid", "seek", - "allocate", "deallocate", "layoutstats", "clone", - "copy", - }, - } - - nfsNetReadsDesc = prometheus.NewDesc( - prometheus.BuildFQName(namespace, "nfs", "net_reads_total"), - "Number of reads at the network layer.", - []string{"protocol"}, - nil, - ) - nfsNetConnectionsDesc = prometheus.NewDesc( - prometheus.BuildFQName(namespace, "nfs", "net_connections_total"), - "Number of connections at the network layer.", - []string{"protocol"}, - nil, - ) - - nfsRPCOperationsDesc = prometheus.NewDesc( - prometheus.BuildFQName(namespace, "nfs", "rpc_operations_total"), - "Number of RPCs performed.", - nil, - nil, - ) - nfsRPCRetransmissionsDesc = prometheus.NewDesc( - prometheus.BuildFQName(namespace, "nfs", "rpc_retransmissions_total"), - "Number of RPC transmissions performed.", - nil, - nil, - ) - nfsRPCAuthenticationRefreshesDesc = prometheus.NewDesc( - prometheus.BuildFQName(namespace, "nfs", "rpc_authentication_refreshes_total"), - "Number of RPC authentication refreshes performed.", - nil, - nil, - ) - - nfsProceduresDesc = prometheus.NewDesc( - prometheus.BuildFQName(namespace, "nfs", "procedures_total"), - "Number of NFS procedures invoked.", - []string{"version", "procedure"}, - nil, - ) +const ( + nfsSubsystem = "nfs" ) -type nfsCollector struct{} +type nfsCollector struct { + fs procfs.FS + nfsNetReadsDesc *prometheus.Desc + nfsNetConnectionsDesc *prometheus.Desc + nfsRPCOperationsDesc *prometheus.Desc + nfsRPCRetransmissionsDesc *prometheus.Desc + nfsRPCAuthenticationRefreshesDesc *prometheus.Desc + nfsProceduresDesc *prometheus.Desc +} func init() { registerCollector("nfs", defaultDisabled, NewNfsCollector) @@ -113,65 +45,170 @@ func init() { // NewNfsCollector returns a new Collector exposing NFS statistics. func NewNfsCollector() (Collector, error) { - return &nfsCollector{}, nil + fs, err := procfs.NewFS(*procPath) + if err != nil { + return nil, fmt.Errorf("failed to open procfs: %v", err) + } + + return &nfsCollector{ + fs: fs, + nfsNetReadsDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, nfsSubsystem, "net_reads_total"), + "Number of reads at the network layer.", + []string{"protocol"}, + nil, + ), + nfsNetConnectionsDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, nfsSubsystem, "net_connections_total"), + "Number of connections at the network layer.", + []string{"protocol"}, + nil, + ), + nfsRPCOperationsDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, nfsSubsystem, "rpc_operations_total"), + "Number of RPCs performed.", + nil, + nil, + ), + nfsRPCRetransmissionsDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, nfsSubsystem, "rpc_retransmissions_total"), + "Number of RPC transmissions performed.", + nil, + nil, + ), + nfsRPCAuthenticationRefreshesDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, nfsSubsystem, "rpc_authentication_refreshes_total"), + "Number of RPC authentication refreshes performed.", + nil, + nil, + ), + nfsProceduresDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "nfs", "procedures_total"), + "Number of NFS procedures invoked.", + []string{"version", "procedure"}, + nil, + ), + }, nil } func (c *nfsCollector) Update(ch chan<- prometheus.Metric) error { - statsFile := procFilePath("net/rpc/nfs") - content, err := ioutil.ReadFile(statsFile) + stats, err := c.fs.NFSClientRPCStats() if err != nil { if os.IsNotExist(err) { - log.Debugf("Not collecting NFS statistics, as %q does not exist", statsFile) + log.Debugf("Not collecting NFS metrics: %s", err) return nil } - return err + return fmt.Errorf("failed to retrieve nfs stats: %v", err) } - for _, line := range strings.Split(string(content), "\n") { - if fields := netLineRE.FindStringSubmatch(line); fields != nil { - value, _ := strconv.ParseFloat(fields[1], 64) - ch <- prometheus.MustNewConstMetric( - nfsNetReadsDesc, prometheus.CounterValue, - value, "udp") + c.updateNFSNetworkStats(ch, &stats.Network) + c.updateNFSClientRPCStats(ch, &stats.ClientRPC) + c.updateNFSRequestsv2Stats(ch, &stats.V2Stats) + c.updateNFSRequestsv3Stats(ch, &stats.V3Stats) + c.updateNFSRequestsv4Stats(ch, &stats.ClientV4Stats) - value, _ = strconv.ParseFloat(fields[2], 64) - ch <- prometheus.MustNewConstMetric( - nfsNetReadsDesc, prometheus.CounterValue, - value, "tcp") - - value, _ = strconv.ParseFloat(fields[3], 64) - ch <- prometheus.MustNewConstMetric( - nfsNetConnectionsDesc, prometheus.CounterValue, - value, "tcp") - } else if fields := rpcLineRE.FindStringSubmatch(line); fields != nil { - value, _ := strconv.ParseFloat(fields[1], 64) - ch <- prometheus.MustNewConstMetric( - nfsRPCOperationsDesc, - prometheus.CounterValue, value) - - value, _ = strconv.ParseFloat(fields[2], 64) - ch <- prometheus.MustNewConstMetric( - nfsRPCRetransmissionsDesc, - prometheus.CounterValue, value) - - value, _ = strconv.ParseFloat(fields[3], 64) - ch <- prometheus.MustNewConstMetric( - nfsRPCAuthenticationRefreshesDesc, - prometheus.CounterValue, value) - } else if fields := procLineRE.FindStringSubmatch(line); fields != nil { - version := fields[1] - for procedure, count := range strings.Split(fields[2], " ") { - value, _ := strconv.ParseFloat(count, 64) - ch <- prometheus.MustNewConstMetric( - nfsProceduresDesc, - prometheus.CounterValue, - value, - version, - nfsProcedures[version][procedure]) - } - } else if line != "" { - return errors.New("Failed to parse line: " + line) - } - } return nil } + +// updateNFSNetworkStats collects statistics for network packets/connections. +func (c *nfsCollector) updateNFSNetworkStats(ch chan<- prometheus.Metric, s *nfs.Network) { + ch <- prometheus.MustNewConstMetric(c.nfsNetReadsDesc, prometheus.CounterValue, + float64(s.UDPCount), "udp") + ch <- prometheus.MustNewConstMetric(c.nfsNetReadsDesc, prometheus.CounterValue, + float64(s.TCPCount), "tcp") + ch <- prometheus.MustNewConstMetric(c.nfsNetConnectionsDesc, prometheus.CounterValue, + float64(s.TCPConnect), "tcp") +} + +// updateNFSClientRPCStats collects statistics for kernel server RPCs. +func (c *nfsCollector) updateNFSClientRPCStats(ch chan<- prometheus.Metric, s *nfs.ClientRPC) { + ch <- prometheus.MustNewConstMetric(c.nfsRPCOperationsDesc, prometheus.CounterValue, + float64(s.RPCCount)) + ch <- prometheus.MustNewConstMetric(c.nfsRPCRetransmissionsDesc, prometheus.CounterValue, + float64(s.Retransmissions)) + ch <- prometheus.MustNewConstMetric(c.nfsRPCAuthenticationRefreshesDesc, prometheus.CounterValue, + float64(s.AuthRefreshes)) +} + +// updateNFSRequestsv2Stats collects statistics for NFSv2 requests. +func (c *nfsCollector) updateNFSRequestsv2Stats(ch chan<- prometheus.Metric, s *nfs.V2Stats) { + const proto = "2" + + v := reflect.ValueOf(s).Elem() + for i := 0; i < v.NumField(); i++ { + field := v.Field(i) + name := strings.ToLower(v.Type().Field(i).Name) + + switch name { + case "wrcache": + name = "writecache" + case "fsstat": + name = "statfs" + } + + ch <- prometheus.MustNewConstMetric(c.nfsProceduresDesc, prometheus.CounterValue, + float64(field.Uint()), proto, name) + } +} + +// updateNFSRequestsv3Stats collects statistics for NFSv3 requests. +func (c *nfsCollector) updateNFSRequestsv3Stats(ch chan<- prometheus.Metric, s *nfs.V3Stats) { + const proto = "3" + + v := reflect.ValueOf(s).Elem() + for i := 0; i < v.NumField(); i++ { + field := v.Field(i) + name := strings.ToLower(v.Type().Field(i).Name) + + ch <- prometheus.MustNewConstMetric(c.nfsProceduresDesc, prometheus.CounterValue, + float64(field.Uint()), proto, name) + } +} + +// updateNFSRequestsv4Stats collects statistics for NFSv4 requests. +func (c *nfsCollector) updateNFSRequestsv4Stats(ch chan<- prometheus.Metric, s *nfs.ClientV4Stats) { + const proto = "4" + + v := reflect.ValueOf(s).Elem() + for i := 0; i < v.NumField(); i++ { + field := v.Field(i) + name := strings.ToLower(v.Type().Field(i).Name) + + switch name { + case "openconfirm": + name = "open_confirm" + case "opendowngrade": + name = "open_downgrade" + case "opennoattr": + name = "open_noattr" + case "setclientidconfirm": + name = "setclientid_confirm" + case "lookuproot": + name = "lookup_root" + case "servercaps": + name = "server_caps" + case "fslocations": + name = "fs_locations" + case "releaselockowner": + name = "release_lockowner" + case "fsidpresent": + name = "fsid_present" + case "exchangeid": + name = "exchange_id" + case "createsession": + name = "create_session" + case "destroysession": + name = "destroy_session" + case "getleasetime": + name = "get_lease_time" + case "reclaimcomplete": + name = "reclaim_complete" + // TODO: Enable these metrics + case "secinfononame", "teststateid", "freestateid", "getdevicelist", "bindconntosession", "destroyclientid", "seek", "allocate", "deallocate", "layoutstats", "clone": + continue + } + + ch <- prometheus.MustNewConstMetric(c.nfsProceduresDesc, prometheus.CounterValue, + float64(field.Uint()), proto, name) + } +} diff --git a/vendor/github.com/prometheus/procfs/fs.go b/vendor/github.com/prometheus/procfs/fs.go index 36c1586a..65f09223 100644 --- a/vendor/github.com/prometheus/procfs/fs.go +++ b/vendor/github.com/prometheus/procfs/fs.go @@ -46,8 +46,8 @@ func (fs FS) XFSStats() (*xfs.Stats, error) { return xfs.ParseStats(f) } -// NFSdClientRPCStats retrieves NFS daemon RPC statistics. -func (fs FS) NFSdClientRPCStats() (*nfs.ClientRPCStats, error) { +// NFSClientRPCStats retrieves NFS client RPC statistics. +func (fs FS) NFSClientRPCStats() (*nfs.ClientRPCStats, error) { f, err := os.Open(fs.Path("net/rpc/nfs")) if err != nil { return nil, err diff --git a/vendor/github.com/prometheus/procfs/nfs/parse.go b/vendor/github.com/prometheus/procfs/nfs/parse.go index 3aa32563..8f568f01 100644 --- a/vendor/github.com/prometheus/procfs/nfs/parse.go +++ b/vendor/github.com/prometheus/procfs/nfs/parse.go @@ -178,8 +178,17 @@ func parseV3Stats(v []uint64) (V3Stats, error) { func parseClientV4Stats(v []uint64) (ClientV4Stats, error) { values := int(v[0]) - if len(v[1:]) != values || values < 59 { - return ClientV4Stats{}, fmt.Errorf("invalid V4Stats line %q", v) + if len(v[1:]) != values { + return ClientV4Stats{}, fmt.Errorf("invalid ClientV4Stats line %q", v) + } + + // This function currently supports mapping 59 NFS v4 client stats. Older + // kernels may emit fewer stats, so we must detect this and pad out the + // values to match the expected slice size. + if values < 59 { + newValues := make([]uint64, 60) + copy(newValues, v) + v = newValues } return ClientV4Stats{ diff --git a/vendor/github.com/prometheus/procfs/nfs/parse_nfs.go b/vendor/github.com/prometheus/procfs/nfs/parse_nfs.go index b5c0b15f..c0d3a5ad 100644 --- a/vendor/github.com/prometheus/procfs/nfs/parse_nfs.go +++ b/vendor/github.com/prometheus/procfs/nfs/parse_nfs.go @@ -32,12 +32,12 @@ func ParseClientRPCStats(r io.Reader) (*ClientRPCStats, error) { parts := strings.Fields(scanner.Text()) // require at least if len(parts) < 2 { - return nil, fmt.Errorf("invalid NFSd metric line %q", line) + return nil, fmt.Errorf("invalid NFS metric line %q", line) } values, err := util.ParseUint64s(parts[1:]) if err != nil { - return nil, fmt.Errorf("error parsing NFSd metric line: %s", err) + return nil, fmt.Errorf("error parsing NFS metric line: %s", err) } switch metricLine := parts[0]; metricLine { @@ -52,15 +52,15 @@ func ParseClientRPCStats(r io.Reader) (*ClientRPCStats, error) { case "proc4": stats.ClientV4Stats, err = parseClientV4Stats(values) default: - return nil, fmt.Errorf("unknown NFSd metric line %q", metricLine) + return nil, fmt.Errorf("unknown NFS metric line %q", metricLine) } if err != nil { - return nil, fmt.Errorf("errors parsing NFSd metric line: %s", err) + return nil, fmt.Errorf("errors parsing NFS metric line: %s", err) } } if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("error scanning NFSd file: %s", err) + return nil, fmt.Errorf("error scanning NFS file: %s", err) } return stats, nil diff --git a/vendor/vendor.json b/vendor/vendor.json index 20391c0b..4b437061 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -161,40 +161,40 @@ "revisionTime": "2018-01-10T21:49:58Z" }, { - "checksumSHA1": "lolK0h7LSVERIX8zLyVQ/+7wEyA=", + "checksumSHA1": "Qvc01kv3ttHV2P2+J68g9ioU4Qs=", "path": "github.com/prometheus/procfs", - "revision": "cb4147076ac75738c9a7d279075a253c0cc5acbd", - "revisionTime": "2018-01-25T13:30:57Z" + "revision": "282c8707aa210456a825798969cc27edda34992a", + "revisionTime": "2018-02-12T14:59:26Z" }, { "checksumSHA1": "O64FotgWPYIpl3m2gvTEPIem+xg=", "path": "github.com/prometheus/procfs/bcache", - "revision": "cb4147076ac75738c9a7d279075a253c0cc5acbd", - "revisionTime": "2018-01-25T13:30:57Z" + "revision": "282c8707aa210456a825798969cc27edda34992a", + "revisionTime": "2018-02-12T14:59:26Z" }, { "checksumSHA1": "lv9rIcjbVEGo8AT1UCUZXhXrfQc=", "path": "github.com/prometheus/procfs/internal/util", - "revision": "cb4147076ac75738c9a7d279075a253c0cc5acbd", - "revisionTime": "2018-01-25T13:30:57Z" + "revision": "282c8707aa210456a825798969cc27edda34992a", + "revisionTime": "2018-02-12T14:59:26Z" }, { - "checksumSHA1": "BXJH5h2ri8SU5qC6kkDvTIGCky4=", + "checksumSHA1": "EekY1iRG9JY74mDD0jsbFCWbAFs=", "path": "github.com/prometheus/procfs/nfs", - "revision": "cb4147076ac75738c9a7d279075a253c0cc5acbd", - "revisionTime": "2018-01-25T13:30:57Z" + "revision": "282c8707aa210456a825798969cc27edda34992a", + "revisionTime": "2018-02-12T14:59:26Z" }, { "checksumSHA1": "wMhQkA/xQw3Q8eI+PIAjFmS94Qo=", "path": "github.com/prometheus/procfs/sysfs", - "revision": "cb4147076ac75738c9a7d279075a253c0cc5acbd", - "revisionTime": "2018-01-25T13:30:57Z" + "revision": "282c8707aa210456a825798969cc27edda34992a", + "revisionTime": "2018-02-12T14:59:26Z" }, { "checksumSHA1": "yItvTQLUVqm/ArLEbvEhqG0T5a0=", "path": "github.com/prometheus/procfs/xfs", - "revision": "cb4147076ac75738c9a7d279075a253c0cc5acbd", - "revisionTime": "2018-01-25T13:30:57Z" + "revision": "282c8707aa210456a825798969cc27edda34992a", + "revisionTime": "2018-02-12T14:59:26Z" }, { "checksumSHA1": "ySaT8G3I3y4MmnoXOYAAX0rC+p8=",