ZFS Collector: Add zpool IO statistics

Signed-Off-By: Joe Handzik <joseph.t.handzik@hpe.com>

collector/fixtures/e2e-output.txt

@@ -2494,6 +2494,54 @@ node_zfs_zil_zil_itx_needcopy_bytes 0
 # HELP node_zfs_zil_zil_itx_needcopy_count kstat.zfs.misc.zil.zil_itx_needcopy_count
 # TYPE node_zfs_zil_zil_itx_needcopy_count untyped
 node_zfs_zil_zil_itx_needcopy_count 0
+# HELP node_zfs_zpool_nread kstat.zfs.misc.io.nread
+# TYPE node_zfs_zpool_nread untyped
+node_zfs_zpool_nread{zpool="pool1"} 1.88416e+06
+node_zfs_zpool_nread{zpool="poolz1"} 2.82624e+06
+# HELP node_zfs_zpool_nwritten kstat.zfs.misc.io.nwritten
+# TYPE node_zfs_zpool_nwritten untyped
+node_zfs_zpool_nwritten{zpool="pool1"} 3.206144e+06
+node_zfs_zpool_nwritten{zpool="poolz1"} 2.680501248e+09
+# HELP node_zfs_zpool_rcnt kstat.zfs.misc.io.rcnt
+# TYPE node_zfs_zpool_rcnt untyped
+node_zfs_zpool_rcnt{zpool="pool1"} 0
+node_zfs_zpool_rcnt{zpool="poolz1"} 0
+# HELP node_zfs_zpool_reads kstat.zfs.misc.io.reads
+# TYPE node_zfs_zpool_reads untyped
+node_zfs_zpool_reads{zpool="pool1"} 22
+node_zfs_zpool_reads{zpool="poolz1"} 33
+# HELP node_zfs_zpool_rlentime kstat.zfs.misc.io.rlentime
+# TYPE node_zfs_zpool_rlentime untyped
+node_zfs_zpool_rlentime{zpool="pool1"} 1.04112268e+08
+node_zfs_zpool_rlentime{zpool="poolz1"} 6.472105124093e+12
+# HELP node_zfs_zpool_rtime kstat.zfs.misc.io.rtime
+# TYPE node_zfs_zpool_rtime untyped
+node_zfs_zpool_rtime{zpool="pool1"} 2.4168078e+07
+node_zfs_zpool_rtime{zpool="poolz1"} 9.82909164e+09
+# HELP node_zfs_zpool_rupdate kstat.zfs.misc.io.rupdate
+# TYPE node_zfs_zpool_rupdate untyped
+node_zfs_zpool_rupdate{zpool="pool1"} 7.921048984922e+13
+node_zfs_zpool_rupdate{zpool="poolz1"} 1.10734831944501e+14
+# HELP node_zfs_zpool_wcnt kstat.zfs.misc.io.wcnt
+# TYPE node_zfs_zpool_wcnt untyped
+node_zfs_zpool_wcnt{zpool="pool1"} 0
+node_zfs_zpool_wcnt{zpool="poolz1"} 0
+# HELP node_zfs_zpool_wlentime kstat.zfs.misc.io.wlentime
+# TYPE node_zfs_zpool_wlentime untyped
+node_zfs_zpool_wlentime{zpool="pool1"} 1.04112268e+08
+node_zfs_zpool_wlentime{zpool="poolz1"} 6.472105124093e+12
+# HELP node_zfs_zpool_writes kstat.zfs.misc.io.writes
+# TYPE node_zfs_zpool_writes untyped
+node_zfs_zpool_writes{zpool="pool1"} 132
+node_zfs_zpool_writes{zpool="poolz1"} 25294
+# HELP node_zfs_zpool_wtime kstat.zfs.misc.io.wtime
+# TYPE node_zfs_zpool_wtime untyped
+node_zfs_zpool_wtime{zpool="pool1"} 7.155162e+06
+node_zfs_zpool_wtime{zpool="poolz1"} 9.673715628e+09
+# HELP node_zfs_zpool_wupdate kstat.zfs.misc.io.wupdate
+# TYPE node_zfs_zpool_wupdate untyped
+node_zfs_zpool_wupdate{zpool="pool1"} 7.9210489694949e+13
+node_zfs_zpool_wupdate{zpool="poolz1"} 1.10734831833266e+14
 # HELP process_cpu_seconds_total Total user and system CPU time spent in seconds.
 # TYPE process_cpu_seconds_total counter
 # HELP process_max_fds Maximum number of open file descriptors.

collector/fixtures/proc/spl/kstat/zfs/pool1/io

@@ -0,0 +1,3 @@
+12 3 0x00 1 80 79205351707403 395818011156865
+nread    nwritten reads    writes   wtime    wlentime wupdate  rtime    rlentime rupdate  wcnt     rcnt    
+1884160  3206144  22       132      7155162  104112268 79210489694949 24168078 104112268 79210489849220 0        0       

collector/fixtures/proc/spl/kstat/zfs/poolz1/io

@@ -0,0 +1,3 @@
+16 3 0x00 1 80 79568650431241 395832279341621
+nread    nwritten reads    writes   wtime    wlentime wupdate  rtime    rlentime rupdate  wcnt     rcnt    
+2826240  2680501248 33       25294    9673715628 6472105124093 110734831833266 9829091640 6472105124093 110734831944501 0        0       

collector/zfs.go

@@ -45,12 +45,14 @@ func init() {
 type zfsCollector struct {
 	zfsMetrics        []zfsMetric
 	linuxProcpathBase string
+	linuxZpoolIoPath  string
 	linuxPathMap      map[string]string
 }
 
 func NewZFSCollector() (Collector, error) {
 	var z zfsCollector
 	z.linuxProcpathBase = "spl/kstat/zfs"
+	z.linuxZpoolIoPath = "/*/io"
 	z.linuxPathMap = map[string]string{
 		"zfs_arc":        "arcstats",
 		"zfs_dmu_tx":     "dmu_tx",
@@ -98,3 +100,19 @@ func (c *zfsCollector) constSysctlMetric(subsystem string, sysctl zfsSysctl, val
 		float64(value),
 	)
 }
+
+func (c *zfsCollector) constPoolMetric(poolName string, sysctl zfsSysctl, value int) prometheus.Metric {
+	metricName := sysctl.metricName()
+
+	return prometheus.MustNewConstMetric(
+		prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, "zfs_zpool", metricName),
+			string(sysctl),
+			[]string{"zpool"},
+			nil,
+		),
+		prometheus.UntypedValue,
+		float64(value),
+		poolName,
+	)
+}

collector/zfs_linux.go

@@ -47,6 +47,35 @@ func (c *zfsCollector) updateZfsStats(subsystem string, ch chan<- prometheus.Met
 	})
 }
 
+func (c *zfsCollector) updatePoolStats(ch chan<- prometheus.Metric) (err error) {
+	zpoolPaths, err := filepath.Glob(procFilePath(filepath.Join(c.linuxProcpathBase, c.linuxZpoolIoPath)))
+	if err != nil {
+		return err
+	}
+
+	if zpoolPaths == nil {
+		return nil
+	}
+
+	for _, zpoolPath := range zpoolPaths {
+		file, err := os.Open(zpoolPath)
+		if err != nil {
+			log.Debugf("Cannot open %q for reading. Is the kernel module loaded?", zpoolPath)
+			return zfsNotAvailableError
+		}
+
+		err = c.parsePoolProcfsFile(file, zpoolPath, func(poolName string, s zfsSysctl, v int) {
+			ch <- c.constPoolMetric(poolName, s, v)
+		})
+		file.Close()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func (c *zfsCollector) parseProcfsFile(reader io.Reader, fmtExt string, handler func(zfsSysctl, int)) (err error) {
 	scanner := bufio.NewScanner(reader)
 
@@ -81,6 +110,44 @@ func (c *zfsCollector) parseProcfsFile(reader io.Reader, fmtExt string, handler
 	return scanner.Err()
 }
 
-func (c *zfsCollector) updatePoolStats(ch chan<- prometheus.Metric) (err error) {
+func (c *zfsCollector) parsePoolProcfsFile(reader io.Reader, zpoolPath string, handler func(string, zfsSysctl, int)) (err error) {
-	return nil
+	scanner := bufio.NewScanner(reader)
+
+	parseLine := false
+	var fields []string
+	for scanner.Scan() {
+
+		line := strings.Fields(scanner.Text())
+
+		if !parseLine && len(line) >= 12 && line[0] == "nread" {
+			//Start parsing from here.
+			parseLine = true
+			fields = make([]string, len(line))
+			copy(fields, line)
+			continue
+		}
+		if !parseLine {
+			continue
+		}
+
+		zpoolPathElements := strings.Split(zpoolPath, "/")
+		pathLen := len(zpoolPathElements)
+		if pathLen < 2 {
+			return fmt.Errorf("zpool path did not return at least two elements")
+		}
+		zpoolName := zpoolPathElements[pathLen-2]
+		zpoolFile := zpoolPathElements[pathLen-1]
+
+		for i, field := range fields {
+			key := fmt.Sprintf("kstat.zfs.misc.%s.%s", zpoolFile, field)
+
+			value, err := strconv.Atoi(line[i])
+			if err != nil {
+				return fmt.Errorf("could not parse expected integer value for %q: %v", key, err)
+			}
+			handler(zpoolName, zfsSysctl(key), value)
+		}
+	}
+
+	return scanner.Err()
 }

collector/zfs_linux_test.go

@@ -15,6 +15,7 @@ package collector
 
 import (
 	"os"
+	"path/filepath"
 	"testing"
 )
 
@@ -269,3 +270,43 @@ func TestDmuTxParsing(t *testing.T) {
 		t.Fatal("DmuTx parsing handler was not called for some expected sysctls")
 	}
 }
+
+func TestZpoolParsing(t *testing.T) {
+	zpoolPaths, err := filepath.Glob("fixtures/proc/spl/kstat/zfs/*/io")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	c := zfsCollector{}
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	handlerCalled := false
+	for _, zpoolPath := range zpoolPaths {
+		file, err := os.Open(zpoolPath)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = c.parsePoolProcfsFile(file, zpoolPath, func(poolName string, s zfsSysctl, v int) {
+			if s != zfsSysctl("kstat.zfs.misc.io.nread") {
+				return
+			}
+
+			handlerCalled = true
+
+			if v != int(1884160) && v != int(2826240) {
+				t.Fatalf("Incorrect value parsed from procfs data %v", v)
+			}
+
+		})
+		file.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	if !handlerCalled {
+		t.Fatal("Zpool parsing handler was not called for some expected sysctls")
+	}
+}