ZFS Collector: Add zfetchstats functionality

Signed-Off-By: Joe Handzik <joseph.t.handzik@hpe.com>

collector/fixtures/e2e-output.txt

@@ -2350,6 +2350,39 @@ node_zfsArc_prefetch_metadata_misses 16071
 # HELP node_zfsArc_size kstat.zfs.misc.arcstats.size
 # TYPE node_zfsArc_size untyped
 node_zfsArc_size 1.603939792e+09
+# HELP node_zfsFetch_bogus_streams kstat.zfs.misc.zfetchstats.bogus_streams
+# TYPE node_zfsFetch_bogus_streams untyped
+node_zfsFetch_bogus_streams 0
+# HELP node_zfsFetch_colinear_hits kstat.zfs.misc.zfetchstats.colinear_hits
+# TYPE node_zfsFetch_colinear_hits untyped
+node_zfsFetch_colinear_hits 0
+# HELP node_zfsFetch_colinear_misses kstat.zfs.misc.zfetchstats.colinear_misses
+# TYPE node_zfsFetch_colinear_misses untyped
+node_zfsFetch_colinear_misses 11
+# HELP node_zfsFetch_hits kstat.zfs.misc.zfetchstats.hits
+# TYPE node_zfsFetch_hits untyped
+node_zfsFetch_hits 7.067992e+06
+# HELP node_zfsFetch_misses kstat.zfs.misc.zfetchstats.misses
+# TYPE node_zfsFetch_misses untyped
+node_zfsFetch_misses 11
+# HELP node_zfsFetch_reclaim_failures kstat.zfs.misc.zfetchstats.reclaim_failures
+# TYPE node_zfsFetch_reclaim_failures untyped
+node_zfsFetch_reclaim_failures 11
+# HELP node_zfsFetch_reclaim_successes kstat.zfs.misc.zfetchstats.reclaim_successes
+# TYPE node_zfsFetch_reclaim_successes untyped
+node_zfsFetch_reclaim_successes 0
+# HELP node_zfsFetch_streams_noresets kstat.zfs.misc.zfetchstats.streams_noresets
+# TYPE node_zfsFetch_streams_noresets untyped
+node_zfsFetch_streams_noresets 2
+# HELP node_zfsFetch_streams_resets kstat.zfs.misc.zfetchstats.streams_resets
+# TYPE node_zfsFetch_streams_resets untyped
+node_zfsFetch_streams_resets 0
+# HELP node_zfsFetch_stride_hits kstat.zfs.misc.zfetchstats.stride_hits
+# TYPE node_zfsFetch_stride_hits untyped
+node_zfsFetch_stride_hits 7.06799e+06
+# HELP node_zfsFetch_stride_misses kstat.zfs.misc.zfetchstats.stride_misses
+# TYPE node_zfsFetch_stride_misses untyped
+node_zfsFetch_stride_misses 0
 # HELP process_cpu_seconds_total Total user and system CPU time spent in seconds.
 # TYPE process_cpu_seconds_total counter
 # HELP process_max_fds Maximum number of open file descriptors.

collector/fixtures/proc/spl/kstat/zfs/zfetchstats

@@ -0,0 +1,13 @@
+4 1 0x01 11 528 8010434610 345692669858836
+name                            type data
+hits                            4    7067992
+misses                          4    11
+colinear_hits                   4    0
+colinear_misses                 4    11
+stride_hits                     4    7067990
+stride_misses                   4    0
+reclaim_successes               4    0
+reclaim_failures                4    11
+streams_resets                  4    0
+streams_noresets                4    2
+bogus_streams                   4    0

collector/zfs.go

@@ -35,6 +35,7 @@ type zfsSubsystemName string
 
 const (
 	arc            = zfsSubsystemName("zfsArc")
+	zfetch         = zfsSubsystemName("zfsFetch")
 	zpoolSubsystem = zfsSubsystemName("zfsPool")
 )
 
@@ -71,6 +72,10 @@ func (c *zfsCollector) Update(ch chan<- prometheus.Metric) (err error) {
 		return err
 	}
 
+	// Zfetchstats
+	err = c.updateZfetchstats(ch)
+	if err != nil { return err }
+
 	// Pool stats
 	return c.updatePoolStats(ch)
 }

collector/zfs_linux.go

@@ -15,10 +15,10 @@ package collector
 
 import (
 	"bufio"
-	"errors"
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 
@@ -27,31 +27,45 @@ import (
 )
 
 const (
-	zfsArcstatsProcpath = "spl/kstat/zfs/arcstats"
+	zfsProcpathBase  = "spl/kstat/zfs/"
+	zfsArcstatsExt   = "arcstats"
+	zfsFetchstatsExt = "zfetchstats"
 )
 
-func (c *zfsCollector) openArcstatsFile() (file *os.File, err error) {
+func (c *zfsCollector) openProcFile(path string) (file *os.File, err error) {
-	file, err = os.Open(procFilePath(zfsArcstatsProcpath))
+	file, err = os.Open(procFilePath(path))
 	if err != nil {
-		log.Debugf("Cannot open %q for reading. Is the kernel module loaded?", procFilePath(zfsArcstatsProcpath))
+		log.Debugf("Cannot open %q for reading. Is the kernel module loaded?", procFilePath(path))
 		err = zfsNotAvailableError
 	}
 	return
 }
 
 func (c *zfsCollector) updateArcstats(ch chan<- prometheus.Metric) (err error) {
-	file, err := c.openArcstatsFile()
+	file, err := c.openProcFile(filepath.Join(zfsProcpathBase, zfsArcstatsExt))
 	if err != nil {
 		return err
 	}
 	defer file.Close()
 
-	return c.parseArcstatsProcfsFile(file, func(s zfsSysctl, v zfsMetricValue) {
+	return c.parseProcfsFile(file, zfsArcstatsExt, func(s zfsSysctl, v zfsMetricValue) {
 		ch <- c.constSysctlMetric(arc, s, v)
 	})
 }
 
-func (c *zfsCollector) parseArcstatsProcfsFile(reader io.Reader, handler func(zfsSysctl, zfsMetricValue)) (err error) {
+func (c *zfsCollector) updateZfetchstats(ch chan<- prometheus.Metric) (err error) {
+	file, err := c.openProcFile(filepath.Join(zfsProcpathBase, zfsFetchstatsExt))
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	return c.parseProcfsFile(file, zfsFetchstatsExt, func(s zfsSysctl, v zfsMetricValue) {
+		ch <- c.constSysctlMetric(zfetch, s, v)
+	})
+}
+
+func (c *zfsCollector) parseProcfsFile(reader io.Reader, fmt_ext string, handler func(zfsSysctl, zfsMetricValue)) (err error) {
 	scanner := bufio.NewScanner(reader)
 
 	parseLine := false
@@ -69,7 +83,7 @@ func (c *zfsCollector) parseArcstatsProcfsFile(reader io.Reader, handler func(zf
 			continue
 		}
 
-		key := fmt.Sprintf("kstat.zfs.misc.arcstats.%s", parts[0])
+		key := fmt.Sprintf("kstat.zfs.misc.%s.%s", fmt_ext, parts[0])
 
 		value, err := strconv.Atoi(parts[2])
 		if err != nil {
@@ -79,7 +93,7 @@ func (c *zfsCollector) parseArcstatsProcfsFile(reader io.Reader, handler func(zf
 
 	}
 	if !parseLine {
-		return errors.New("did not parse a single arcstat metric")
+		return fmt.Errorf("did not parse a single %q metric", fmt_ext)
 	}
 
 	return scanner.Err()

collector/zfs_linux_test.go

@@ -31,7 +31,7 @@ func TestArcstatsParsing(t *testing.T) {
 	}
 
 	handlerCalled := false
-	err = c.parseArcstatsProcfsFile(arcstatsFile, func(s zfsSysctl, v zfsMetricValue) {
+	err = c.parseProcfsFile(arcstatsFile, "arcstats", func(s zfsSysctl, v zfsMetricValue) {
 
 		if s != zfsSysctl("kstat.zfs.misc.arcstats.hits") {
 			return
@@ -53,3 +53,39 @@ func TestArcstatsParsing(t *testing.T) {
 		t.Fatal("Arcstats parsing handler was not called for some expected sysctls")
 	}
 }
+
+func TestZfetchstatsParsing(t *testing.T) {
+	zfetchstatsFile, err := os.Open("fixtures/proc/spl/kstat/zfs/zfetchstats")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer zfetchstatsFile.Close()
+
+	c := zfsCollector{}
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	handlerCalled := false
+	err = c.parseProcfsFile(zfetchstatsFile, "zfetchstats", func(s zfsSysctl, v zfsMetricValue) {
+
+		if s != zfsSysctl("kstat.zfs.misc.zfetchstats.hits") {
+			return
+		}
+
+		handlerCalled = true
+
+		if v != zfsMetricValue(7067992) {
+			t.Fatalf("Incorrect value parsed from procfs data")
+		}
+
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !handlerCalled {
+		t.Fatal("Zfetchstats parsing handler was not called for some expected sysctls")
+	}
+}