read contents of objset file (#1632)

* added objread functionality

Signed-off-by: Sudharshann D <sudhar287@gmail.com>

collector/fixtures/e2e-64k-page-output.txt

@@ -3584,6 +3584,42 @@ node_zfs_zil_zil_itx_needcopy_bytes 1.8446744073709537e+19
 # HELP node_zfs_zil_zil_itx_needcopy_count kstat.zfs.misc.zil.zil_itx_needcopy_count
 # TYPE node_zfs_zil_zil_itx_needcopy_count untyped
 node_zfs_zil_zil_itx_needcopy_count 0
+# HELP node_zfs_zpool_dataset_nread kstat.zfs.misc.objset.nread
+# TYPE node_zfs_zpool_dataset_nread untyped
+node_zfs_zpool_dataset_nread{dataset="pool1",zpool="pool1"} 0
+node_zfs_zpool_dataset_nread{dataset="pool1/dataset1",zpool="pool1"} 28
+node_zfs_zpool_dataset_nread{dataset="poolz1",zpool="poolz1"} 0
+node_zfs_zpool_dataset_nread{dataset="poolz1/dataset1",zpool="poolz1"} 28
+# HELP node_zfs_zpool_dataset_nunlinked kstat.zfs.misc.objset.nunlinked
+# TYPE node_zfs_zpool_dataset_nunlinked untyped
+node_zfs_zpool_dataset_nunlinked{dataset="pool1",zpool="pool1"} 0
+node_zfs_zpool_dataset_nunlinked{dataset="pool1/dataset1",zpool="pool1"} 3
+node_zfs_zpool_dataset_nunlinked{dataset="poolz1",zpool="poolz1"} 0
+node_zfs_zpool_dataset_nunlinked{dataset="poolz1/dataset1",zpool="poolz1"} 14
+# HELP node_zfs_zpool_dataset_nunlinks kstat.zfs.misc.objset.nunlinks
+# TYPE node_zfs_zpool_dataset_nunlinks untyped
+node_zfs_zpool_dataset_nunlinks{dataset="pool1",zpool="pool1"} 0
+node_zfs_zpool_dataset_nunlinks{dataset="pool1/dataset1",zpool="pool1"} 3
+node_zfs_zpool_dataset_nunlinks{dataset="poolz1",zpool="poolz1"} 0
+node_zfs_zpool_dataset_nunlinks{dataset="poolz1/dataset1",zpool="poolz1"} 14
+# HELP node_zfs_zpool_dataset_nwritten kstat.zfs.misc.objset.nwritten
+# TYPE node_zfs_zpool_dataset_nwritten untyped
+node_zfs_zpool_dataset_nwritten{dataset="pool1",zpool="pool1"} 0
+node_zfs_zpool_dataset_nwritten{dataset="pool1/dataset1",zpool="pool1"} 12302
+node_zfs_zpool_dataset_nwritten{dataset="poolz1",zpool="poolz1"} 0
+node_zfs_zpool_dataset_nwritten{dataset="poolz1/dataset1",zpool="poolz1"} 32806
+# HELP node_zfs_zpool_dataset_reads kstat.zfs.misc.objset.reads
+# TYPE node_zfs_zpool_dataset_reads untyped
+node_zfs_zpool_dataset_reads{dataset="pool1",zpool="pool1"} 0
+node_zfs_zpool_dataset_reads{dataset="pool1/dataset1",zpool="pool1"} 2
+node_zfs_zpool_dataset_reads{dataset="poolz1",zpool="poolz1"} 0
+node_zfs_zpool_dataset_reads{dataset="poolz1/dataset1",zpool="poolz1"} 2
+# HELP node_zfs_zpool_dataset_writes kstat.zfs.misc.objset.writes
+# TYPE node_zfs_zpool_dataset_writes untyped
+node_zfs_zpool_dataset_writes{dataset="pool1",zpool="pool1"} 0
+node_zfs_zpool_dataset_writes{dataset="pool1/dataset1",zpool="pool1"} 4
+node_zfs_zpool_dataset_writes{dataset="poolz1",zpool="poolz1"} 0
+node_zfs_zpool_dataset_writes{dataset="poolz1/dataset1",zpool="poolz1"} 10
 # HELP node_zfs_zpool_nread kstat.zfs.misc.io.nread
 # TYPE node_zfs_zpool_nread untyped
 node_zfs_zpool_nread{zpool="pool1"} 1.88416e+06

collector/fixtures/e2e-output.txt

@@ -3677,6 +3677,42 @@ node_zfs_zil_zil_itx_needcopy_bytes 1.8446744073709537e+19
 # HELP node_zfs_zil_zil_itx_needcopy_count kstat.zfs.misc.zil.zil_itx_needcopy_count
 # TYPE node_zfs_zil_zil_itx_needcopy_count untyped
 node_zfs_zil_zil_itx_needcopy_count 0
+# HELP node_zfs_zpool_dataset_nread kstat.zfs.misc.objset.nread
+# TYPE node_zfs_zpool_dataset_nread untyped
+node_zfs_zpool_dataset_nread{dataset="pool1",zpool="pool1"} 0
+node_zfs_zpool_dataset_nread{dataset="pool1/dataset1",zpool="pool1"} 28
+node_zfs_zpool_dataset_nread{dataset="poolz1",zpool="poolz1"} 0
+node_zfs_zpool_dataset_nread{dataset="poolz1/dataset1",zpool="poolz1"} 28
+# HELP node_zfs_zpool_dataset_nunlinked kstat.zfs.misc.objset.nunlinked
+# TYPE node_zfs_zpool_dataset_nunlinked untyped
+node_zfs_zpool_dataset_nunlinked{dataset="pool1",zpool="pool1"} 0
+node_zfs_zpool_dataset_nunlinked{dataset="pool1/dataset1",zpool="pool1"} 3
+node_zfs_zpool_dataset_nunlinked{dataset="poolz1",zpool="poolz1"} 0
+node_zfs_zpool_dataset_nunlinked{dataset="poolz1/dataset1",zpool="poolz1"} 14
+# HELP node_zfs_zpool_dataset_nunlinks kstat.zfs.misc.objset.nunlinks
+# TYPE node_zfs_zpool_dataset_nunlinks untyped
+node_zfs_zpool_dataset_nunlinks{dataset="pool1",zpool="pool1"} 0
+node_zfs_zpool_dataset_nunlinks{dataset="pool1/dataset1",zpool="pool1"} 3
+node_zfs_zpool_dataset_nunlinks{dataset="poolz1",zpool="poolz1"} 0
+node_zfs_zpool_dataset_nunlinks{dataset="poolz1/dataset1",zpool="poolz1"} 14
+# HELP node_zfs_zpool_dataset_nwritten kstat.zfs.misc.objset.nwritten
+# TYPE node_zfs_zpool_dataset_nwritten untyped
+node_zfs_zpool_dataset_nwritten{dataset="pool1",zpool="pool1"} 0
+node_zfs_zpool_dataset_nwritten{dataset="pool1/dataset1",zpool="pool1"} 12302
+node_zfs_zpool_dataset_nwritten{dataset="poolz1",zpool="poolz1"} 0
+node_zfs_zpool_dataset_nwritten{dataset="poolz1/dataset1",zpool="poolz1"} 32806
+# HELP node_zfs_zpool_dataset_reads kstat.zfs.misc.objset.reads
+# TYPE node_zfs_zpool_dataset_reads untyped
+node_zfs_zpool_dataset_reads{dataset="pool1",zpool="pool1"} 0
+node_zfs_zpool_dataset_reads{dataset="pool1/dataset1",zpool="pool1"} 2
+node_zfs_zpool_dataset_reads{dataset="poolz1",zpool="poolz1"} 0
+node_zfs_zpool_dataset_reads{dataset="poolz1/dataset1",zpool="poolz1"} 2
+# HELP node_zfs_zpool_dataset_writes kstat.zfs.misc.objset.writes
+# TYPE node_zfs_zpool_dataset_writes untyped
+node_zfs_zpool_dataset_writes{dataset="pool1",zpool="pool1"} 0
+node_zfs_zpool_dataset_writes{dataset="pool1/dataset1",zpool="pool1"} 4
+node_zfs_zpool_dataset_writes{dataset="poolz1",zpool="poolz1"} 0
+node_zfs_zpool_dataset_writes{dataset="poolz1/dataset1",zpool="poolz1"} 10
 # HELP node_zfs_zpool_nread kstat.zfs.misc.io.nread
 # TYPE node_zfs_zpool_nread untyped
 node_zfs_zpool_nread{zpool="pool1"} 1.88416e+06

collector/fixtures/proc/spl/kstat/zfs/pool1/objset-1

@@ -0,0 +1,9 @@
+23 1 0x01 7 2160 221578688875 6665999035587
+name                            type data
+dataset_name                    7    pool1
+writes                          4    0
+nwritten                        4    0
+reads                           4    0
+nread                           4    0
+nunlinks                        4    0
+nunlinked                       4    0

collector/fixtures/proc/spl/kstat/zfs/pool1/objset-2

@@ -0,0 +1,9 @@
+24 1 0x01 7 2160 221611904716 7145015038451
+name                            type data
+dataset_name                    7    pool1/dataset1
+writes                          4    4
+nwritten                        4    12302
+reads                           4    2
+nread                           4    28
+nunlinks                        4    3
+nunlinked                       4    3

collector/fixtures/proc/spl/kstat/zfs/poolz1/objset-1

@@ -0,0 +1,9 @@
+30 1 0x01 7 2160 217993779684 2621674546179
+name                            type data
+dataset_name                    7    poolz1
+writes                          4    0
+nwritten                        4    0
+reads                           4    0
+nread                           4    0
+nunlinks                        4    0
+nunlinked                       4    0

collector/fixtures/proc/spl/kstat/zfs/poolz1/objset-2

@@ -0,0 +1,9 @@
+31 1 0x01 7 2160 218133979890 3024169078920
+name                            type data
+dataset_name                    7    poolz1/dataset1
+writes                          4    10
+nwritten                        4    32806
+reads                           4    2
+nread                           4    28
+nunlinks                        4    14
+nunlinked                       4    14

collector/zfs.go

@@ -34,17 +34,19 @@ func init() {
 }
 
 type zfsCollector struct {
-	linuxProcpathBase string
+	linuxProcpathBase    string
-	linuxZpoolIoPath  string
+	linuxZpoolIoPath     string
-	linuxPathMap      map[string]string
+	linuxZpoolObjsetPath string
-	logger            log.Logger
+	linuxPathMap         map[string]string
+	logger               log.Logger
 }
 
 // NewZFSCollector returns a new Collector exposing ZFS statistics.
 func NewZFSCollector(logger log.Logger) (Collector, error) {
 	return &zfsCollector{
-		linuxProcpathBase: "spl/kstat/zfs",
+		linuxProcpathBase:    "spl/kstat/zfs",
-		linuxZpoolIoPath:  "/*/io",
+		linuxZpoolIoPath:     "/*/io",
+		linuxZpoolObjsetPath: "/*/objset-*",
 		linuxPathMap: map[string]string{
 			"zfs_abd":         "abdstats",
 			"zfs_arc":         "arcstats",
@@ -113,3 +115,20 @@ func (c *zfsCollector) constPoolMetric(poolName string, sysctl zfsSysctl, value
 		poolName,
 	)
 }
+
+func (c *zfsCollector) constPoolObjsetMetric(poolName string, datasetName string, sysctl zfsSysctl, value uint64) prometheus.Metric {
+	metricName := sysctl.metricName()
+
+	return prometheus.MustNewConstMetric(
+		prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, "zfs_zpool_dataset", metricName),
+			string(sysctl),
+			[]string{"zpool", "dataset"},
+			nil,
+		),
+		prometheus.UntypedValue,
+		float64(value),
+		poolName,
+		datasetName,
+	)
+}

collector/zfs_linux.go

@@ -90,6 +90,31 @@ func (c *zfsCollector) updatePoolStats(ch chan<- prometheus.Metric) error {
 		}
 	}
 
+	zpoolObjsetPaths, err := filepath.Glob(procFilePath(filepath.Join(c.linuxProcpathBase, c.linuxZpoolObjsetPath)))
+	if err != nil {
+		return err
+	}
+
+	if zpoolObjsetPaths == nil {
+		return nil
+	}
+
+	for _, zpoolPath := range zpoolObjsetPaths {
+		file, err := os.Open(zpoolPath)
+		if err != nil {
+			// this file should exist, but there is a race where an exporting pool can remove the files -- ok to ignore
+			level.Debug(c.logger).Log("msg", "Cannot open file for reading", "path", zpoolPath)
+			return errZFSNotAvailable
+		}
+
+		err = c.parsePoolObjsetFile(file, zpoolPath, func(poolName string, datasetName string, s zfsSysctl, v uint64) {
+			ch <- c.constPoolObjsetMetric(poolName, datasetName, s, v)
+		})
+		file.Close()
+		if err != nil {
+			return err
+		}
+	}
 	return nil
 }
 
@@ -168,3 +193,43 @@ func (c *zfsCollector) parsePoolProcfsFile(reader io.Reader, zpoolPath string, h
 
 	return scanner.Err()
 }
+
+func (c *zfsCollector) parsePoolObjsetFile(reader io.Reader, zpoolPath string, handler func(string, string, zfsSysctl, uint64)) error {
+	scanner := bufio.NewScanner(reader)
+
+	parseLine := false
+	var zpoolName, datasetName string
+	for scanner.Scan() {
+		parts := strings.Fields(scanner.Text())
+
+		if !parseLine && len(parts) == 3 && parts[0] == "name" && parts[1] == "type" && parts[2] == "data" {
+			parseLine = true
+			continue
+		}
+
+		if !parseLine || len(parts) < 3 {
+			continue
+		}
+		if parts[0] == "dataset_name" {
+			zpoolPathElements := strings.Split(zpoolPath, "/")
+			pathLen := len(zpoolPathElements)
+			zpoolName = zpoolPathElements[pathLen-2]
+			datasetName = parts[2]
+			continue
+		}
+
+		if parts[1] == kstatDataUint64 {
+			key := fmt.Sprintf("kstat.zfs.misc.objset.%s", parts[0])
+			value, err := strconv.ParseUint(parts[2], 10, 64)
+			if err != nil {
+				return fmt.Errorf("could not parse expected integer value for %q", key)
+			}
+			handler(zpoolName, datasetName, zfsSysctl(key), value)
+		}
+	}
+	if !parseLine {
+		return fmt.Errorf("did not parse a single %s %s metric", zpoolName, datasetName)
+	}
+
+	return scanner.Err()
+}

collector/zfs_linux_test.go

@@ -311,6 +311,46 @@ func TestZpoolParsing(t *testing.T) {
 	}
 }
 
+func TestZpoolObjsetParsing(t *testing.T) {
+	zpoolPaths, err := filepath.Glob("fixtures/proc/spl/kstat/zfs/*/objset-*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	c := zfsCollector{}
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	handlerCalled := false
+	for _, zpoolPath := range zpoolPaths {
+		file, err := os.Open(zpoolPath)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = c.parsePoolObjsetFile(file, zpoolPath, func(poolName string, datasetName string, s zfsSysctl, v uint64) {
+			if s != zfsSysctl("kstat.zfs.misc.objset.writes") {
+				return
+			}
+
+			handlerCalled = true
+
+			if v != uint64(0) && v != uint64(4) && v != uint64(10) {
+				t.Fatalf("Incorrect value parsed from procfs data %v", v)
+			}
+
+		})
+		file.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	if !handlerCalled {
+		t.Fatal("Zpool parsing handler was not called for some expected sysctls")
+	}
+}
+
 func TestAbdstatsParsing(t *testing.T) {
 	abdstatsFile, err := os.Open("fixtures/proc/spl/kstat/zfs/abdstats")
 	if err != nil {