Add ZFS freebsd per dataset stats (#2753)

* Rename parsePoolObjsetFile to parseLinuxPoolObjsetFile to better reflect
it's scope
* Create a new parseFreeBSDPoolObjsetStats function, to generate a list
of per pool metrics to be queried via sysctl


---------

Signed-off-by: Conall O'Brien <conall@conall.net>

collector/exec_bsd.go

@@ -49,31 +49,37 @@ func NewExecCollector(logger log.Logger) (Collector, error) {
 				name:        "exec_context_switches_total",
 				description: "Context switches since system boot.  Resets at architecture unsigned integer.",
 				mib:         "vm.stats.sys.v_swtch",
+				labels:      nil,
 			},
 			{
 				name:        "exec_traps_total",
 				description: "Traps since system boot.  Resets at architecture unsigned integer.",
 				mib:         "vm.stats.sys.v_trap",
+				labels:      nil,
 			},
 			{
 				name:        "exec_system_calls_total",
 				description: "System calls since system boot.  Resets at architecture unsigned integer.",
 				mib:         "vm.stats.sys.v_syscall",
 			},
+			labels: nil,
 			{
 				name:        "exec_device_interrupts_total",
 				description: "Device interrupts since system boot.  Resets at architecture unsigned integer.",
 				mib:         "vm.stats.sys.v_intr",
+				labels:      nil,
 			},
 			{
 				name:        "exec_software_interrupts_total",
 				description: "Software interrupts since system boot.  Resets at architecture unsigned integer.",
 				mib:         "vm.stats.sys.v_soft",
+				labels:      nil,
 			},
 			{
 				name:        "exec_forks_total",
 				description: "Number of fork() calls since system boot.  Resets at architecture unsigned integer.",
 				mib:         "vm.stats.vm.v_forks",
+				labels:      nil,
 			},
 		},
 		logger: logger,

collector/memory_bsd.go

@@ -69,18 +69,21 @@ func NewMemoryCollector(logger log.Logger) (Collector, error) {
 				description: "Recently used by userland",
 				mib:         "vm.stats.vm.v_active_count",
 				conversion:  fromPage,
+				labels:      nil,
 			},
 			{
 				name:        "inactive_bytes",
 				description: "Not recently used by userland",
 				mib:         "vm.stats.vm.v_inactive_count",
 				conversion:  fromPage,
+				labels:      nil,
 			},
 			{
 				name:        "wired_bytes",
 				description: "Locked in memory by kernel, mlock, etc",
 				mib:         "vm.stats.vm.v_wire_count",
 				conversion:  fromPage,
+				labels:      nil,
 			},
 			{
 				name:        "user_wired_bytes",
@@ -88,42 +91,49 @@ func NewMemoryCollector(logger log.Logger) (Collector, error) {
 				mib:         "vm.stats.vm.v_user_wire_count",
 				conversion:  fromPage,
 				dataType:    bsdSysctlTypeCLong,
+				labels:      nil,
 			},
 			{
 				name:        "cache_bytes",
 				description: "Almost free, backed by swap or files, available for re-allocation",
 				mib:         "vm.stats.vm.v_cache_count",
 				conversion:  fromPage,
+				labels:      nil,
 			},
 			{
 				name:        "buffer_bytes",
 				description: "Disk IO Cache entries for non ZFS filesystems, only usable by kernel",
 				mib:         "vfs.bufspace",
 				dataType:    bsdSysctlTypeCLong,
+				labels:      nil,
 			},
 			{
 				name:        "free_bytes",
 				description: "Unallocated, available for allocation",
 				mib:         "vm.stats.vm.v_free_count",
 				conversion:  fromPage,
+				labels:      nil,
 			},
 			{
 				name:        "laundry_bytes",
 				description: "Dirty not recently used by userland",
 				mib:         "vm.stats.vm.v_laundry_count",
 				conversion:  fromPage,
+				labels:      nil,
 			},
 			{
 				name:        "size_bytes",
 				description: "Total physical memory size",
 				mib:         "vm.stats.vm.v_page_count",
 				conversion:  fromPage,
+				labels:      nil,
 			},
 			{
 				name:        "swap_size_bytes",
 				description: "Total swap memory size",
 				mib:         mibSwapTotal,
 				dataType:    bsdSysctlTypeUint64,
+				labels:      nil,
 			},
 			// Descriptions via: top(1)
 			{
@@ -132,6 +142,7 @@ func NewMemoryCollector(logger log.Logger) (Collector, error) {
 				mib:         "vm.stats.vm.v_swappgsin",
 				valueType:   prometheus.CounterValue,
 				conversion:  fromPage,
+				labels:      nil,
 			},
 			{
 				name:        "swap_out_bytes_total",
@@ -139,6 +150,7 @@ func NewMemoryCollector(logger log.Logger) (Collector, error) {
 				mib:         "vm.stats.vm.v_swappgsout",
 				valueType:   prometheus.CounterValue,
 				conversion:  fromPage,
+				labels:      nil,
 			},
 		},
 	}, nil

collector/netisr_freebsd.go

@@ -45,6 +45,7 @@ func NewNetisrCollector(logger log.Logger) (Collector, error) {
 				mib:         "net.isr.numthreads",
 				dataType:    bsdSysctlTypeUint32,
 				valueType:   prometheus.GaugeValue,
+				labels:      nil,
 			},
 			{
 				name:        "maxprot",
@@ -52,6 +53,7 @@ func NewNetisrCollector(logger log.Logger) (Collector, error) {
 				mib:         "net.isr.maxprot",
 				dataType:    bsdSysctlTypeUint32,
 				valueType:   prometheus.GaugeValue,
+				labels:      nil,
 			},
 			{
 				name:        "defaultqlimit",
@@ -59,6 +61,7 @@ func NewNetisrCollector(logger log.Logger) (Collector, error) {
 				mib:         "net.isr.defaultqlimit",
 				dataType:    bsdSysctlTypeUint32,
 				valueType:   prometheus.GaugeValue,
+				labels:      nil,
 			},
 			{
 				name:        "maxqlimit",
@@ -66,6 +69,7 @@ func NewNetisrCollector(logger log.Logger) (Collector, error) {
 				mib:         "net.isr.maxqlimit",
 				dataType:    bsdSysctlTypeUint32,
 				valueType:   prometheus.GaugeValue,
+				labels:      nil,
 			},
 			{
 				name:        "bindthreads",
@@ -73,6 +77,7 @@ func NewNetisrCollector(logger log.Logger) (Collector, error) {
 				mib:         "net.isr.bindthreads",
 				dataType:    bsdSysctlTypeUint32,
 				valueType:   prometheus.GaugeValue,
+				labels:      nil,
 			},
 			{
 				name:        "maxthreads",
@@ -80,6 +85,7 @@ func NewNetisrCollector(logger log.Logger) (Collector, error) {
 				mib:         "net.isr.maxthreads",
 				dataType:    bsdSysctlTypeUint32,
 				valueType:   prometheus.GaugeValue,
+				labels:      nil,
 			},
 		},
 		logger: logger,

collector/sysctl_bsd.go

@@ -59,6 +59,9 @@ type bsdSysctl struct {
 
 	// Post-retrieval conversion hooks
 	conversion func(float64) float64
+
+	// Prometheus labels
+	labels prometheus.Labels
 }
 
 func (b bsdSysctl) Value() (float64, error) {

collector/zfs_freebsd.go

@@ -21,6 +21,8 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/prometheus/client_golang/prometheus"
+
+	"golang.org/x/sys/unix"
 )
 
 type zfsCollector struct {
@@ -45,6 +47,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.abdstats.linear_cnt",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.CounterValue,
+				labels:      nil,
 			},
 			{
 				name:        "abdstats_linear_data_bytes",
@@ -52,6 +55,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.abdstats.linear_data_size",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
+				labels:      nil,
 			},
 			{
 				name:        "abdstats_scatter_chunk_waste_bytes",
@@ -59,6 +63,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.abdstats.scatter_chunk_waste",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
+				labels:      nil,
 			},
 			{
 				name:        "abdstats_scatter_count_total",
@@ -66,6 +71,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.abdstats.scatter_cnt",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.CounterValue,
+				labels:      nil,
 			},
 			{
 				name:        "abdstats_scatter_data_bytes",
@@ -73,6 +79,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.abdstats.scatter_data_size",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
+				labels:      nil,
 			},
 			{
 				name:        "abdstats_struct_bytes",
@@ -80,6 +87,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.abdstats.struct_size",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
+				labels:      nil,
 			},
 			{
 				name:        "arcstats_anon_bytes",
@@ -87,6 +95,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.anon_size",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
+				labels:      nil,
 			},
 			{
 				name:        "arcstats_c_bytes",
@@ -94,6 +103,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.c",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
+				labels:      nil,
 			},
 			{
 				name:        "arcstats_c_max_bytes",
@@ -101,6 +111,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.c_max",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
+				labels:      nil,
 			},
 			{
 				name:        "arcstats_c_min_bytes",
@@ -108,6 +119,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.c_min",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
+				labels:      nil,
 			},
 			{
 				name:        "arcstats_data_bytes",
@@ -115,6 +127,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.data_size",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
+				labels:      nil,
 			},
 			{
 				name:        "arcstats_demand_data_hits_total",
@@ -122,6 +135,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.demand_data_hits",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.CounterValue,
+				labels:      nil,
 			},
 			{
 				name:        "arcstats_demand_data_misses_total",
@@ -129,6 +143,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.demand_data_misses",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.CounterValue,
+				labels:      nil,
 			},
 			{
 				name:        "arcstats_demand_metadata_hits_total",
@@ -136,6 +151,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.demand_metadata_hits",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.CounterValue,
+				labels:      nil,
 			},
 			{
 				name:        "arcstats_demand_metadata_misses_total",
@@ -143,6 +159,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.demand_metadata_misses",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.CounterValue,
+				labels:      nil,
 			},
 			{
 				name:        "arcstats_hdr_bytes",
@@ -150,6 +167,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.hdr_size",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
+				labels:      nil,
 			},
 			{
 				name:        "arcstats_hits_total",
@@ -157,6 +175,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.hits",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.CounterValue,
+				labels:      nil,
 			},
 			{
 				name:        "arcstats_misses_total",
@@ -164,6 +183,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.misses",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.CounterValue,
+				labels:      nil,
 			},
 			{
 				name:        "arcstats_mfu_ghost_hits_total",
@@ -171,6 +191,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.mfu_ghost_hits",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.CounterValue,
+				labels:      nil,
 			},
 			{
 				name:        "arcstats_mfu_ghost_size",
@@ -178,6 +199,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.mfu_ghost_size",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
+				labels:      nil,
 			},
 			{
 				name:        "arcstats_mfu_bytes",
@@ -185,6 +207,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.mfu_size",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
+				labels:      nil,
 			},
 			{
 				name:        "arcstats_mru_ghost_hits_total",
@@ -192,6 +215,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.mru_ghost_hits",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.CounterValue,
+				labels:      nil,
 			},
 			{
 				name:        "arcstats_mru_ghost_bytes",
@@ -199,6 +223,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.mru_ghost_size",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
+				labels:      nil,
 			},
 			{
 				name:        "arcstats_mru_bytes",
@@ -206,6 +231,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.mru_size",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
+				labels:      nil,
 			},
 			{
 				name:        "arcstats_other_bytes",
@@ -213,6 +239,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.other_size",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
+				labels:      nil,
 			},
 			{
 				name:        "arcstats_p_bytes",
@@ -220,6 +247,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.p",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
+				labels:      nil,
 			},
 			{
 				name:        "arcstats_size_bytes",
@@ -227,6 +255,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.size",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
+				labels:      nil,
 			},
 			{
 				name:        "zfetchstats_hits_total",
@@ -234,6 +263,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.zfetchstats.hits",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.CounterValue,
+				labels:      nil,
 			},
 			{
 				name:        "zfetchstats_misses_total",
@@ -241,6 +271,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.zfetchstats.misses",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.CounterValue,
+				labels:      nil,
 			},
 		},
 		logger: logger,
@@ -264,3 +295,42 @@ func (c *zfsCollector) Update(ch chan<- prometheus.Metric) error {
 
 	return nil
 }
+
+func (c *zfsCollector) parseFreeBSDPoolObjsetStats() error {
+
+	sysCtlMetrics := []string{
+		"nunlinked", "nunlinks", "nread", "reads", "nwritten", "writes",
+	}
+	zfsPoolMibPrefix := "kstat.zfs.pool.dataset"
+	zfsDatasetsNames := []string{}
+
+	zfsDatasets, err := unix.Sysctl(zfsPoolMibPrefix)
+	if err != nil {
+		return fmt.Errorf("couldn't get sysctl: %w", err)
+	}
+
+	for dataset, _ := range zfsDatasets {
+		if strings.HasSuffix(dataset, ".dataset_name") {
+			zfsDatasetNames = append(zfsDatasetNames, strings.SplitAfter(dataset, ".")[3])
+		}
+	}
+
+	for zpoolDataset := range zfsDatasetsNames {
+		zfsDatasetLabels := map[string]string{
+			"dataset": zpoolDataset,
+			"zpool":   strings.SplitAfter(zpoolDataset, "/")[0],
+		}
+		for metric := range sysCtlMetrics {
+			c.sysctls = append(c.sysctls, bsdSysctl{
+				name:        fmt.SprintF("node_zfs_zpool_dataset_%s", metric),
+				description: fmt.SprintF("node_zfs_zpool_dataset_%s", metric),
+				mib:         fmt.Sprintf("%s.%s.%s", zfsPoolMibPrefix, poolObj, metric),
+				dataType:    bsdSysctlTypeUint64,
+				valueType:   prometheus.CounterValue,
+				labels:      zfsDatasetLabels,
+			})
+		}
+	}
+
+	return nil
+}

collector/zfs_linux.go

@@ -104,7 +104,7 @@ func (c *zfsCollector) updatePoolStats(ch chan<- prometheus.Metric) error {
 			return errZFSNotAvailable
 		}
 
-		err = c.parsePoolObjsetFile(file, zpoolPath, func(poolName string, datasetName string, s zfsSysctl, v uint64) {
+		err = c.parseLinuxPoolObjsetFile(file, zpoolPath, func(poolName string, datasetName string, s zfsSysctl, v uint64) {
 			ch <- c.constPoolObjsetMetric(poolName, datasetName, s, v)
 		})
 		file.Close()
@@ -220,7 +220,7 @@ func (c *zfsCollector) parsePoolProcfsFile(reader io.Reader, zpoolPath string, h
 	return scanner.Err()
 }
 
-func (c *zfsCollector) parsePoolObjsetFile(reader io.Reader, zpoolPath string, handler func(string, string, zfsSysctl, uint64)) error {
+func (c *zfsCollector) parseLinuxPoolObjsetFile(reader io.Reader, zpoolPath string, handler func(string, string, zfsSysctl, uint64)) error {
 	scanner := bufio.NewScanner(reader)
 
 	parseLine := false

collector/zfs_linux_test.go

@@ -332,7 +332,7 @@ func TestZpoolObjsetParsing(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		err = c.parsePoolObjsetFile(file, zpoolPath, func(poolName string, datasetName string, s zfsSysctl, v uint64) {
+		err = c.parseLinuxPoolObjsetFile(file, zpoolPath, func(poolName string, datasetName string, s zfsSysctl, v uint64) {
 			if s != zfsSysctl("kstat.zfs.misc.objset.writes") {
 				return
 			}