// Copyright 2016 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !nozfs // +build !nozfs package collector import ( "bufio" "errors" "fmt" "io" "log/slog" "os" "path/filepath" "strconv" "strings" "github.com/prometheus/client_golang/prometheus" ) // constants from https://github.com/zfsonlinux/zfs/blob/master/lib/libspl/include/sys/kstat.h // kept as strings for comparison thus avoiding conversion to int const ( // kstatDataChar = "0" // kstatDataInt32 = "1" // kstatDataUint32 = "2" kstatDataInt64 = "3" kstatDataUint64 = "4" // kstatDataLong = "5" // kstatDataUlong = "6" // kstatDataString = "7" ) var ( errZFSNotAvailable = errors.New("ZFS / ZFS statistics are not available") zfsPoolStatesName = [...]string{"online", "degraded", "faulted", "offline", "removed", "unavail", "suspended"} ) type zfsCollector struct { linuxProcpathBase string linuxZpoolIoPath string linuxZpoolObjsetPath string linuxZpoolStatePath string linuxPathMap map[string]string logger *slog.Logger } // NewZFSCollector returns a new Collector exposing ZFS statistics. func NewZFSCollector(logger *slog.Logger) (Collector, error) { return &zfsCollector{ linuxProcpathBase: "spl/kstat/zfs", linuxZpoolIoPath: "/*/io", linuxZpoolObjsetPath: "/*/objset-*", linuxZpoolStatePath: "/*/state", linuxPathMap: map[string]string{ "zfs_abd": "abdstats", "zfs_arc": "arcstats", "zfs_dbuf": "dbufstats", "zfs_dmu_tx": "dmu_tx", "zfs_dnode": "dnodestats", "zfs_fm": "fm", "zfs_vdev_cache": "vdev_cache_stats", // vdev_cache is deprecated "zfs_vdev_mirror": "vdev_mirror_stats", "zfs_xuio": "xuio_stats", // no known consumers of the XUIO interface on Linux exist "zfs_zfetch": "zfetchstats", "zfs_zil": "zil", }, logger: logger, }, nil } func (c *zfsCollector) Update(ch chan<- prometheus.Metric) error { if _, err := c.openProcFile(c.linuxProcpathBase); err != nil { if err == errZFSNotAvailable { c.logger.Debug(err.Error()) return ErrNoData } } for subsystem := range c.linuxPathMap { if err := c.updateZfsStats(subsystem, ch); err != nil { if err == errZFSNotAvailable { c.logger.Debug(err.Error()) // ZFS /proc files are added as new features to ZFS arrive, it is ok to continue continue } return err } } // Pool stats return c.updatePoolStats(ch) } func (c *zfsCollector) openProcFile(path string) (*os.File, error) { file, err := os.Open(procFilePath(path)) if err != nil { // file not found error can occur if: // 1. zfs module is not loaded // 2. zfs version does not have the feature with metrics -- ok to ignore c.logger.Debug("Cannot open file for reading", "path", procFilePath(path)) return nil, errZFSNotAvailable } return file, nil } func (c *zfsCollector) updateZfsStats(subsystem string, ch chan<- prometheus.Metric) error { file, err := c.openProcFile(filepath.Join(c.linuxProcpathBase, c.linuxPathMap[subsystem])) if err != nil { return err } defer file.Close() return c.parseProcfsFile(file, c.linuxPathMap[subsystem], func(s zfsSysctl, v interface{}) { var valueAsFloat64 float64 switch value := v.(type) { case int64: valueAsFloat64 = float64(value) case uint64: valueAsFloat64 = float64(value) } ch <- c.constSysctlMetric(subsystem, s, valueAsFloat64) }) } func (c *zfsCollector) updatePoolStats(ch chan<- prometheus.Metric) error { zpoolPaths, err := filepath.Glob(procFilePath(filepath.Join(c.linuxProcpathBase, c.linuxZpoolIoPath))) if err != nil { return err } for _, zpoolPath := range zpoolPaths { file, err := os.Open(zpoolPath) if err != nil { // this file should exist, but there is a race where an exporting pool can remove the files -- ok to ignore c.logger.Debug("Cannot open file for reading", "path", zpoolPath) return errZFSNotAvailable } err = c.parsePoolProcfsFile(file, zpoolPath, func(poolName string, s zfsSysctl, v uint64) { ch <- c.constPoolMetric(poolName, s, v) }) file.Close() if err != nil { return err } } zpoolObjsetPaths, err := filepath.Glob(procFilePath(filepath.Join(c.linuxProcpathBase, c.linuxZpoolObjsetPath))) if err != nil { return err } for _, zpoolPath := range zpoolObjsetPaths { file, err := os.Open(zpoolPath) if err != nil { // This file should exist, but there is a race where an exporting pool can remove the files. Ok to ignore. c.logger.Debug("Cannot open file for reading", "path", zpoolPath) return errZFSNotAvailable } err = c.parsePoolObjsetFile(file, zpoolPath, func(poolName string, datasetName string, s zfsSysctl, v uint64) { ch <- c.constPoolObjsetMetric(poolName, datasetName, s, v) }) file.Close() if err != nil { return err } } zpoolStatePaths, err := filepath.Glob(procFilePath(filepath.Join(c.linuxProcpathBase, c.linuxZpoolStatePath))) if err != nil { return err } if zpoolStatePaths == nil { c.logger.Debug("No pool state files found") return nil } for _, zpoolPath := range zpoolStatePaths { file, err := os.Open(zpoolPath) if err != nil { // This file should exist, but there is a race where an exporting pool can remove the files. Ok to ignore. c.logger.Debug("Cannot open file for reading", "path", zpoolPath) return errZFSNotAvailable } err = c.parsePoolStateFile(file, zpoolPath, func(poolName string, stateName string, isActive uint64) { ch <- c.constPoolStateMetric(poolName, stateName, isActive) }) file.Close() if err != nil { return err } } return nil } func (c *zfsCollector) parseProcfsFile(reader io.Reader, fmtExt string, handler func(zfsSysctl, interface{})) error { scanner := bufio.NewScanner(reader) parseLine := false for scanner.Scan() { parts := strings.Fields(scanner.Text()) if !parseLine && len(parts) == 3 && parts[0] == "name" && parts[1] == "type" && parts[2] == "data" { // Start parsing from here. parseLine = true continue } if !parseLine || len(parts) < 3 { continue } // kstat data type (column 2) should be KSTAT_DATA_UINT64, otherwise ignore // TODO: when other KSTAT_DATA_* types arrive, much of this will need to be restructured key := fmt.Sprintf("kstat.zfs.misc.%s.%s", fmtExt, parts[0]) switch parts[1] { case kstatDataUint64: value, err := strconv.ParseUint(parts[2], 10, 64) if err != nil { return fmt.Errorf("could not parse expected unsigned integer value for %q: %w", key, err) } handler(zfsSysctl(key), value) case kstatDataInt64: value, err := strconv.ParseInt(parts[2], 10, 64) if err != nil { return fmt.Errorf("could not parse expected signed integer value for %q: %w", key, err) } handler(zfsSysctl(key), value) } } if !parseLine { return fmt.Errorf("did not parse a single %q metric", fmtExt) } return scanner.Err() } func (c *zfsCollector) parsePoolProcfsFile(reader io.Reader, zpoolPath string, handler func(string, zfsSysctl, uint64)) error { scanner := bufio.NewScanner(reader) parseLine := false var fields []string for scanner.Scan() { line := strings.Fields(scanner.Text()) if !parseLine && len(line) >= 12 && line[0] == "nread" { //Start parsing from here. parseLine = true fields = make([]string, len(line)) copy(fields, line) continue } if !parseLine { continue } zpoolPathElements := strings.Split(zpoolPath, "/") pathLen := len(zpoolPathElements) if pathLen < 2 { return fmt.Errorf("zpool path did not return at least two elements") } zpoolName := zpoolPathElements[pathLen-2] zpoolFile := zpoolPathElements[pathLen-1] for i, field := range fields { key := fmt.Sprintf("kstat.zfs.misc.%s.%s", zpoolFile, field) value, err := strconv.ParseUint(line[i], 10, 64) if err != nil { return fmt.Errorf("could not parse expected integer value for %q: %w", key, err) } handler(zpoolName, zfsSysctl(key), value) } } return scanner.Err() } func (c *zfsCollector) parsePoolObjsetFile(reader io.Reader, zpoolPath string, handler func(string, string, zfsSysctl, uint64)) error { scanner := bufio.NewScanner(reader) parseLine := false var zpoolName, datasetName string for scanner.Scan() { parts := strings.Fields(scanner.Text()) if !parseLine && len(parts) == 3 && parts[0] == "name" && parts[1] == "type" && parts[2] == "data" { parseLine = true continue } if !parseLine || len(parts) < 3 { continue } if parts[0] == "dataset_name" { zpoolPathElements := strings.Split(zpoolPath, "/") pathLen := len(zpoolPathElements) zpoolName = zpoolPathElements[pathLen-2] datasetName = parts[2] continue } if parts[1] == kstatDataUint64 { key := fmt.Sprintf("kstat.zfs.misc.objset.%s", parts[0]) value, err := strconv.ParseUint(parts[2], 10, 64) if err != nil { return fmt.Errorf("could not parse expected integer value for %q", key) } handler(zpoolName, datasetName, zfsSysctl(key), value) } } if !parseLine { return fmt.Errorf("did not parse a single %s %s metric", zpoolName, datasetName) } return scanner.Err() } func (c *zfsCollector) parsePoolStateFile(reader io.Reader, zpoolPath string, handler func(string, string, uint64)) error { scanner := bufio.NewScanner(reader) scanner.Scan() actualStateName, err := scanner.Text(), scanner.Err() if err != nil { return err } actualStateName = strings.ToLower(actualStateName) zpoolPathElements := strings.Split(zpoolPath, "/") pathLen := len(zpoolPathElements) if pathLen < 2 { return fmt.Errorf("zpool path did not return at least two elements") } zpoolName := zpoolPathElements[pathLen-2] for _, stateName := range zfsPoolStatesName { isActive := uint64(0) if actualStateName == stateName { isActive = 1 } handler(zpoolName, stateName, isActive) } return nil } func (c *zfsCollector) constSysctlMetric(subsystem string, sysctl zfsSysctl, value float64) prometheus.Metric { metricName := sysctl.metricName() return prometheus.MustNewConstMetric( prometheus.NewDesc( prometheus.BuildFQName(namespace, subsystem, metricName), string(sysctl), nil, nil, ), prometheus.UntypedValue, value, ) } func (c *zfsCollector) constPoolMetric(poolName string, sysctl zfsSysctl, value uint64) prometheus.Metric { metricName := sysctl.metricName() return prometheus.MustNewConstMetric( prometheus.NewDesc( prometheus.BuildFQName(namespace, "zfs_zpool", metricName), string(sysctl), []string{"zpool"}, nil, ), prometheus.UntypedValue, float64(value), poolName, ) } func (c *zfsCollector) constPoolObjsetMetric(poolName string, datasetName string, sysctl zfsSysctl, value uint64) prometheus.Metric { metricName := sysctl.metricName() return prometheus.MustNewConstMetric( prometheus.NewDesc( prometheus.BuildFQName(namespace, "zfs_zpool_dataset", metricName), string(sysctl), []string{"zpool", "dataset"}, nil, ), prometheus.UntypedValue, float64(value), poolName, datasetName, ) } func (c *zfsCollector) constPoolStateMetric(poolName string, stateName string, isActive uint64) prometheus.Metric { return prometheus.MustNewConstMetric( prometheus.NewDesc( prometheus.BuildFQName(namespace, "zfs_zpool", "state"), "kstat.zfs.misc.state", []string{"zpool", "state"}, nil, ), prometheus.GaugeValue, float64(isActive), poolName, stateName, ) } type zfsSysctl string func (s zfsSysctl) metricName() string { parts := strings.Split(string(s), ".") return strings.Replace(parts[len(parts)-1], "-", "_", -1) }