Merge branch 'prometheus:master' into add_infiniband_hwcounters

.circleci/config.yml

@@ -46,7 +46,7 @@ jobs:
     parallelism: 3
     steps:
       - prometheus/setup_environment
-      - run: docker run --privileged linuxkit/binfmt:v0.8
+      - run: docker run --privileged linuxkit/binfmt:af88a591f9cc896a52ce596b9cf7ca26a061ef97
       - run: promu crossbuild -v --parallelism $CIRCLE_NODE_TOTAL --parallelism-thread $CIRCLE_NODE_INDEX
       - run: promu --config .promu-cgo.yml crossbuild -v --parallelism $CIRCLE_NODE_TOTAL --parallelism-thread $CIRCLE_NODE_INDEX
       - persist_to_workspace:
@@ -60,7 +60,7 @@ jobs:
     machine:
       image: ubuntu-2204:current
     environment:
-      DOCKER_TEST_IMAGE_NAME: quay.io/prometheus/golang-builder:1.18-base
+      DOCKER_TEST_IMAGE_NAME: quay.io/prometheus/golang-builder:1.21-base
       REPO_PATH: github.com/prometheus/node_exporter
     steps:
       - prometheus/setup_environment

.github/workflows/container_description.yml

@@ -0,0 +1,52 @@
+---
+name: Push README to Docker Hub
+on:
+  push:
+    paths:
+      - "README.md"
+      - ".github/workflows/container_description.yml"
+    branches: [ main, master ]
+
+permissions:
+  contents: read
+
+jobs:
+  PushDockerHubReadme:
+    runs-on: ubuntu-latest
+    name: Push README to Docker Hub
+    if: github.repository_owner == 'prometheus' || github.repository_owner == 'prometheus-community' # Don't run this workflow on forks.
+    steps:
+      - name: git checkout
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: Set docker hub repo name
+        run: echo "DOCKER_REPO_NAME=$(make docker-repo-name)" >> $GITHUB_ENV
+      - name: Push README to Dockerhub
+        uses: christian-korneck/update-container-description-action@d36005551adeaba9698d8d67a296bd16fa91f8e8 # v1
+        env:
+          DOCKER_USER: ${{ secrets.DOCKER_HUB_LOGIN }}
+          DOCKER_PASS: ${{ secrets.DOCKER_HUB_PASSWORD }}
+        with:
+          destination_container_repo: ${{ env.DOCKER_REPO_NAME }}
+          provider: dockerhub
+          short_description: ${{ env.DOCKER_REPO_NAME }}
+          readme_file: 'README.md'
+
+  PushQuayIoReadme:
+    runs-on: ubuntu-latest
+    name: Push README to quay.io
+    if: github.repository_owner == 'prometheus' || github.repository_owner == 'prometheus-community' # Don't run this workflow on forks.
+    steps:
+      - name: git checkout
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: Set quay.io org name
+        run: echo "DOCKER_REPO=$(echo quay.io/${GITHUB_REPOSITORY_OWNER} | tr -d '-')" >> $GITHUB_ENV
+      - name: Set quay.io repo name
+        run: echo "DOCKER_REPO_NAME=$(make docker-repo-name)" >> $GITHUB_ENV
+      - name: Push README to quay.io
+        uses: christian-korneck/update-container-description-action@d36005551adeaba9698d8d67a296bd16fa91f8e8 # v1
+        env:
+          DOCKER_APIKEY: ${{ secrets.QUAY_IO_API_TOKEN }}
+        with:
+          destination_container_repo: ${{ env.DOCKER_REPO_NAME }}
+          provider: quay
+          readme_file: 'README.md'

.github/workflows/golangci-lint.yml

@@ -12,21 +12,27 @@ on:
       - ".golangci.yml"
   pull_request:
 
+permissions:  # added using https://github.com/step-security/secure-repo
+  contents: read
+
 jobs:
   golangci:
+    permissions:
+      contents: read  # for actions/checkout to fetch code
+      pull-requests: read  # for golangci/golangci-lint-action to fetch pull requests
     name: lint
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - name: install Go
-        uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
+        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
         with:
-          go-version: 1.20.x
+          go-version: 1.22.x
       - name: Install snmp_exporter/generator dependencies
         run: sudo apt-get update && sudo apt-get -y install libsnmp-dev
         if: github.repository == 'prometheus/snmp_exporter'
       - name: Lint
-        uses: golangci/golangci-lint-action@3a919529898de77ec3da873e3063ca4b10e7f5cc # v3.7.0
+        uses: golangci/golangci-lint-action@3cfe3a4abbb849e10058ce4af15d205b6da42804 # v4.0.0
         with:
-          version: v1.54.2
+          version: v1.55.2

.golangci.yml

@@ -1,5 +1,6 @@
 linters:
   enable:
+    - depguard
     - misspell
     - revive
   disable:
@@ -19,6 +20,14 @@ issues:
         - errcheck
 
 linters-settings:
+  depguard:
+    rules:
+      no_exec_policy:
+        files:
+          - "!$test"
+        deny:
+          - pkg: "os/exec"
+            desc: "Using os/exec to run sub processes it not allowed by policy"
   errcheck:
     exclude-functions:
       # Used in HTTP handlers, any error is handled by the server itself.

CHANGELOG.md

@@ -5,6 +5,23 @@
 * [ENHANCEMENT]
 * [BUGFIX]
 
+## 1.7.0 / 2023-11-11
+
+* [FEATURE] Add ZFS freebsd per dataset stats #2753
+* [FEATURE] Add cpu vulnerabilities reporting from sysfs #2721
+* [ENHANCEMENT] Parallelize stat calls in Linux filesystem collector #1772
+* [ENHANCEMENT] Add missing linkspeeds to ethtool collector 2711
+* [ENHANCEMENT] Add CPU MHz as the value for `node_cpu_info` metric #2778
+* [ENHANCEMENT] Improve qdisc collector performance #2779
+* [ENHANCEMENT] Add include and exclude filter for hwmon collector #2699
+* [ENHANCEMENT] Optionally fetch ARP stats via rtnetlink instead of procfs #2777
+* [BUFFIX] Fix ZFS arcstats on FreeBSD 14.0+ 2754
+* [BUGFIX] Fallback to 32-bit stats in netdev #2757
+* [BUGFIX] Close btrfs.FS handle after use #2780
+* [BUGFIX] Move RO status before error return #2807
+* [BUFFIX] Fix `promhttp_metric_handler_errors_total` being always active #2808
+* [BUGFIX] Fix nfsd v4 index miss #2824
+
 ## 1.6.1 / 2023-06-17
 
 Rebuild with latest Go compiler bugfix release.

Makefile.common

@@ -61,11 +61,11 @@ PROMU_URL     := https://github.com/prometheus/promu/releases/download/v$(PROMU_
 SKIP_GOLANGCI_LINT :=
 GOLANGCI_LINT :=
 GOLANGCI_LINT_OPTS ?=
-GOLANGCI_LINT_VERSION ?= v1.54.2
+GOLANGCI_LINT_VERSION ?= v1.55.2
-# golangci-lint only supports linux, darwin and windows platforms on i386/amd64.
+# golangci-lint only supports linux, darwin and windows platforms on i386/amd64/arm64.
 # windows isn't included here because of the path separator being different.
 ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin))
-	ifeq ($(GOHOSTARCH),$(filter $(GOHOSTARCH),amd64 i386))
+	ifeq ($(GOHOSTARCH),$(filter $(GOHOSTARCH),amd64 i386 arm64))
 		# If we're in CI and there is an Actions file, that means the linter
 		# is being run in Actions, so we don't need to run it here.
 		ifneq (,$(SKIP_GOLANGCI_LINT))
@@ -169,12 +169,16 @@ common-vet:
 common-lint: $(GOLANGCI_LINT)
 ifdef GOLANGCI_LINT
 	@echo ">> running golangci-lint"
-# 'go list' needs to be executed before staticcheck to prepopulate the modules cache.
-# Otherwise staticcheck might fail randomly for some reason not yet explained.
-	$(GO) list -e -compiled -test=true -export=false -deps=true -find=false -tags= -- ./... > /dev/null
 	$(GOLANGCI_LINT) run $(GOLANGCI_LINT_OPTS) $(pkgs)
 endif
 
+.PHONY: common-lint-fix
+common-lint-fix: $(GOLANGCI_LINT)
+ifdef GOLANGCI_LINT
+	@echo ">> running golangci-lint fix"
+	$(GOLANGCI_LINT) run --fix $(GOLANGCI_LINT_OPTS) $(pkgs)
+endif
+
 .PHONY: common-yamllint
 common-yamllint:
 	@echo ">> running yamllint on all YAML files in the repository"
@@ -204,6 +208,10 @@ common-tarball: promu
 	@echo ">> building release tarball"
 	$(PROMU) tarball --prefix $(PREFIX) $(BIN_DIR)
 
+.PHONY: common-docker-repo-name
+common-docker-repo-name:
+	@echo "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)"
+
 .PHONY: common-docker $(BUILD_DOCKER_ARCHS)
 common-docker: $(BUILD_DOCKER_ARCHS)
 $(BUILD_DOCKER_ARCHS): common-docker-%:

README.md

@@ -26,10 +26,10 @@ For automated installs with [Ansible](https://www.ansible.com/), there is the [P
 
 ### Docker
 
-The `node_exporter` is designed to monitor the host system. It's not recommended
+The `node_exporter` is designed to monitor the host system. Deploying in containers requires
-to deploy it as a Docker container because it requires access to the host system.
+extra care in order to avoid monitoring the container itself.
 
-For situations where Docker deployment is needed, some extra flags must be used to allow
+For situations where containerized deployment is needed, some extra flags must be used to allow
 the `node_exporter` access to the host namespaces.
 
 Be aware that any non-root mount points you want to monitor will need to be bind-mounted
@@ -158,6 +158,7 @@ timex | Exposes selected adjtimex(2) system call stats. | Linux
 udp_queues | Exposes UDP total lengths of the rx_queue and tx_queue from `/proc/net/udp` and `/proc/net/udp6`. | Linux
 uname | Exposes system information as provided by the uname system call. | Darwin, FreeBSD, Linux, OpenBSD
 vmstat | Exposes statistics from `/proc/vmstat`. | Linux
+watchdog | Exposes statistics from `/sys/class/watchdog` | Linux
 xfs | Exposes XFS runtime statistics. | Linux (kernel 4.4+)
 zfs | Exposes [ZFS](http://open-zfs.org/) performance statistics. | FreeBSD, [Linux](http://zfsonlinux.org/), Solaris
 
@@ -205,6 +206,7 @@ sysctl | Expose sysctl values from `/proc/sys`. Use `--collector.sysctl.include(
 systemd | Exposes service and system status from [systemd](http://www.freedesktop.org/wiki/Software/systemd/). | Linux
 tcpstat | Exposes TCP connection status information from `/proc/net/tcp` and `/proc/net/tcp6`. (Warning: the current version has potential performance issues in high load situations.) | Linux
 wifi | Exposes WiFi device and station statistics. | Linux
+xfrm | Exposes statistics from `/proc/net/xfrm_stat` | Linux
 zoneinfo | Exposes NUMA memory zone metrics. | Linux
 
 ### Deprecated

VERSION

@@ -1 +1 @@
-1.6.1
+1.7.0

collector/cpu_linux.go

@@ -205,7 +205,7 @@ func (c *cpuCollector) updateInfo(ch chan<- prometheus.Metric) error {
 
 	cpuFreqEnabled, ok := collectorState["cpufreq"]
 	if !ok || cpuFreqEnabled == nil {
-		level.Debug(c.logger).Log("cpufreq key missing or nil value in collectorState map", err)
+		level.Debug(c.logger).Log("msg", "cpufreq key missing or nil value in collectorState map")
 	} else if !*cpuFreqEnabled {
 		for _, cpu := range info {
 			ch <- prometheus.MustNewConstMetric(c.cpuFrequencyHz,

collector/cpu_vulnerabilities_linux.go

@@ -29,7 +29,7 @@ var (
 	vulnerabilityDesc = prometheus.NewDesc(
 		prometheus.BuildFQName(namespace, cpuVulerabilitiesCollector, "info"),
 		"Details of each CPU vulnerability reported by sysfs. The value of the series is an int encoded state of the vulnerability. The same state is stored as a string in the label",
-		[]string{"codename", "state"},
+		[]string{"codename", "state", "mitigation"},
 		nil,
 	)
 )
@@ -62,6 +62,7 @@ func (v *cpuVulnerabilitiesCollector) Update(ch chan<- prometheus.Metric) error
 			1.0,
 			vulnerability.CodeName,
 			sysfs.VulnerabilityHumanEncoding[vulnerability.State],
+			vulnerability.Mitigation,
 		)
 	}
 	return nil

collector/diskstats_linux.go

@@ -36,7 +36,7 @@ const (
 	// See also https://www.kernel.org/doc/Documentation/block/stat.txt
 	unixSectorSize = 512.0
 
-	diskstatsDefaultIgnoredDevices = "^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\\d+n\\d+p)\\d+$"
+	diskstatsDefaultIgnoredDevices = "^(z?ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\\d+n\\d+p)\\d+$"
 
 	// See udevadm(8).
 	udevDevicePropertyPrefix = "E:"

collector/diskstats_linux_test.go

@@ -53,7 +53,7 @@ func TestDiskStats(t *testing.T) {
 	*sysPath = "fixtures/sys"
 	*procPath = "fixtures/proc"
 	*udevDataPath = "fixtures/udev/data"
-	*diskstatsDeviceExclude = "^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\\d+n\\d+p)\\d+$"
+	*diskstatsDeviceExclude = "^(z?ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\\d+n\\d+p)\\d+$"
 	testcase := `# HELP node_disk_ata_rotation_rate_rpm ATA disk rotation rate in RPMs (0 for SSDs).
 # TYPE node_disk_ata_rotation_rate_rpm gauge
 node_disk_ata_rotation_rate_rpm{device="sda"} 7200

collector/ethtool_linux.go

@@ -445,18 +445,19 @@ func (c *ethtoolCollector) Update(ch chan<- prometheus.Metric) error {
 		// Sanitizing the metric names can lead to duplicate metric names. Therefore check for clashes beforehand.
 		metricFQNames := make(map[string]string)
 		for metric := range stats {
-			if !c.metricsPattern.MatchString(metric) {
+			metricName := SanitizeMetricName(metric)
+			if !c.metricsPattern.MatchString(metricName) {
 				continue
 			}
-			metricFQName := buildEthtoolFQName(metric)
+			metricFQName := buildEthtoolFQName(metricName)
 			existingMetric, exists := metricFQNames[metricFQName]
 			if exists {
 				level.Debug(c.logger).Log("msg", "dropping duplicate metric name", "device", device,
-					"metricFQName", metricFQName, "metric1", existingMetric, "metric2", metric)
+					"metricFQName", metricFQName, "metric1", existingMetric, "metric2", metricName)
-				// Keep the metric as "deleted" in the dict in case there are 3 duplicates.
+				// Keep the metricName as "deleted" in the dict in case there are 3 duplicates.
 				metricFQNames[metricFQName] = ""
 			} else {
-				metricFQNames[metricFQName] = metric
+				metricFQNames[metricFQName] = metricName
 			}
 		}
 

collector/exec_bsd.go

@@ -49,37 +49,31 @@ func NewExecCollector(logger log.Logger) (Collector, error) {
 				name:        "exec_context_switches_total",
 				description: "Context switches since system boot.  Resets at architecture unsigned integer.",
 				mib:         "vm.stats.sys.v_swtch",
-				labels:      nil,
 			},
 			{
 				name:        "exec_traps_total",
 				description: "Traps since system boot.  Resets at architecture unsigned integer.",
 				mib:         "vm.stats.sys.v_trap",
-				labels:      nil,
 			},
 			{
 				name:        "exec_system_calls_total",
 				description: "System calls since system boot.  Resets at architecture unsigned integer.",
 				mib:         "vm.stats.sys.v_syscall",
 			},
-			labels: nil,
 			{
 				name:        "exec_device_interrupts_total",
 				description: "Device interrupts since system boot.  Resets at architecture unsigned integer.",
 				mib:         "vm.stats.sys.v_intr",
-				labels:      nil,
 			},
 			{
 				name:        "exec_software_interrupts_total",
 				description: "Software interrupts since system boot.  Resets at architecture unsigned integer.",
 				mib:         "vm.stats.sys.v_soft",
-				labels:      nil,
 			},
 			{
 				name:        "exec_forks_total",
 				description: "Number of fork() calls since system boot.  Resets at architecture unsigned integer.",
 				mib:         "vm.stats.vm.v_forks",
-				labels:      nil,
 			},
 		},
 		logger: logger,

collector/filesystem_common.go

@@ -60,7 +60,7 @@ var (
 		"Regexp of filesystem types to ignore for filesystem collector.",
 	).Hidden().String()
 
-	filesystemLabelNames = []string{"device", "mountpoint", "fstype"}
+	filesystemLabelNames = []string{"device", "mountpoint", "fstype", "device_error"}
 )
 
 type filesystemCollector struct {
@@ -73,7 +73,7 @@ type filesystemCollector struct {
 }
 
 type filesystemLabels struct {
-	device, mountPoint, fsType, options string
+	device, mountPoint, fsType, options, deviceError string
 }
 
 type filesystemStats struct {
@@ -184,11 +184,11 @@ func (c *filesystemCollector) Update(ch chan<- prometheus.Metric) error {
 
 		ch <- prometheus.MustNewConstMetric(
 			c.deviceErrorDesc, prometheus.GaugeValue,
-			s.deviceError, s.labels.device, s.labels.mountPoint, s.labels.fsType,
+			s.deviceError, s.labels.device, s.labels.mountPoint, s.labels.fsType, s.labels.deviceError,
 		)
 		ch <- prometheus.MustNewConstMetric(
 			c.roDesc, prometheus.GaugeValue,
-			s.ro, s.labels.device, s.labels.mountPoint, s.labels.fsType,
+			s.ro, s.labels.device, s.labels.mountPoint, s.labels.fsType, s.labels.deviceError,
 		)
 
 		if s.deviceError > 0 {
@@ -197,23 +197,23 @@ func (c *filesystemCollector) Update(ch chan<- prometheus.Metric) error {
 
 		ch <- prometheus.MustNewConstMetric(
 			c.sizeDesc, prometheus.GaugeValue,
-			s.size, s.labels.device, s.labels.mountPoint, s.labels.fsType,
+			s.size, s.labels.device, s.labels.mountPoint, s.labels.fsType, s.labels.deviceError,
 		)
 		ch <- prometheus.MustNewConstMetric(
 			c.freeDesc, prometheus.GaugeValue,
-			s.free, s.labels.device, s.labels.mountPoint, s.labels.fsType,
+			s.free, s.labels.device, s.labels.mountPoint, s.labels.fsType, s.labels.deviceError,
 		)
 		ch <- prometheus.MustNewConstMetric(
 			c.availDesc, prometheus.GaugeValue,
-			s.avail, s.labels.device, s.labels.mountPoint, s.labels.fsType,
+			s.avail, s.labels.device, s.labels.mountPoint, s.labels.fsType, s.labels.deviceError,
 		)
 		ch <- prometheus.MustNewConstMetric(
 			c.filesDesc, prometheus.GaugeValue,
-			s.files, s.labels.device, s.labels.mountPoint, s.labels.fsType,
+			s.files, s.labels.device, s.labels.mountPoint, s.labels.fsType, s.labels.deviceError,
 		)
 		ch <- prometheus.MustNewConstMetric(
 			c.filesFreeDesc, prometheus.GaugeValue,
-			s.filesFree, s.labels.device, s.labels.mountPoint, s.labels.fsType,
+			s.filesFree, s.labels.device, s.labels.mountPoint, s.labels.fsType, s.labels.deviceError,
 		)
 	}
 	return nil

collector/filesystem_linux.go

@@ -85,6 +85,7 @@ func (c *filesystemCollector) GetStats() ([]filesystemStats, error) {
 
 			stuckMountsMtx.Lock()
 			if _, ok := stuckMounts[labels.mountPoint]; ok {
+				labels.deviceError = "mountpoint timeout"
 				stats = append(stats, filesystemStats{
 					labels:      labels,
 					deviceError: 1,
@@ -133,6 +134,7 @@ func (c *filesystemCollector) processStat(labels filesystemLabels) filesystemSta
 	stuckMountsMtx.Unlock()
 
 	if err != nil {
+		labels.deviceError = err.Error()
 		level.Debug(c.logger).Log("msg", "Error on statfs() system call", "rootfs", rootfsFilePath(labels.mountPoint), "err", err)
 		return filesystemStats{
 			labels:      labels,
@@ -211,6 +213,7 @@ func parseFilesystemLabels(r io.Reader) ([]filesystemLabels, error) {
 			mountPoint:  rootfsStripPrefix(parts[1]),
 			fsType:      parts[2],
 			options:     parts[3],
+			deviceError: "",
 		})
 	}
 

collector/fixtures/e2e-64k-page-output.txt

@@ -404,11 +404,11 @@ node_cpu_seconds_total{cpu="7",mode="system"} 101.64
 node_cpu_seconds_total{cpu="7",mode="user"} 290.98
 # HELP node_cpu_vulnerabilities_info Details of each CPU vulnerability reported by sysfs. The value of the series is an int encoded state of the vulnerability. The same state is stored as a string in the label
 # TYPE node_cpu_vulnerabilities_info gauge
-node_cpu_vulnerabilities_info{codename="itlb_multihit",state="not affected"} 1
+node_cpu_vulnerabilities_info{codename="itlb_multihit",mitigation="",state="not affected"} 1
-node_cpu_vulnerabilities_info{codename="mds",state="vulnerable"} 1
+node_cpu_vulnerabilities_info{codename="mds",mitigation="",state="vulnerable"} 1
-node_cpu_vulnerabilities_info{codename="retbleed",state="mitigation"} 1
+node_cpu_vulnerabilities_info{codename="retbleed",mitigation="untrained return thunk; SMT enabled with STIBP protection",state="mitigation"} 1
-node_cpu_vulnerabilities_info{codename="spectre_v1",state="mitigation"} 1
+node_cpu_vulnerabilities_info{codename="spectre_v1",mitigation="usercopy/swapgs barriers and __user pointer sanitization",state="mitigation"} 1
-node_cpu_vulnerabilities_info{codename="spectre_v2",state="mitigation"} 1
+node_cpu_vulnerabilities_info{codename="spectre_v2",mitigation="Retpolines, IBPB: conditional, STIBP: always-on, RSB filling, PBRSB-eIBRS: Not affected",state="mitigation"} 1
 # HELP node_disk_ata_rotation_rate_rpm ATA disk rotation rate in RPMs (0 for SSDs).
 # TYPE node_disk_ata_rotation_rate_rpm gauge
 node_disk_ata_rotation_rate_rpm{device="sda"} 7200
@@ -2303,6 +2303,9 @@ node_netstat_TcpExt_SyncookiesRecv 0
 # HELP node_netstat_TcpExt_SyncookiesSent Statistic TcpExtSyncookiesSent.
 # TYPE node_netstat_TcpExt_SyncookiesSent untyped
 node_netstat_TcpExt_SyncookiesSent 0
+# HELP node_netstat_TcpExt_TCPOFOQueue Statistic TcpExtTCPOFOQueue.
+# TYPE node_netstat_TcpExt_TCPOFOQueue untyped
+node_netstat_TcpExt_TCPOFOQueue 42
 # HELP node_netstat_TcpExt_TCPTimeouts Statistic TcpExtTCPTimeouts.
 # TYPE node_netstat_TcpExt_TCPTimeouts untyped
 node_netstat_TcpExt_TCPTimeouts 115
@@ -2743,6 +2746,7 @@ node_nfsd_requests_total{method="SetClientIDConfirm",proto="4"} 3
 node_nfsd_requests_total{method="SymLink",proto="2"} 0
 node_nfsd_requests_total{method="SymLink",proto="3"} 0
 node_nfsd_requests_total{method="Verify",proto="4"} 0
+node_nfsd_requests_total{method="WdelegGetattr",proto="4"} 15
 node_nfsd_requests_total{method="WrCache",proto="2"} 0
 node_nfsd_requests_total{method="Write",proto="2"} 0
 node_nfsd_requests_total{method="Write",proto="3"} 0
@@ -2929,7 +2933,6 @@ node_scrape_collector_success{collector="processes"} 1
 node_scrape_collector_success{collector="qdisc"} 1
 node_scrape_collector_success{collector="rapl"} 1
 node_scrape_collector_success{collector="schedstat"} 1
-node_scrape_collector_success{collector="selinux"} 1
 node_scrape_collector_success{collector="slabinfo"} 1
 node_scrape_collector_success{collector="sockstat"} 1
 node_scrape_collector_success{collector="softirqs"} 1
@@ -2942,13 +2945,12 @@ node_scrape_collector_success{collector="thermal_zone"} 1
 node_scrape_collector_success{collector="time"} 1
 node_scrape_collector_success{collector="udp_queues"} 1
 node_scrape_collector_success{collector="vmstat"} 1
+node_scrape_collector_success{collector="watchdog"} 1
 node_scrape_collector_success{collector="wifi"} 1
+node_scrape_collector_success{collector="xfrm"} 1
 node_scrape_collector_success{collector="xfs"} 1
 node_scrape_collector_success{collector="zfs"} 1
 node_scrape_collector_success{collector="zoneinfo"} 1
-# HELP node_selinux_enabled SELinux is enabled, 1 is true, 0 is false
-# TYPE node_selinux_enabled gauge
-node_selinux_enabled 0
 # HELP node_slabinfo_active_objects The number of objects that are currently active (i.e., in use).
 # TYPE node_slabinfo_active_objects gauge
 node_slabinfo_active_objects{slab="dmaengine-unmap-128"} 1206
@@ -3217,6 +3219,31 @@ node_vmstat_pswpin 1476
 # HELP node_vmstat_pswpout /proc/vmstat information field pswpout.
 # TYPE node_vmstat_pswpout untyped
 node_vmstat_pswpout 35045
+# HELP node_watchdog_access_cs0 Value of /sys/class/watchdog/<watchdog>/access_cs0
+# TYPE node_watchdog_access_cs0 gauge
+node_watchdog_access_cs0{name="watchdog0"} 0
+# HELP node_watchdog_bootstatus Value of /sys/class/watchdog/<watchdog>/bootstatus
+# TYPE node_watchdog_bootstatus gauge
+node_watchdog_bootstatus{name="watchdog0"} 1
+# HELP node_watchdog_fw_version Value of /sys/class/watchdog/<watchdog>/fw_version
+# TYPE node_watchdog_fw_version gauge
+node_watchdog_fw_version{name="watchdog0"} 2
+# HELP node_watchdog_info Info of /sys/class/watchdog/<watchdog>
+# TYPE node_watchdog_info gauge
+node_watchdog_info{identity="",name="watchdog1",options="",pretimeout_governor="",state="",status=""} 1
+node_watchdog_info{identity="Software Watchdog",name="watchdog0",options="0x8380",pretimeout_governor="noop",state="active",status="0x8000"} 1
+# HELP node_watchdog_nowayout Value of /sys/class/watchdog/<watchdog>/nowayout
+# TYPE node_watchdog_nowayout gauge
+node_watchdog_nowayout{name="watchdog0"} 0
+# HELP node_watchdog_pretimeout_seconds Value of /sys/class/watchdog/<watchdog>/pretimeout
+# TYPE node_watchdog_pretimeout_seconds gauge
+node_watchdog_pretimeout_seconds{name="watchdog0"} 120
+# HELP node_watchdog_timeleft_seconds Value of /sys/class/watchdog/<watchdog>/timeleft
+# TYPE node_watchdog_timeleft_seconds gauge
+node_watchdog_timeleft_seconds{name="watchdog0"} 300
+# HELP node_watchdog_timeout_seconds Value of /sys/class/watchdog/<watchdog>/timeout
+# TYPE node_watchdog_timeout_seconds gauge
+node_watchdog_timeout_seconds{name="watchdog0"} 60
 # HELP node_wifi_interface_frequency_hertz The current frequency a WiFi interface is operating at, in hertz.
 # TYPE node_wifi_interface_frequency_hertz gauge
 node_wifi_interface_frequency_hertz{device="wlan0"} 2.412e+09
@@ -3264,6 +3291,90 @@ node_wifi_station_transmit_failed_total{device="wlan0",mac_address="aa:bb:cc:dd:
 # TYPE node_wifi_station_transmit_retries_total counter
 node_wifi_station_transmit_retries_total{device="wlan0",mac_address="01:02:03:04:05:06"} 20
 node_wifi_station_transmit_retries_total{device="wlan0",mac_address="aa:bb:cc:dd:ee:ff"} 10
+# HELP node_xfrm_acquire_error_packets_total State hasn’t been fully acquired before use
+# TYPE node_xfrm_acquire_error_packets_total counter
+node_xfrm_acquire_error_packets_total 24532
+# HELP node_xfrm_fwd_hdr_error_packets_total Forward routing of a packet is not allowed
+# TYPE node_xfrm_fwd_hdr_error_packets_total counter
+node_xfrm_fwd_hdr_error_packets_total 6654
+# HELP node_xfrm_in_buffer_error_packets_total No buffer is left
+# TYPE node_xfrm_in_buffer_error_packets_total counter
+node_xfrm_in_buffer_error_packets_total 2
+# HELP node_xfrm_in_error_packets_total All errors not matched by other
+# TYPE node_xfrm_in_error_packets_total counter
+node_xfrm_in_error_packets_total 1
+# HELP node_xfrm_in_hdr_error_packets_total Header error
+# TYPE node_xfrm_in_hdr_error_packets_total counter
+node_xfrm_in_hdr_error_packets_total 4
+# HELP node_xfrm_in_no_pols_packets_total No policy is found for states e.g. Inbound SAs are correct but no SP is found
+# TYPE node_xfrm_in_no_pols_packets_total counter
+node_xfrm_in_no_pols_packets_total 65432
+# HELP node_xfrm_in_no_states_packets_total No state is found i.e. Either inbound SPI, address, or IPsec protocol at SA is wrong
+# TYPE node_xfrm_in_no_states_packets_total counter
+node_xfrm_in_no_states_packets_total 3
+# HELP node_xfrm_in_pol_block_packets_total Policy discards
+# TYPE node_xfrm_in_pol_block_packets_total counter
+node_xfrm_in_pol_block_packets_total 100
+# HELP node_xfrm_in_pol_error_packets_total Policy error
+# TYPE node_xfrm_in_pol_error_packets_total counter
+node_xfrm_in_pol_error_packets_total 10000
+# HELP node_xfrm_in_state_expired_packets_total State is expired
+# TYPE node_xfrm_in_state_expired_packets_total counter
+node_xfrm_in_state_expired_packets_total 7
+# HELP node_xfrm_in_state_invalid_packets_total State is invalid
+# TYPE node_xfrm_in_state_invalid_packets_total counter
+node_xfrm_in_state_invalid_packets_total 55555
+# HELP node_xfrm_in_state_mismatch_packets_total State has mismatch option e.g. UDP encapsulation type is mismatch
+# TYPE node_xfrm_in_state_mismatch_packets_total counter
+node_xfrm_in_state_mismatch_packets_total 23451
+# HELP node_xfrm_in_state_mode_error_packets_total Transformation mode specific error
+# TYPE node_xfrm_in_state_mode_error_packets_total counter
+node_xfrm_in_state_mode_error_packets_total 100
+# HELP node_xfrm_in_state_proto_error_packets_total Transformation protocol specific error e.g. SA key is wrong
+# TYPE node_xfrm_in_state_proto_error_packets_total counter
+node_xfrm_in_state_proto_error_packets_total 40
+# HELP node_xfrm_in_state_seq_error_packets_total Sequence error i.e. Sequence number is out of window
+# TYPE node_xfrm_in_state_seq_error_packets_total counter
+node_xfrm_in_state_seq_error_packets_total 6000
+# HELP node_xfrm_in_tmpl_mismatch_packets_total No matching template for states e.g. Inbound SAs are correct but SP rule is wrong
+# TYPE node_xfrm_in_tmpl_mismatch_packets_total counter
+node_xfrm_in_tmpl_mismatch_packets_total 51
+# HELP node_xfrm_out_bundle_check_error_packets_total Bundle check error
+# TYPE node_xfrm_out_bundle_check_error_packets_total counter
+node_xfrm_out_bundle_check_error_packets_total 555
+# HELP node_xfrm_out_bundle_gen_error_packets_total Bundle generation error
+# TYPE node_xfrm_out_bundle_gen_error_packets_total counter
+node_xfrm_out_bundle_gen_error_packets_total 43321
+# HELP node_xfrm_out_error_packets_total All errors which is not matched others
+# TYPE node_xfrm_out_error_packets_total counter
+node_xfrm_out_error_packets_total 1e+06
+# HELP node_xfrm_out_no_states_packets_total No state is found
+# TYPE node_xfrm_out_no_states_packets_total counter
+node_xfrm_out_no_states_packets_total 869
+# HELP node_xfrm_out_pol_block_packets_total Policy discards
+# TYPE node_xfrm_out_pol_block_packets_total counter
+node_xfrm_out_pol_block_packets_total 43456
+# HELP node_xfrm_out_pol_dead_packets_total Policy is dead
+# TYPE node_xfrm_out_pol_dead_packets_total counter
+node_xfrm_out_pol_dead_packets_total 7656
+# HELP node_xfrm_out_pol_error_packets_total Policy error
+# TYPE node_xfrm_out_pol_error_packets_total counter
+node_xfrm_out_pol_error_packets_total 1454
+# HELP node_xfrm_out_state_expired_packets_total State is expired
+# TYPE node_xfrm_out_state_expired_packets_total counter
+node_xfrm_out_state_expired_packets_total 565
+# HELP node_xfrm_out_state_invalid_packets_total State is invalid, perhaps expired
+# TYPE node_xfrm_out_state_invalid_packets_total counter
+node_xfrm_out_state_invalid_packets_total 28765
+# HELP node_xfrm_out_state_mode_error_packets_total Transformation mode specific error
+# TYPE node_xfrm_out_state_mode_error_packets_total counter
+node_xfrm_out_state_mode_error_packets_total 8
+# HELP node_xfrm_out_state_proto_error_packets_total Transformation protocol specific error
+# TYPE node_xfrm_out_state_proto_error_packets_total counter
+node_xfrm_out_state_proto_error_packets_total 4542
+# HELP node_xfrm_out_state_seq_error_packets_total Sequence error i.e. Sequence number overflow
+# TYPE node_xfrm_out_state_seq_error_packets_total counter
+node_xfrm_out_state_seq_error_packets_total 543
 # HELP node_xfs_allocation_btree_compares_total Number of allocation B-tree compares for a filesystem.
 # TYPE node_xfs_allocation_btree_compares_total counter
 node_xfs_allocation_btree_compares_total{device="sda1"} 0

collector/fixtures/e2e-output.txt

@@ -426,11 +426,11 @@ node_cpu_seconds_total{cpu="7",mode="system"} 101.64
 node_cpu_seconds_total{cpu="7",mode="user"} 290.98
 # HELP node_cpu_vulnerabilities_info Details of each CPU vulnerability reported by sysfs. The value of the series is an int encoded state of the vulnerability. The same state is stored as a string in the label
 # TYPE node_cpu_vulnerabilities_info gauge
-node_cpu_vulnerabilities_info{codename="itlb_multihit",state="not affected"} 1
+node_cpu_vulnerabilities_info{codename="itlb_multihit",mitigation="",state="not affected"} 1
-node_cpu_vulnerabilities_info{codename="mds",state="vulnerable"} 1
+node_cpu_vulnerabilities_info{codename="mds",mitigation="",state="vulnerable"} 1
-node_cpu_vulnerabilities_info{codename="retbleed",state="mitigation"} 1
+node_cpu_vulnerabilities_info{codename="retbleed",mitigation="untrained return thunk; SMT enabled with STIBP protection",state="mitigation"} 1
-node_cpu_vulnerabilities_info{codename="spectre_v1",state="mitigation"} 1
+node_cpu_vulnerabilities_info{codename="spectre_v1",mitigation="usercopy/swapgs barriers and __user pointer sanitization",state="mitigation"} 1
-node_cpu_vulnerabilities_info{codename="spectre_v2",state="mitigation"} 1
+node_cpu_vulnerabilities_info{codename="spectre_v2",mitigation="Retpolines, IBPB: conditional, STIBP: always-on, RSB filling, PBRSB-eIBRS: Not affected",state="mitigation"} 1
 # HELP node_disk_ata_rotation_rate_rpm ATA disk rotation rate in RPMs (0 for SSDs).
 # TYPE node_disk_ata_rotation_rate_rpm gauge
 node_disk_ata_rotation_rate_rpm{device="sda"} 7200
@@ -2325,6 +2325,9 @@ node_netstat_TcpExt_SyncookiesRecv 0
 # HELP node_netstat_TcpExt_SyncookiesSent Statistic TcpExtSyncookiesSent.
 # TYPE node_netstat_TcpExt_SyncookiesSent untyped
 node_netstat_TcpExt_SyncookiesSent 0
+# HELP node_netstat_TcpExt_TCPOFOQueue Statistic TcpExtTCPOFOQueue.
+# TYPE node_netstat_TcpExt_TCPOFOQueue untyped
+node_netstat_TcpExt_TCPOFOQueue 42
 # HELP node_netstat_TcpExt_TCPTimeouts Statistic TcpExtTCPTimeouts.
 # TYPE node_netstat_TcpExt_TCPTimeouts untyped
 node_netstat_TcpExt_TCPTimeouts 115
@@ -2765,6 +2768,7 @@ node_nfsd_requests_total{method="SetClientIDConfirm",proto="4"} 3
 node_nfsd_requests_total{method="SymLink",proto="2"} 0
 node_nfsd_requests_total{method="SymLink",proto="3"} 0
 node_nfsd_requests_total{method="Verify",proto="4"} 0
+node_nfsd_requests_total{method="WdelegGetattr",proto="4"} 15
 node_nfsd_requests_total{method="WrCache",proto="2"} 0
 node_nfsd_requests_total{method="Write",proto="2"} 0
 node_nfsd_requests_total{method="Write",proto="3"} 0
@@ -2951,7 +2955,6 @@ node_scrape_collector_success{collector="processes"} 1
 node_scrape_collector_success{collector="qdisc"} 1
 node_scrape_collector_success{collector="rapl"} 1
 node_scrape_collector_success{collector="schedstat"} 1
-node_scrape_collector_success{collector="selinux"} 1
 node_scrape_collector_success{collector="slabinfo"} 1
 node_scrape_collector_success{collector="sockstat"} 1
 node_scrape_collector_success{collector="softirqs"} 1
@@ -2964,13 +2967,12 @@ node_scrape_collector_success{collector="thermal_zone"} 1
 node_scrape_collector_success{collector="time"} 1
 node_scrape_collector_success{collector="udp_queues"} 1
 node_scrape_collector_success{collector="vmstat"} 1
+node_scrape_collector_success{collector="watchdog"} 1
 node_scrape_collector_success{collector="wifi"} 1
+node_scrape_collector_success{collector="xfrm"} 1
 node_scrape_collector_success{collector="xfs"} 1
 node_scrape_collector_success{collector="zfs"} 1
 node_scrape_collector_success{collector="zoneinfo"} 1
-# HELP node_selinux_enabled SELinux is enabled, 1 is true, 0 is false
-# TYPE node_selinux_enabled gauge
-node_selinux_enabled 0
 # HELP node_slabinfo_active_objects The number of objects that are currently active (i.e., in use).
 # TYPE node_slabinfo_active_objects gauge
 node_slabinfo_active_objects{slab="dmaengine-unmap-128"} 1206
@@ -3239,6 +3241,31 @@ node_vmstat_pswpin 1476
 # HELP node_vmstat_pswpout /proc/vmstat information field pswpout.
 # TYPE node_vmstat_pswpout untyped
 node_vmstat_pswpout 35045
+# HELP node_watchdog_access_cs0 Value of /sys/class/watchdog/<watchdog>/access_cs0
+# TYPE node_watchdog_access_cs0 gauge
+node_watchdog_access_cs0{name="watchdog0"} 0
+# HELP node_watchdog_bootstatus Value of /sys/class/watchdog/<watchdog>/bootstatus
+# TYPE node_watchdog_bootstatus gauge
+node_watchdog_bootstatus{name="watchdog0"} 1
+# HELP node_watchdog_fw_version Value of /sys/class/watchdog/<watchdog>/fw_version
+# TYPE node_watchdog_fw_version gauge
+node_watchdog_fw_version{name="watchdog0"} 2
+# HELP node_watchdog_info Info of /sys/class/watchdog/<watchdog>
+# TYPE node_watchdog_info gauge
+node_watchdog_info{identity="",name="watchdog1",options="",pretimeout_governor="",state="",status=""} 1
+node_watchdog_info{identity="Software Watchdog",name="watchdog0",options="0x8380",pretimeout_governor="noop",state="active",status="0x8000"} 1
+# HELP node_watchdog_nowayout Value of /sys/class/watchdog/<watchdog>/nowayout
+# TYPE node_watchdog_nowayout gauge
+node_watchdog_nowayout{name="watchdog0"} 0
+# HELP node_watchdog_pretimeout_seconds Value of /sys/class/watchdog/<watchdog>/pretimeout
+# TYPE node_watchdog_pretimeout_seconds gauge
+node_watchdog_pretimeout_seconds{name="watchdog0"} 120
+# HELP node_watchdog_timeleft_seconds Value of /sys/class/watchdog/<watchdog>/timeleft
+# TYPE node_watchdog_timeleft_seconds gauge
+node_watchdog_timeleft_seconds{name="watchdog0"} 300
+# HELP node_watchdog_timeout_seconds Value of /sys/class/watchdog/<watchdog>/timeout
+# TYPE node_watchdog_timeout_seconds gauge
+node_watchdog_timeout_seconds{name="watchdog0"} 60
 # HELP node_wifi_interface_frequency_hertz The current frequency a WiFi interface is operating at, in hertz.
 # TYPE node_wifi_interface_frequency_hertz gauge
 node_wifi_interface_frequency_hertz{device="wlan0"} 2.412e+09
@@ -3286,6 +3313,90 @@ node_wifi_station_transmit_failed_total{device="wlan0",mac_address="aa:bb:cc:dd:
 # TYPE node_wifi_station_transmit_retries_total counter
 node_wifi_station_transmit_retries_total{device="wlan0",mac_address="01:02:03:04:05:06"} 20
 node_wifi_station_transmit_retries_total{device="wlan0",mac_address="aa:bb:cc:dd:ee:ff"} 10
+# HELP node_xfrm_acquire_error_packets_total State hasn’t been fully acquired before use
+# TYPE node_xfrm_acquire_error_packets_total counter
+node_xfrm_acquire_error_packets_total 24532
+# HELP node_xfrm_fwd_hdr_error_packets_total Forward routing of a packet is not allowed
+# TYPE node_xfrm_fwd_hdr_error_packets_total counter
+node_xfrm_fwd_hdr_error_packets_total 6654
+# HELP node_xfrm_in_buffer_error_packets_total No buffer is left
+# TYPE node_xfrm_in_buffer_error_packets_total counter
+node_xfrm_in_buffer_error_packets_total 2
+# HELP node_xfrm_in_error_packets_total All errors not matched by other
+# TYPE node_xfrm_in_error_packets_total counter
+node_xfrm_in_error_packets_total 1
+# HELP node_xfrm_in_hdr_error_packets_total Header error
+# TYPE node_xfrm_in_hdr_error_packets_total counter
+node_xfrm_in_hdr_error_packets_total 4
+# HELP node_xfrm_in_no_pols_packets_total No policy is found for states e.g. Inbound SAs are correct but no SP is found
+# TYPE node_xfrm_in_no_pols_packets_total counter
+node_xfrm_in_no_pols_packets_total 65432
+# HELP node_xfrm_in_no_states_packets_total No state is found i.e. Either inbound SPI, address, or IPsec protocol at SA is wrong
+# TYPE node_xfrm_in_no_states_packets_total counter
+node_xfrm_in_no_states_packets_total 3
+# HELP node_xfrm_in_pol_block_packets_total Policy discards
+# TYPE node_xfrm_in_pol_block_packets_total counter
+node_xfrm_in_pol_block_packets_total 100
+# HELP node_xfrm_in_pol_error_packets_total Policy error
+# TYPE node_xfrm_in_pol_error_packets_total counter
+node_xfrm_in_pol_error_packets_total 10000
+# HELP node_xfrm_in_state_expired_packets_total State is expired
+# TYPE node_xfrm_in_state_expired_packets_total counter
+node_xfrm_in_state_expired_packets_total 7
+# HELP node_xfrm_in_state_invalid_packets_total State is invalid
+# TYPE node_xfrm_in_state_invalid_packets_total counter
+node_xfrm_in_state_invalid_packets_total 55555
+# HELP node_xfrm_in_state_mismatch_packets_total State has mismatch option e.g. UDP encapsulation type is mismatch
+# TYPE node_xfrm_in_state_mismatch_packets_total counter
+node_xfrm_in_state_mismatch_packets_total 23451
+# HELP node_xfrm_in_state_mode_error_packets_total Transformation mode specific error
+# TYPE node_xfrm_in_state_mode_error_packets_total counter
+node_xfrm_in_state_mode_error_packets_total 100
+# HELP node_xfrm_in_state_proto_error_packets_total Transformation protocol specific error e.g. SA key is wrong
+# TYPE node_xfrm_in_state_proto_error_packets_total counter
+node_xfrm_in_state_proto_error_packets_total 40
+# HELP node_xfrm_in_state_seq_error_packets_total Sequence error i.e. Sequence number is out of window
+# TYPE node_xfrm_in_state_seq_error_packets_total counter
+node_xfrm_in_state_seq_error_packets_total 6000
+# HELP node_xfrm_in_tmpl_mismatch_packets_total No matching template for states e.g. Inbound SAs are correct but SP rule is wrong
+# TYPE node_xfrm_in_tmpl_mismatch_packets_total counter
+node_xfrm_in_tmpl_mismatch_packets_total 51
+# HELP node_xfrm_out_bundle_check_error_packets_total Bundle check error
+# TYPE node_xfrm_out_bundle_check_error_packets_total counter
+node_xfrm_out_bundle_check_error_packets_total 555
+# HELP node_xfrm_out_bundle_gen_error_packets_total Bundle generation error
+# TYPE node_xfrm_out_bundle_gen_error_packets_total counter
+node_xfrm_out_bundle_gen_error_packets_total 43321
+# HELP node_xfrm_out_error_packets_total All errors which is not matched others
+# TYPE node_xfrm_out_error_packets_total counter
+node_xfrm_out_error_packets_total 1e+06
+# HELP node_xfrm_out_no_states_packets_total No state is found
+# TYPE node_xfrm_out_no_states_packets_total counter
+node_xfrm_out_no_states_packets_total 869
+# HELP node_xfrm_out_pol_block_packets_total Policy discards
+# TYPE node_xfrm_out_pol_block_packets_total counter
+node_xfrm_out_pol_block_packets_total 43456
+# HELP node_xfrm_out_pol_dead_packets_total Policy is dead
+# TYPE node_xfrm_out_pol_dead_packets_total counter
+node_xfrm_out_pol_dead_packets_total 7656
+# HELP node_xfrm_out_pol_error_packets_total Policy error
+# TYPE node_xfrm_out_pol_error_packets_total counter
+node_xfrm_out_pol_error_packets_total 1454
+# HELP node_xfrm_out_state_expired_packets_total State is expired
+# TYPE node_xfrm_out_state_expired_packets_total counter
+node_xfrm_out_state_expired_packets_total 565
+# HELP node_xfrm_out_state_invalid_packets_total State is invalid, perhaps expired
+# TYPE node_xfrm_out_state_invalid_packets_total counter
+node_xfrm_out_state_invalid_packets_total 28765
+# HELP node_xfrm_out_state_mode_error_packets_total Transformation mode specific error
+# TYPE node_xfrm_out_state_mode_error_packets_total counter
+node_xfrm_out_state_mode_error_packets_total 8
+# HELP node_xfrm_out_state_proto_error_packets_total Transformation protocol specific error
+# TYPE node_xfrm_out_state_proto_error_packets_total counter
+node_xfrm_out_state_proto_error_packets_total 4542
+# HELP node_xfrm_out_state_seq_error_packets_total Sequence error i.e. Sequence number overflow
+# TYPE node_xfrm_out_state_seq_error_packets_total counter
+node_xfrm_out_state_seq_error_packets_total 543
 # HELP node_xfs_allocation_btree_compares_total Number of allocation B-tree compares for a filesystem.
 # TYPE node_xfs_allocation_btree_compares_total counter
 node_xfs_allocation_btree_compares_total{device="sda1"} 0

collector/fixtures/proc/net/netstat

@@ -1,4 +1,4 @@
-TcpExt: SyncookiesSent SyncookiesRecv SyncookiesFailed EmbryonicRsts PruneCalled RcvPruned OfoPruned OutOfWindowIcmps LockDroppedIcmps ArpFilter TW TWRecycled TWKilled PAWSPassive PAWSActive PAWSEstab DelayedACKs DelayedACKLocked DelayedACKLost ListenOverflows ListenDrops TCPPrequeued TCPDirectCopyFromBacklog TCPDirectCopyFromPrequeue TCPPrequeueDropped TCPHPHits TCPHPHitsToUser TCPPureAcks TCPHPAcks TCPRenoRecovery TCPSackRecovery TCPSACKReneging TCPFACKReorder TCPSACKReorder TCPRenoReorder TCPTSReorder TCPFullUndo TCPPartialUndo TCPDSACKUndo TCPLossUndo TCPLoss TCPLostRetransmit TCPRenoFailures TCPSackFailures TCPLossFailures TCPFastRetrans TCPForwardRetrans TCPSlowStartRetrans TCPTimeouts TCPRenoRecoveryFail TCPSackRecoveryFail TCPSchedulerFailed TCPRcvCollapsed TCPDSACKOldSent TCPDSACKOfoSent TCPDSACKRecv TCPDSACKOfoRecv TCPAbortOnData TCPAbortOnClose TCPAbortOnMemory TCPAbortOnTimeout TCPAbortOnLinger TCPAbortFailed TCPMemoryPressures TCPSACKDiscard TCPDSACKIgnoredOld TCPDSACKIgnoredNoUndo TCPSpuriousRTOs TCPMD5NotFound TCPMD5Unexpected TCPSackShifted TCPSackMerged TCPSackShiftFallback TCPBacklogDrop TCPMinTTLDrop TCPDeferAcceptDrop IPReversePathFilter TCPTimeWaitOverflow TCPReqQFullDoCookies TCPReqQFullDrop TCPChallengeACK TCPSYNChallenge
+TcpExt: SyncookiesSent SyncookiesRecv SyncookiesFailed EmbryonicRsts PruneCalled RcvPruned OfoPruned OutOfWindowIcmps LockDroppedIcmps ArpFilter TW TWRecycled TWKilled PAWSPassive PAWSActive PAWSEstab DelayedACKs DelayedACKLocked DelayedACKLost ListenOverflows ListenDrops TCPPrequeued TCPDirectCopyFromBacklog TCPDirectCopyFromPrequeue TCPPrequeueDropped TCPHPHits TCPHPHitsToUser TCPPureAcks TCPHPAcks TCPRenoRecovery TCPSackRecovery TCPSACKReneging TCPFACKReorder TCPSACKReorder TCPRenoReorder TCPTSReorder TCPFullUndo TCPPartialUndo TCPDSACKUndo TCPLossUndo TCPLoss TCPLostRetransmit TCPRenoFailures TCPSackFailures TCPLossFailures TCPFastRetrans TCPForwardRetrans TCPSlowStartRetrans TCPTimeouts TCPRenoRecoveryFail TCPSackRecoveryFail TCPSchedulerFailed TCPRcvCollapsed TCPDSACKOldSent TCPDSACKOfoSent TCPDSACKRecv TCPDSACKOfoRecv TCPAbortOnData TCPAbortOnClose TCPAbortOnMemory TCPAbortOnTimeout TCPAbortOnLinger TCPAbortFailed TCPMemoryPressures TCPSACKDiscard TCPDSACKIgnoredOld TCPDSACKIgnoredNoUndo TCPSpuriousRTOs TCPMD5NotFound TCPMD5Unexpected TCPSackShifted TCPSackMerged TCPSackShiftFallback TCPBacklogDrop TCPMinTTLDrop TCPDeferAcceptDrop IPReversePathFilter TCPTimeWaitOverflow TCPReqQFullDoCookies TCPReqQFullDrop TCPChallengeACK TCPSYNChallenge TCPOFOQueue
-TcpExt: 0 0 2 0 0 0 0 0 0 0 388812 0 0 0 0 6 102471 17 9 0 0 80568 0 168808 0 4471289 26 1433940 3744565 0 1 0 0 0 0 0 0 0 0 48 0 0 0 1 0 1 0 1 115 0 0 0 0 9 0 5 0 41 4 0 0 0 0 0 0 0 1 0 0 0 0 2 5 0 0 0 0 0 0 0 2 2
+TcpExt: 0 0 2 0 0 0 0 0 0 0 388812 0 0 0 0 6 102471 17 9 0 0 80568 0 168808 0 4471289 26 1433940 3744565 0 1 0 0 0 0 0 0 0 0 48 0 0 0 1 0 1 0 1 115 0 0 0 0 9 0 5 0 41 4 0 0 0 0 0 0 0 1 0 0 0 0 2 5 0 0 0 0 0 0 0 2 2 42
 IpExt: InNoRoutes InTruncatedPkts InMcastPkts OutMcastPkts InBcastPkts OutBcastPkts InOctets OutOctets InMcastOctets OutMcastOctets InBcastOctets OutBcastOctets
 IpExt: 0 0 0 0 0 0 6286396970 2786264347 0 0 0 0

collector/fixtures/proc/net/rpc/nfsd

@@ -9,3 +9,4 @@ proc2 18 2 69 0 0 4410 0 0 0 0 0 0 0 0 0 0 0 99 2
 proc3 22 2 112 0 2719 111 0 0 0 0 0 0 0 0 0 0 0 27 216 0 2 1 0
 proc4 2 2 10853
 proc4ops 72 0 0 0 1098 2 0 0 0 0 8179 5896 0 0 0 0 5900 0 0 2 0 2 0 9609 0 2 150 1272 0 0 0 1236 0 0 0 0 3 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
+wdeleg_getattr 15

collector/fixtures/proc/net/xfrm_stat

@@ -0,0 +1,28 @@
+XfrmInError                     1
+XfrmInBufferError               2
+XfrmInHdrError                  4
+XfrmInNoStates                  3
+XfrmInStateProtoError           40
+XfrmInStateModeError            100
+XfrmInStateSeqError             6000
+XfrmInStateExpired              7
+XfrmInStateMismatch             23451
+XfrmInStateInvalid              55555
+XfrmInTmplMismatch              51
+XfrmInNoPols                    65432
+XfrmInPolBlock                  100
+XfrmInPolError                  10000
+XfrmOutError                    1000000
+XfrmOutBundleGenError           43321
+XfrmOutBundleCheckError         555
+XfrmOutNoStates                 869
+XfrmOutStateProtoError          4542
+XfrmOutStateModeError           8
+XfrmOutStateSeqError            543
+XfrmOutStateExpired             565
+XfrmOutPolBlock                 43456
+XfrmOutPolDead                  7656
+XfrmOutPolError                 1454
+XfrmFwdHdrError                 6654
+XfrmOutStateInvalid             28765
+XfrmAcquireError                24532

collector/fixtures/sys.ttar

@@ -1717,6 +1717,75 @@ SymlinkTo: ../../devices/virtual/thermal/cooling_device0
 Path: sys/class/thermal/thermal_zone0
 SymlinkTo: ../../devices/virtual/thermal/thermal_zone0
 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Directory: sys/class/watchdog
+Mode: 775
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Directory: sys/class/watchdog/watchdog0
+Mode: 775
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/watchdog/watchdog0/access_cs0
+Lines: 1
+0EOF
+Mode: 644
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/watchdog/watchdog0/bootstatus
+Lines: 1
+1EOF
+Mode: 444
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/watchdog/watchdog0/fw_version
+Lines: 1
+2EOF
+Mode: 444
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/watchdog/watchdog0/identity
+Lines: 1
+Software WatchdogEOF
+Mode: 444
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/watchdog/watchdog0/nowayout
+Lines: 1
+0EOF
+Mode: 644
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/watchdog/watchdog0/options
+Lines: 1
+0x8380EOF
+Mode: 444
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/watchdog/watchdog0/pretimeout
+Lines: 1
+120EOF
+Mode: 444
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/watchdog/watchdog0/pretimeout_governor
+Lines: 1
+noopEOF
+Mode: 644
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/watchdog/watchdog0/state
+Lines: 1
+activeEOF
+Mode: 444
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/watchdog/watchdog0/status
+Lines: 1
+0x8000EOF
+Mode: 444
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/watchdog/watchdog0/timeleft
+Lines: 1
+300EOF
+Mode: 444
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/watchdog/watchdog0/timeout
+Lines: 1
+60EOF
+Mode: 444
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Directory: sys/class/watchdog/watchdog1
+Mode: 775
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 Directory: sys/devices
 Mode: 755
 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

collector/hwmon_linux.go

@@ -444,9 +444,13 @@ func (c *hwMonCollector) Update(ch chan<- prometheus.Metric) error {
 		return err
 	}
 
+	var lastErr error
 	for _, hwDir := range hwmonFiles {
 		hwmonXPathName := filepath.Join(hwmonPathName, hwDir.Name())
-		fileInfo, _ := os.Lstat(hwmonXPathName)
+		fileInfo, err := os.Lstat(hwmonXPathName)
+		if err != nil {
+			continue
+		}
 
 		if fileInfo.Mode()&os.ModeSymlink > 0 {
 			fileInfo, err = os.Stat(hwmonXPathName)
@@ -459,10 +463,10 @@ func (c *hwMonCollector) Update(ch chan<- prometheus.Metric) error {
 			continue
 		}
 
-		if lastErr := c.updateHwmon(ch, hwmonXPathName); lastErr != nil {
+		if err = c.updateHwmon(ch, hwmonXPathName); err != nil {
-			err = lastErr
+			lastErr = err
 		}
 	}
 
-	return err
+	return lastErr
 }

collector/memory_bsd.go

@@ -69,21 +69,18 @@ func NewMemoryCollector(logger log.Logger) (Collector, error) {
 				description: "Recently used by userland",
 				mib:         "vm.stats.vm.v_active_count",
 				conversion:  fromPage,
-				labels:      nil,
 			},
 			{
 				name:        "inactive_bytes",
 				description: "Not recently used by userland",
 				mib:         "vm.stats.vm.v_inactive_count",
 				conversion:  fromPage,
-				labels:      nil,
 			},
 			{
 				name:        "wired_bytes",
 				description: "Locked in memory by kernel, mlock, etc",
 				mib:         "vm.stats.vm.v_wire_count",
 				conversion:  fromPage,
-				labels:      nil,
 			},
 			{
 				name:        "user_wired_bytes",
@@ -91,49 +88,42 @@ func NewMemoryCollector(logger log.Logger) (Collector, error) {
 				mib:         "vm.stats.vm.v_user_wire_count",
 				conversion:  fromPage,
 				dataType:    bsdSysctlTypeCLong,
-				labels:      nil,
 			},
 			{
 				name:        "cache_bytes",
 				description: "Almost free, backed by swap or files, available for re-allocation",
 				mib:         "vm.stats.vm.v_cache_count",
 				conversion:  fromPage,
-				labels:      nil,
 			},
 			{
 				name:        "buffer_bytes",
 				description: "Disk IO Cache entries for non ZFS filesystems, only usable by kernel",
 				mib:         "vfs.bufspace",
 				dataType:    bsdSysctlTypeCLong,
-				labels:      nil,
 			},
 			{
 				name:        "free_bytes",
 				description: "Unallocated, available for allocation",
 				mib:         "vm.stats.vm.v_free_count",
 				conversion:  fromPage,
-				labels:      nil,
 			},
 			{
 				name:        "laundry_bytes",
 				description: "Dirty not recently used by userland",
 				mib:         "vm.stats.vm.v_laundry_count",
 				conversion:  fromPage,
-				labels:      nil,
 			},
 			{
 				name:        "size_bytes",
 				description: "Total physical memory size",
 				mib:         "vm.stats.vm.v_page_count",
 				conversion:  fromPage,
-				labels:      nil,
 			},
 			{
 				name:        "swap_size_bytes",
 				description: "Total swap memory size",
 				mib:         mibSwapTotal,
 				dataType:    bsdSysctlTypeUint64,
-				labels:      nil,
 			},
 			// Descriptions via: top(1)
 			{
@@ -142,7 +132,6 @@ func NewMemoryCollector(logger log.Logger) (Collector, error) {
 				mib:         "vm.stats.vm.v_swappgsin",
 				valueType:   prometheus.CounterValue,
 				conversion:  fromPage,
-				labels:      nil,
 			},
 			{
 				name:        "swap_out_bytes_total",
@@ -150,7 +139,6 @@ func NewMemoryCollector(logger log.Logger) (Collector, error) {
 				mib:         "vm.stats.vm.v_swappgsout",
 				valueType:   prometheus.CounterValue,
 				conversion:  fromPage,
-				labels:      nil,
 			},
 		},
 	}, nil

collector/netisr_freebsd.go

@@ -45,7 +45,6 @@ func NewNetisrCollector(logger log.Logger) (Collector, error) {
 				mib:         "net.isr.numthreads",
 				dataType:    bsdSysctlTypeUint32,
 				valueType:   prometheus.GaugeValue,
-				labels:      nil,
 			},
 			{
 				name:        "maxprot",
@@ -53,7 +52,6 @@ func NewNetisrCollector(logger log.Logger) (Collector, error) {
 				mib:         "net.isr.maxprot",
 				dataType:    bsdSysctlTypeUint32,
 				valueType:   prometheus.GaugeValue,
-				labels:      nil,
 			},
 			{
 				name:        "defaultqlimit",
@@ -61,7 +59,6 @@ func NewNetisrCollector(logger log.Logger) (Collector, error) {
 				mib:         "net.isr.defaultqlimit",
 				dataType:    bsdSysctlTypeUint32,
 				valueType:   prometheus.GaugeValue,
-				labels:      nil,
 			},
 			{
 				name:        "maxqlimit",
@@ -69,7 +66,6 @@ func NewNetisrCollector(logger log.Logger) (Collector, error) {
 				mib:         "net.isr.maxqlimit",
 				dataType:    bsdSysctlTypeUint32,
 				valueType:   prometheus.GaugeValue,
-				labels:      nil,
 			},
 			{
 				name:        "bindthreads",
@@ -77,7 +73,6 @@ func NewNetisrCollector(logger log.Logger) (Collector, error) {
 				mib:         "net.isr.bindthreads",
 				dataType:    bsdSysctlTypeUint32,
 				valueType:   prometheus.GaugeValue,
-				labels:      nil,
 			},
 			{
 				name:        "maxthreads",
@@ -85,7 +80,6 @@ func NewNetisrCollector(logger log.Logger) (Collector, error) {
 				mib:         "net.isr.maxthreads",
 				dataType:    bsdSysctlTypeUint32,
 				valueType:   prometheus.GaugeValue,
-				labels:      nil,
 			},
 		},
 		logger: logger,

collector/netstat_linux.go

@@ -36,7 +36,7 @@ const (
 )
 
 var (
-	netStatFields = kingpin.Flag("collector.netstat.fields", "Regexp of fields to return for netstat collector.").Default("^(.*_(InErrors|InErrs)|Ip_Forwarding|Ip(6|Ext)_(InOctets|OutOctets)|Icmp6?_(InMsgs|OutMsgs)|TcpExt_(Listen.*|Syncookies.*|TCPSynRetrans|TCPTimeouts)|Tcp_(ActiveOpens|InSegs|OutSegs|OutRsts|PassiveOpens|RetransSegs|CurrEstab)|Udp6?_(InDatagrams|OutDatagrams|NoPorts|RcvbufErrors|SndbufErrors))$").String()
+	netStatFields = kingpin.Flag("collector.netstat.fields", "Regexp of fields to return for netstat collector.").Default("^(.*_(InErrors|InErrs)|Ip_Forwarding|Ip(6|Ext)_(InOctets|OutOctets)|Icmp6?_(InMsgs|OutMsgs)|TcpExt_(Listen.*|Syncookies.*|TCPSynRetrans|TCPTimeouts|TCPOFOQueue)|Tcp_(ActiveOpens|InSegs|OutSegs|OutRsts|PassiveOpens|RetransSegs|CurrEstab)|Udp6?_(InDatagrams|OutDatagrams|NoPorts|RcvbufErrors|SndbufErrors))$").String()
 )
 
 type netStatCollector struct {

collector/nfsd_linux.go

@@ -82,6 +82,8 @@ func (c *nfsdCollector) Update(ch chan<- prometheus.Metric) error {
 	c.updateNFSdRequestsv2Stats(ch, &stats.V2Stats)
 	c.updateNFSdRequestsv3Stats(ch, &stats.V3Stats)
 	c.updateNFSdRequestsv4Stats(ch, &stats.V4Ops)
+	ch <- prometheus.MustNewConstMetric(c.requestsDesc, prometheus.CounterValue,
+		float64(stats.WdelegGetattr), "4", "WdelegGetattr")
 
 	return nil
 }

collector/sysctl_bsd.go

@@ -59,9 +59,6 @@ type bsdSysctl struct {
 
 	// Post-retrieval conversion hooks
 	conversion func(float64) float64
-
-	// Prometheus labels
-	labels prometheus.Labels
 }
 
 func (b bsdSysctl) Value() (float64, error) {

collector/timex.go

@@ -38,6 +38,9 @@ const (
 	// 1 second in
 	nanoSeconds  = 1000000000
 	microSeconds = 1000000
+
+	// See NOTES in adjtimex(2).
+	ppm16frac = 1000000.0 * 65536.0
 )
 
 type timexCollector struct {
@@ -183,8 +186,6 @@ func (c *timexCollector) Update(ch chan<- prometheus.Metric) error {
 	} else {
 		divisor = microSeconds
 	}
-	// See NOTES in adjtimex(2).
-	const ppm16frac = 1000000.0 * 65536.0
 
 	ch <- c.syncStatus.mustNewConstMetric(syncStatus)
 	ch <- c.offset.mustNewConstMetric(float64(timex.Offset) / divisor)

collector/watchdog.go

@@ -0,0 +1,140 @@
+// Copyright 2023 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux && !nowatchdog
+// +build linux,!nowatchdog
+
+package collector
+
+import (
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/procfs/sysfs"
+)
+
+type watchdogCollector struct {
+	fs     sysfs.FS
+	logger log.Logger
+}
+
+func init() {
+	registerCollector("watchdog", defaultEnabled, NewWatchdogCollector)
+}
+
+// NewWatchdogCollector returns a new Collector exposing watchdog stats.
+func NewWatchdogCollector(logger log.Logger) (Collector, error) {
+	fs, err := sysfs.NewFS(*sysPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open procfs: %w", err)
+	}
+
+	return &watchdogCollector{
+		fs:     fs,
+		logger: logger,
+	}, nil
+}
+
+var (
+	watchdogBootstatusDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "watchdog", "bootstatus"),
+		"Value of /sys/class/watchdog/<watchdog>/bootstatus",
+		[]string{"name"}, nil,
+	)
+	watchdogFwVersionDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "watchdog", "fw_version"),
+		"Value of /sys/class/watchdog/<watchdog>/fw_version",
+		[]string{"name"}, nil,
+	)
+	watchdogNowayoutDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "watchdog", "nowayout"),
+		"Value of /sys/class/watchdog/<watchdog>/nowayout",
+		[]string{"name"}, nil,
+	)
+	watchdogTimeleftDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "watchdog", "timeleft_seconds"),
+		"Value of /sys/class/watchdog/<watchdog>/timeleft",
+		[]string{"name"}, nil,
+	)
+	watchdogTimeoutDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "watchdog", "timeout_seconds"),
+		"Value of /sys/class/watchdog/<watchdog>/timeout",
+		[]string{"name"}, nil,
+	)
+	watchdogPretimeoutDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "watchdog", "pretimeout_seconds"),
+		"Value of /sys/class/watchdog/<watchdog>/pretimeout",
+		[]string{"name"}, nil,
+	)
+	watchdogAccessCs0Desc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "watchdog", "access_cs0"),
+		"Value of /sys/class/watchdog/<watchdog>/access_cs0",
+		[]string{"name"}, nil,
+	)
+	watchdogInfoDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "watchdog", "info"),
+		"Info of /sys/class/watchdog/<watchdog>",
+		[]string{"name", "options", "identity", "state", "status", "pretimeout_governor"}, nil,
+	)
+)
+
+func toLabelValue(ptr *string) string {
+	if ptr == nil {
+		return ""
+	}
+	return *ptr
+}
+
+func (c *watchdogCollector) Update(ch chan<- prometheus.Metric) error {
+	watchdogClass, err := c.fs.WatchdogClass()
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) || errors.Is(err, os.ErrPermission) || errors.Is(err, os.ErrInvalid) {
+			level.Debug(c.logger).Log("msg", "Could not read watchdog stats", "err", err)
+			return ErrNoData
+		}
+		return err
+	}
+
+	for _, wd := range watchdogClass {
+		if wd.Bootstatus != nil {
+			ch <- prometheus.MustNewConstMetric(watchdogBootstatusDesc, prometheus.GaugeValue, float64(*wd.Bootstatus), wd.Name)
+		}
+		if wd.FwVersion != nil {
+			ch <- prometheus.MustNewConstMetric(watchdogFwVersionDesc, prometheus.GaugeValue, float64(*wd.FwVersion), wd.Name)
+		}
+		if wd.Nowayout != nil {
+			ch <- prometheus.MustNewConstMetric(watchdogNowayoutDesc, prometheus.GaugeValue, float64(*wd.Nowayout), wd.Name)
+		}
+		if wd.Timeleft != nil {
+			ch <- prometheus.MustNewConstMetric(watchdogTimeleftDesc, prometheus.GaugeValue, float64(*wd.Timeleft), wd.Name)
+		}
+		if wd.Timeout != nil {
+			ch <- prometheus.MustNewConstMetric(watchdogTimeoutDesc, prometheus.GaugeValue, float64(*wd.Timeout), wd.Name)
+		}
+		if wd.Pretimeout != nil {
+			ch <- prometheus.MustNewConstMetric(watchdogPretimeoutDesc, prometheus.GaugeValue, float64(*wd.Pretimeout), wd.Name)
+		}
+		if wd.AccessCs0 != nil {
+			ch <- prometheus.MustNewConstMetric(watchdogAccessCs0Desc, prometheus.GaugeValue, float64(*wd.AccessCs0), wd.Name)
+		}
+
+		ch <- prometheus.MustNewConstMetric(watchdogInfoDesc, prometheus.GaugeValue, 1.0,
+			wd.Name, toLabelValue(wd.Options), toLabelValue(wd.Identity), toLabelValue(wd.State), toLabelValue(wd.Status), toLabelValue(wd.PretimeoutGovernor))
+	}
+
+	return nil
+}

collector/watchdog_test.go

@@ -0,0 +1,92 @@
+// Copyright 2023 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file ewcept in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !nowatchdog
+// +build !nowatchdog
+
+package collector
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+type testWatchdogCollector struct {
+	wc Collector
+}
+
+func (c testWatchdogCollector) Collect(ch chan<- prometheus.Metric) {
+	c.wc.Update(ch)
+}
+
+func (c testWatchdogCollector) Describe(ch chan<- *prometheus.Desc) {
+	prometheus.DescribeByCollect(c, ch)
+}
+
+func TestWatchdogStats(t *testing.T) {
+	testcase := `# HELP node_watchdog_access_cs0 Value of /sys/class/watchdog/<watchdog>/access_cs0
+	# TYPE node_watchdog_access_cs0 gauge
+	node_watchdog_access_cs0{name="watchdog0"} 0
+	# HELP node_watchdog_bootstatus Value of /sys/class/watchdog/<watchdog>/bootstatus
+	# TYPE node_watchdog_bootstatus gauge
+	node_watchdog_bootstatus{name="watchdog0"} 1
+	# HELP node_watchdog_fw_version Value of /sys/class/watchdog/<watchdog>/fw_version
+	# TYPE node_watchdog_fw_version gauge
+	node_watchdog_fw_version{name="watchdog0"} 2
+	# HELP node_watchdog_info Info of /sys/class/watchdog/<watchdog>
+	# TYPE node_watchdog_info gauge
+	node_watchdog_info{identity="",name="watchdog1",options="",pretimeout_governor="",state="",status=""} 1
+	node_watchdog_info{identity="Software Watchdog",name="watchdog0",options="0x8380",pretimeout_governor="noop",state="active",status="0x8000"} 1
+	# HELP node_watchdog_nowayout Value of /sys/class/watchdog/<watchdog>/nowayout
+	# TYPE node_watchdog_nowayout gauge
+	node_watchdog_nowayout{name="watchdog0"} 0
+	# HELP node_watchdog_pretimeout_seconds Value of /sys/class/watchdog/<watchdog>/pretimeout
+	# TYPE node_watchdog_pretimeout_seconds gauge
+	node_watchdog_pretimeout_seconds{name="watchdog0"} 120
+	# HELP node_watchdog_timeleft_seconds Value of /sys/class/watchdog/<watchdog>/timeleft
+	# TYPE node_watchdog_timeleft_seconds gauge
+	node_watchdog_timeleft_seconds{name="watchdog0"} 300
+	# HELP node_watchdog_timeout_seconds Value of /sys/class/watchdog/<watchdog>/timeout
+	# TYPE node_watchdog_timeout_seconds gauge
+	node_watchdog_timeout_seconds{name="watchdog0"} 60
+	`
+	*sysPath = "fixtures/sys"
+
+	logger := log.NewLogfmtLogger(os.Stderr)
+	c, err := NewWatchdogCollector(logger)
+	if err != nil {
+		t.Fatal(err)
+	}
+	reg := prometheus.NewRegistry()
+	reg.MustRegister(&testWatchdogCollector{wc: c})
+
+	sink := make(chan prometheus.Metric)
+	go func() {
+		err = c.Update(sink)
+		if err != nil {
+			panic(fmt.Errorf("failed to update collector: %s", err))
+		}
+		close(sink)
+	}()
+
+	err = testutil.GatherAndCompare(reg, strings.NewReader(testcase))
+	if err != nil {
+		t.Fatal(err)
+	}
+}

collector/xfrm.go

@@ -0,0 +1,228 @@
+// Copyright 2023 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !noxfrm
+// +build !noxfrm
+
+package collector
+
+import (
+	"fmt"
+
+	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/procfs"
+)
+
+type xfrmCollector struct {
+	fs     procfs.FS
+	logger log.Logger
+}
+
+func init() {
+	registerCollector("xfrm", defaultDisabled, NewXfrmCollector)
+}
+
+// NewXfrmCollector returns a new Collector exposing XFRM stats.
+func NewXfrmCollector(logger log.Logger) (Collector, error) {
+	fs, err := procfs.NewFS(*procPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open procfs: %w", err)
+	}
+
+	return &xfrmCollector{
+		fs:     fs,
+		logger: logger,
+	}, nil
+}
+
+var (
+	xfrmInErrorDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "in_error_packets_total"),
+		"All errors not matched by other",
+		nil, nil,
+	)
+	xfrmInBufferErrorDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "in_buffer_error_packets_total"),
+		"No buffer is left",
+		nil, nil,
+	)
+	xfrmInHdrErrorDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "in_hdr_error_packets_total"),
+		"Header error",
+		nil, nil,
+	)
+	xfrmInNoStatesDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "in_no_states_packets_total"),
+		"No state is found i.e. Either inbound SPI, address, or IPsec protocol at SA is wrong",
+		nil, nil,
+	)
+	xfrmInStateProtoErrorDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "in_state_proto_error_packets_total"),
+		"Transformation protocol specific error e.g. SA key is wrong",
+		nil, nil,
+	)
+	xfrmInStateModeErrorDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "in_state_mode_error_packets_total"),
+		"Transformation mode specific error",
+		nil, nil,
+	)
+	xfrmInStateSeqErrorDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "in_state_seq_error_packets_total"),
+		"Sequence error i.e. Sequence number is out of window",
+		nil, nil,
+	)
+	xfrmInStateExpiredDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "in_state_expired_packets_total"),
+		"State is expired",
+		nil, nil,
+	)
+	xfrmInStateMismatchDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "in_state_mismatch_packets_total"),
+		"State has mismatch option e.g. UDP encapsulation type is mismatch",
+		nil, nil,
+	)
+	xfrmInStateInvalidDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "in_state_invalid_packets_total"),
+		"State is invalid",
+		nil, nil,
+	)
+	xfrmInTmplMismatchDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "in_tmpl_mismatch_packets_total"),
+		"No matching template for states e.g. Inbound SAs are correct but SP rule is wrong",
+		nil, nil,
+	)
+	xfrmInNoPolsDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "in_no_pols_packets_total"),
+		"No policy is found for states e.g. Inbound SAs are correct but no SP is found",
+		nil, nil,
+	)
+	xfrmInPolBlockDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "in_pol_block_packets_total"),
+		"Policy discards",
+		nil, nil,
+	)
+	xfrmInPolErrorDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "in_pol_error_packets_total"),
+		"Policy error",
+		nil, nil,
+	)
+	xfrmOutErrorDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "out_error_packets_total"),
+		"All errors which is not matched others",
+		nil, nil,
+	)
+	xfrmOutBundleGenErrorDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "out_bundle_gen_error_packets_total"),
+		"Bundle generation error",
+		nil, nil,
+	)
+	xfrmOutBundleCheckErrorDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "out_bundle_check_error_packets_total"),
+		"Bundle check error",
+		nil, nil,
+	)
+	xfrmOutNoStatesDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "out_no_states_packets_total"),
+		"No state is found",
+		nil, nil,
+	)
+	xfrmOutStateProtoErrorDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "out_state_proto_error_packets_total"),
+		"Transformation protocol specific error",
+		nil, nil,
+	)
+	xfrmOutStateModeErrorDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "out_state_mode_error_packets_total"),
+		"Transformation mode specific error",
+		nil, nil,
+	)
+	xfrmOutStateSeqErrorDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "out_state_seq_error_packets_total"),
+		"Sequence error i.e. Sequence number overflow",
+		nil, nil,
+	)
+	xfrmOutStateExpiredDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "out_state_expired_packets_total"),
+		"State is expired",
+		nil, nil,
+	)
+	xfrmOutPolBlockDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "out_pol_block_packets_total"),
+		"Policy discards",
+		nil, nil,
+	)
+	xfrmOutPolDeadDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "out_pol_dead_packets_total"),
+		"Policy is dead",
+		nil, nil,
+	)
+	xfrmOutPolErrorDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "out_pol_error_packets_total"),
+		"Policy error",
+		nil, nil,
+	)
+	xfrmFwdHdrErrorDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "fwd_hdr_error_packets_total"),
+		"Forward routing of a packet is not allowed",
+		nil, nil,
+	)
+	xfrmOutStateInvalidDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "out_state_invalid_packets_total"),
+		"State is invalid, perhaps expired",
+		nil, nil,
+	)
+	xfrmAcquireErrorDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "xfrm", "acquire_error_packets_total"),
+		"State hasn’t been fully acquired before use",
+		nil, nil,
+	)
+)
+
+func (c *xfrmCollector) Update(ch chan<- prometheus.Metric) error {
+	stat, err := c.fs.NewXfrmStat()
+	if err != nil {
+		return err
+	}
+
+	ch <- prometheus.MustNewConstMetric(xfrmInErrorDesc, prometheus.CounterValue, float64(stat.XfrmInError))
+	ch <- prometheus.MustNewConstMetric(xfrmInBufferErrorDesc, prometheus.CounterValue, float64(stat.XfrmInBufferError))
+	ch <- prometheus.MustNewConstMetric(xfrmInHdrErrorDesc, prometheus.CounterValue, float64(stat.XfrmInHdrError))
+	ch <- prometheus.MustNewConstMetric(xfrmInNoStatesDesc, prometheus.CounterValue, float64(stat.XfrmInNoStates))
+	ch <- prometheus.MustNewConstMetric(xfrmInStateProtoErrorDesc, prometheus.CounterValue, float64(stat.XfrmInStateProtoError))
+	ch <- prometheus.MustNewConstMetric(xfrmInStateModeErrorDesc, prometheus.CounterValue, float64(stat.XfrmInStateModeError))
+	ch <- prometheus.MustNewConstMetric(xfrmInStateSeqErrorDesc, prometheus.CounterValue, float64(stat.XfrmInStateSeqError))
+	ch <- prometheus.MustNewConstMetric(xfrmInStateExpiredDesc, prometheus.CounterValue, float64(stat.XfrmInStateExpired))
+	ch <- prometheus.MustNewConstMetric(xfrmInStateMismatchDesc, prometheus.CounterValue, float64(stat.XfrmInStateMismatch))
+	ch <- prometheus.MustNewConstMetric(xfrmInStateInvalidDesc, prometheus.CounterValue, float64(stat.XfrmInStateInvalid))
+	ch <- prometheus.MustNewConstMetric(xfrmInTmplMismatchDesc, prometheus.CounterValue, float64(stat.XfrmInTmplMismatch))
+	ch <- prometheus.MustNewConstMetric(xfrmInNoPolsDesc, prometheus.CounterValue, float64(stat.XfrmInNoPols))
+	ch <- prometheus.MustNewConstMetric(xfrmInPolBlockDesc, prometheus.CounterValue, float64(stat.XfrmInPolBlock))
+	ch <- prometheus.MustNewConstMetric(xfrmInPolErrorDesc, prometheus.CounterValue, float64(stat.XfrmInPolError))
+	ch <- prometheus.MustNewConstMetric(xfrmOutErrorDesc, prometheus.CounterValue, float64(stat.XfrmOutError))
+	ch <- prometheus.MustNewConstMetric(xfrmOutBundleGenErrorDesc, prometheus.CounterValue, float64(stat.XfrmOutBundleGenError))
+	ch <- prometheus.MustNewConstMetric(xfrmOutBundleCheckErrorDesc, prometheus.CounterValue, float64(stat.XfrmOutBundleCheckError))
+	ch <- prometheus.MustNewConstMetric(xfrmOutNoStatesDesc, prometheus.CounterValue, float64(stat.XfrmOutNoStates))
+	ch <- prometheus.MustNewConstMetric(xfrmOutStateProtoErrorDesc, prometheus.CounterValue, float64(stat.XfrmOutStateProtoError))
+	ch <- prometheus.MustNewConstMetric(xfrmOutStateModeErrorDesc, prometheus.CounterValue, float64(stat.XfrmOutStateModeError))
+	ch <- prometheus.MustNewConstMetric(xfrmOutStateSeqErrorDesc, prometheus.CounterValue, float64(stat.XfrmOutStateSeqError))
+	ch <- prometheus.MustNewConstMetric(xfrmOutStateExpiredDesc, prometheus.CounterValue, float64(stat.XfrmOutStateExpired))
+	ch <- prometheus.MustNewConstMetric(xfrmOutPolBlockDesc, prometheus.CounterValue, float64(stat.XfrmOutPolBlock))
+	ch <- prometheus.MustNewConstMetric(xfrmOutPolDeadDesc, prometheus.CounterValue, float64(stat.XfrmOutPolDead))
+	ch <- prometheus.MustNewConstMetric(xfrmOutPolErrorDesc, prometheus.CounterValue, float64(stat.XfrmOutPolError))
+	ch <- prometheus.MustNewConstMetric(xfrmFwdHdrErrorDesc, prometheus.CounterValue, float64(stat.XfrmFwdHdrError))
+	ch <- prometheus.MustNewConstMetric(xfrmOutStateInvalidDesc, prometheus.CounterValue, float64(stat.XfrmOutStateInvalid))
+	ch <- prometheus.MustNewConstMetric(xfrmAcquireErrorDesc, prometheus.CounterValue, float64(stat.XfrmAcquireError))
+
+	return err
+}

collector/xfrm_test.go

@@ -0,0 +1,151 @@
+// Copyright 2023 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !noxfrm
+// +build !noxfrm
+
+package collector
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+type testXfrmCollector struct {
+	xc Collector
+}
+
+func (c testXfrmCollector) Collect(ch chan<- prometheus.Metric) {
+	c.xc.Update(ch)
+}
+
+func (c testXfrmCollector) Describe(ch chan<- *prometheus.Desc) {
+	prometheus.DescribeByCollect(c, ch)
+}
+
+func TestXfrmStats(t *testing.T) {
+	testcase := `# HELP node_xfrm_acquire_error_packets_total State hasn’t been fully acquired before use
+	# TYPE node_xfrm_acquire_error_packets_total counter
+	node_xfrm_acquire_error_packets_total 24532
+	# HELP node_xfrm_fwd_hdr_error_packets_total Forward routing of a packet is not allowed
+	# TYPE node_xfrm_fwd_hdr_error_packets_total counter
+	node_xfrm_fwd_hdr_error_packets_total 6654
+	# HELP node_xfrm_in_buffer_error_packets_total No buffer is left
+	# TYPE node_xfrm_in_buffer_error_packets_total counter
+	node_xfrm_in_buffer_error_packets_total 2
+	# HELP node_xfrm_in_error_packets_total All errors not matched by other
+	# TYPE node_xfrm_in_error_packets_total counter
+	node_xfrm_in_error_packets_total 1
+	# HELP node_xfrm_in_hdr_error_packets_total Header error
+	# TYPE node_xfrm_in_hdr_error_packets_total counter
+	node_xfrm_in_hdr_error_packets_total 4
+	# HELP node_xfrm_in_no_pols_packets_total No policy is found for states e.g. Inbound SAs are correct but no SP is found
+	# TYPE node_xfrm_in_no_pols_packets_total counter
+	node_xfrm_in_no_pols_packets_total 65432
+	# HELP node_xfrm_in_no_states_packets_total No state is found i.e. Either inbound SPI, address, or IPsec protocol at SA is wrong
+	# TYPE node_xfrm_in_no_states_packets_total counter
+	node_xfrm_in_no_states_packets_total 3
+	# HELP node_xfrm_in_pol_block_packets_total Policy discards
+	# TYPE node_xfrm_in_pol_block_packets_total counter
+	node_xfrm_in_pol_block_packets_total 100
+	# HELP node_xfrm_in_pol_error_packets_total Policy error
+	# TYPE node_xfrm_in_pol_error_packets_total counter
+	node_xfrm_in_pol_error_packets_total 10000
+	# HELP node_xfrm_in_state_expired_packets_total State is expired
+	# TYPE node_xfrm_in_state_expired_packets_total counter
+	node_xfrm_in_state_expired_packets_total 7
+	# HELP node_xfrm_in_state_invalid_packets_total State is invalid
+	# TYPE node_xfrm_in_state_invalid_packets_total counter
+	node_xfrm_in_state_invalid_packets_total 55555
+	# HELP node_xfrm_in_state_mismatch_packets_total State has mismatch option e.g. UDP encapsulation type is mismatch
+	# TYPE node_xfrm_in_state_mismatch_packets_total counter
+	node_xfrm_in_state_mismatch_packets_total 23451
+	# HELP node_xfrm_in_state_mode_error_packets_total Transformation mode specific error
+	# TYPE node_xfrm_in_state_mode_error_packets_total counter
+	node_xfrm_in_state_mode_error_packets_total 100
+	# HELP node_xfrm_in_state_proto_error_packets_total Transformation protocol specific error e.g. SA key is wrong
+	# TYPE node_xfrm_in_state_proto_error_packets_total counter
+	node_xfrm_in_state_proto_error_packets_total 40
+	# HELP node_xfrm_in_state_seq_error_packets_total Sequence error i.e. Sequence number is out of window
+	# TYPE node_xfrm_in_state_seq_error_packets_total counter
+	node_xfrm_in_state_seq_error_packets_total 6000
+	# HELP node_xfrm_in_tmpl_mismatch_packets_total No matching template for states e.g. Inbound SAs are correct but SP rule is wrong
+	# TYPE node_xfrm_in_tmpl_mismatch_packets_total counter
+	node_xfrm_in_tmpl_mismatch_packets_total 51
+	# HELP node_xfrm_out_bundle_check_error_packets_total Bundle check error
+	# TYPE node_xfrm_out_bundle_check_error_packets_total counter
+	node_xfrm_out_bundle_check_error_packets_total 555
+	# HELP node_xfrm_out_bundle_gen_error_packets_total Bundle generation error
+	# TYPE node_xfrm_out_bundle_gen_error_packets_total counter
+	node_xfrm_out_bundle_gen_error_packets_total 43321
+	# HELP node_xfrm_out_error_packets_total All errors which is not matched others
+	# TYPE node_xfrm_out_error_packets_total counter
+	node_xfrm_out_error_packets_total 1e+06
+	# HELP node_xfrm_out_no_states_packets_total No state is found
+	# TYPE node_xfrm_out_no_states_packets_total counter
+	node_xfrm_out_no_states_packets_total 869
+	# HELP node_xfrm_out_pol_block_packets_total Policy discards
+	# TYPE node_xfrm_out_pol_block_packets_total counter
+	node_xfrm_out_pol_block_packets_total 43456
+	# HELP node_xfrm_out_pol_dead_packets_total Policy is dead
+	# TYPE node_xfrm_out_pol_dead_packets_total counter
+	node_xfrm_out_pol_dead_packets_total 7656
+	# HELP node_xfrm_out_pol_error_packets_total Policy error
+	# TYPE node_xfrm_out_pol_error_packets_total counter
+	node_xfrm_out_pol_error_packets_total 1454
+	# HELP node_xfrm_out_state_expired_packets_total State is expired
+	# TYPE node_xfrm_out_state_expired_packets_total counter
+	node_xfrm_out_state_expired_packets_total 565
+	# HELP node_xfrm_out_state_invalid_packets_total State is invalid, perhaps expired
+	# TYPE node_xfrm_out_state_invalid_packets_total counter
+	node_xfrm_out_state_invalid_packets_total 28765
+	# HELP node_xfrm_out_state_mode_error_packets_total Transformation mode specific error
+	# TYPE node_xfrm_out_state_mode_error_packets_total counter
+	node_xfrm_out_state_mode_error_packets_total 8
+	# HELP node_xfrm_out_state_proto_error_packets_total Transformation protocol specific error
+	# TYPE node_xfrm_out_state_proto_error_packets_total counter
+	node_xfrm_out_state_proto_error_packets_total 4542
+	# HELP node_xfrm_out_state_seq_error_packets_total Sequence error i.e. Sequence number overflow
+	# TYPE node_xfrm_out_state_seq_error_packets_total counter
+	node_xfrm_out_state_seq_error_packets_total 543
+	`
+	*procPath = "fixtures/proc"
+
+	logger := log.NewLogfmtLogger(os.Stderr)
+	c, err := NewXfrmCollector(logger)
+	if err != nil {
+		t.Fatal(err)
+	}
+	reg := prometheus.NewRegistry()
+	reg.MustRegister(&testXfrmCollector{xc: c})
+
+	sink := make(chan prometheus.Metric)
+	go func() {
+		err = c.Update(sink)
+		if err != nil {
+			panic(fmt.Errorf("failed to update collector: %s", err))
+		}
+		close(sink)
+	}()
+
+	err = testutil.GatherAndCompare(reg, strings.NewReader(testcase))
+	if err != nil {
+		t.Fatal(err)
+	}
+}

collector/zfs_freebsd.go

@@ -17,13 +17,9 @@
 package collector
 
 import (
-	"fmt"
-
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
-
-	"golang.org/x/sys/unix"
 )
 
 type zfsCollector struct {
@@ -48,7 +44,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.abdstats.linear_cnt",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.CounterValue,
-				labels:      nil,
 			},
 			{
 				name:        "abdstats_linear_data_bytes",
@@ -56,7 +51,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.abdstats.linear_data_size",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
-				labels:      nil,
 			},
 			{
 				name:        "abdstats_scatter_chunk_waste_bytes",
@@ -64,7 +58,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.abdstats.scatter_chunk_waste",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
-				labels:      nil,
 			},
 			{
 				name:        "abdstats_scatter_count_total",
@@ -72,7 +65,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.abdstats.scatter_cnt",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.CounterValue,
-				labels:      nil,
 			},
 			{
 				name:        "abdstats_scatter_data_bytes",
@@ -80,7 +72,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.abdstats.scatter_data_size",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
-				labels:      nil,
 			},
 			{
 				name:        "abdstats_struct_bytes",
@@ -88,7 +79,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.abdstats.struct_size",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
-				labels:      nil,
 			},
 			{
 				name:        "arcstats_anon_bytes",
@@ -96,7 +86,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.anon_size",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
-				labels:      nil,
 			},
 			{
 				name:        "arcstats_c_bytes",
@@ -104,7 +93,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.c",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
-				labels:      nil,
 			},
 			{
 				name:        "arcstats_c_max_bytes",
@@ -112,7 +100,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.c_max",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
-				labels:      nil,
 			},
 			{
 				name:        "arcstats_c_min_bytes",
@@ -120,7 +107,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.c_min",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
-				labels:      nil,
 			},
 			{
 				name:        "arcstats_data_bytes",
@@ -128,7 +114,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.data_size",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
-				labels:      nil,
 			},
 			{
 				name:        "arcstats_demand_data_hits_total",
@@ -136,7 +121,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.demand_data_hits",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.CounterValue,
-				labels:      nil,
 			},
 			{
 				name:        "arcstats_demand_data_misses_total",
@@ -144,7 +128,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.demand_data_misses",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.CounterValue,
-				labels:      nil,
 			},
 			{
 				name:        "arcstats_demand_metadata_hits_total",
@@ -152,7 +135,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.demand_metadata_hits",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.CounterValue,
-				labels:      nil,
 			},
 			{
 				name:        "arcstats_demand_metadata_misses_total",
@@ -160,7 +142,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.demand_metadata_misses",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.CounterValue,
-				labels:      nil,
 			},
 			{
 				name:        "arcstats_hdr_bytes",
@@ -168,7 +149,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.hdr_size",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
-				labels:      nil,
 			},
 			{
 				name:        "arcstats_hits_total",
@@ -176,7 +156,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.hits",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.CounterValue,
-				labels:      nil,
 			},
 			{
 				name:        "arcstats_misses_total",
@@ -184,7 +163,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.misses",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.CounterValue,
-				labels:      nil,
 			},
 			{
 				name:        "arcstats_mfu_ghost_hits_total",
@@ -192,7 +170,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.mfu_ghost_hits",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.CounterValue,
-				labels:      nil,
 			},
 			{
 				name:        "arcstats_mfu_ghost_size",
@@ -200,7 +177,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.mfu_ghost_size",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
-				labels:      nil,
 			},
 			{
 				name:        "arcstats_mfu_bytes",
@@ -208,7 +184,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.mfu_size",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
-				labels:      nil,
 			},
 			{
 				name:        "arcstats_mru_ghost_hits_total",
@@ -216,7 +191,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.mru_ghost_hits",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.CounterValue,
-				labels:      nil,
 			},
 			{
 				name:        "arcstats_mru_ghost_bytes",
@@ -224,7 +198,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.mru_ghost_size",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
-				labels:      nil,
 			},
 			{
 				name:        "arcstats_mru_bytes",
@@ -232,7 +205,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.mru_size",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
-				labels:      nil,
 			},
 			{
 				name:        "arcstats_other_bytes",
@@ -240,7 +212,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.other_size",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
-				labels:      nil,
 			},
 			// when FreeBSD 14.0+, `meta/pm/pd` install of `p`.
 			{
@@ -249,7 +220,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.p",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
-				labels:      nil,
 			},
 			{
 				name:        "arcstats_meta_bytes",
@@ -278,7 +248,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.arcstats.size",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.GaugeValue,
-				labels:      nil,
 			},
 			{
 				name:        "zfetchstats_hits_total",
@@ -286,7 +255,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.zfetchstats.hits",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.CounterValue,
-				labels:      nil,
 			},
 			{
 				name:        "zfetchstats_misses_total",
@@ -294,7 +262,6 @@ func NewZfsCollector(logger log.Logger) (Collector, error) {
 				mib:         "kstat.zfs.misc.zfetchstats.misses",
 				dataType:    bsdSysctlTypeUint64,
 				valueType:   prometheus.CounterValue,
-				labels:      nil,
 			},
 		},
 		logger: logger,
@@ -320,42 +287,3 @@ func (c *zfsCollector) Update(ch chan<- prometheus.Metric) error {
 
 	return nil
 }
-
-func (c *zfsCollector) parseFreeBSDPoolObjsetStats() error {
-
-	sysCtlMetrics := []string{
-		"nunlinked", "nunlinks", "nread", "reads", "nwritten", "writes",
-	}
-	zfsPoolMibPrefix := "kstat.zfs.pool.dataset"
-	zfsDatasetNames := []string{}
-
-	zfsDatasets, err := unix.Sysctl(zfsPoolMibPrefix)
-	if err != nil {
-		return fmt.Errorf("couldn't get sysctl: %w", err)
-	}
-
-	for dataset, _ := range zfsDatasets {
-		if strings.HasSuffix(dataset, ".dataset_name") {
-			zfsDatasetNames = append(zfsDatasetNames, strings.SplitAfter(dataset, ".")[3])
-		}
-	}
-
-	for zpoolDataset := range zfsDatasetsNames {
-		zfsDatasetLabels := map[string]string{
-			"dataset": zpoolDataset,
-			"zpool":   strings.SplitAfter(zpoolDataset, "/")[0],
-		}
-		for metric := range sysCtlMetrics {
-			c.sysctls = append(c.sysctls, bsdSysctl{
-				name:        fmt.SprintF("node_zfs_zpool_dataset_%s", metric),
-				description: fmt.SprintF("node_zfs_zpool_dataset_%s", metric),
-				mib:         fmt.Sprintf("%s.%s.%s", zfsPoolMibPrefix, poolObj, metric),
-				dataType:    bsdSysctlTypeUint64,
-				valueType:   prometheus.CounterValue,
-				labels:      zfsDatasetLabels,
-			})
-		}
-	}
-
-	return nil
-}

collector/zfs_linux.go

@@ -104,7 +104,7 @@ func (c *zfsCollector) updatePoolStats(ch chan<- prometheus.Metric) error {
 			return errZFSNotAvailable
 		}
 
-		err = c.parseLinuxPoolObjsetFile(file, zpoolPath, func(poolName string, datasetName string, s zfsSysctl, v uint64) {
+		err = c.parsePoolObjsetFile(file, zpoolPath, func(poolName string, datasetName string, s zfsSysctl, v uint64) {
 			ch <- c.constPoolObjsetMetric(poolName, datasetName, s, v)
 		})
 		file.Close()
@@ -220,7 +220,7 @@ func (c *zfsCollector) parsePoolProcfsFile(reader io.Reader, zpoolPath string, h
 	return scanner.Err()
 }
 
-func (c *zfsCollector) parseLinuxPoolObjsetFile(reader io.Reader, zpoolPath string, handler func(string, string, zfsSysctl, uint64)) error {
+func (c *zfsCollector) parsePoolObjsetFile(reader io.Reader, zpoolPath string, handler func(string, string, zfsSysctl, uint64)) error {
 	scanner := bufio.NewScanner(reader)
 
 	parseLine := false

collector/zfs_linux_test.go

@@ -332,7 +332,7 @@ func TestZpoolObjsetParsing(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		err = c.parseLinuxPoolObjsetFile(file, zpoolPath, func(poolName string, datasetName string, s zfsSysctl, v uint64) {
+		err = c.parsePoolObjsetFile(file, zpoolPath, func(poolName string, datasetName string, s zfsSysctl, v uint64) {
 			if s != zfsSysctl("kstat.zfs.misc.objset.writes") {
 				return
 			}

docs/node-mixin/alerts/alerts.libsonnet

@@ -10,7 +10,7 @@
               (
                 node_filesystem_avail_bytes{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} / node_filesystem_size_bytes{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} * 100 < %(fsSpaceFillingUpWarningThreshold)d
               and
-                predict_linear(node_filesystem_avail_bytes{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s}[6h], 24*60*60) < 0
+                predict_linear(node_filesystem_avail_bytes{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s}[%(fsSpaceFillingUpPredictionWindow)s], 24*60*60) < 0
               and
                 node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} == 0
               )
@@ -388,7 +388,7 @@
             annotations: {
               summary: 'Disk IO queue is high.',
               description: |||
-                Disk IO queue (aqu-sq) is high on {{ $labels.device }} at {{ $labels.instance }}, has been above %(diskIOSaturationThreshold)d for the last 15 minutes, is currently at {{ printf "%%.2f" $value }}.
+                Disk IO queue (aqu-sq) is high on {{ $labels.device }} at {{ $labels.instance }}, has been above %(diskIOSaturationThreshold)d for the last 30 minutes, is currently at {{ printf "%%.2f" $value }}.
                 This symptom might indicate disk saturation.
               ||| % $._config,
             },
@@ -407,6 +407,20 @@
               description: 'Systemd service {{ $labels.name }} has entered failed state at {{ $labels.instance }}',
             },
           },
+          {
+            alert: 'NodeBondingDegraded',
+            expr: |||
+              (node_bonding_slaves - node_bonding_active) != 0
+            ||| % $._config,
+            'for': '5m',
+            labels: {
+              severity: 'warning',
+            },
+            annotations: {
+              summary: 'Bonding interface is degraded',
+              description: 'Bonding interface {{ $labels.master }} on {{ $labels.instance }} is in degraded state due to one or more slave failures.',
+            },
+          },
         ],
       },
     ],

docs/node-mixin/config.libsonnet

@@ -54,13 +54,19 @@
     // 'NodeFilesystemSpaceFillingUp' alerts. These alerts fire if the disk
     // usage grows in a way that it is predicted to run out in 4h or 1d
     // and if the provided thresholds have been reached right now.
-    // In some cases you'll want to adjust these, e.g. by default Kubernetes
+    // In some cases you'll want to adjust these, e.g., by default, Kubernetes
     // runs the image garbage collection when the disk usage reaches 85%
     // of its available space. In that case, you'll want to reduce the
     // critical threshold below to something like 14 or 15, otherwise
     // the alert could fire under normal node usage.
+    // Additionally, the prediction window for the alert can be configured
+    // to account for environments where disk usage can fluctuate within
+    // a short time frame. By extending the prediction window, you can
+    // reduce false positives caused by temporary spikes, providing a
+    // more accurate prediction of disk space issues.
     fsSpaceFillingUpWarningThreshold: 40,
     fsSpaceFillingUpCriticalThreshold: 20,
+    fsSpaceFillingUpPredictionWindow: '6h',
 
     // Available disk space (%) thresholds on which to trigger the
     // 'NodeFilesystemAlmostOutOfSpace' alerts.

docs/node-mixin/lib/prom-mixin.libsonnet

@@ -27,6 +27,23 @@ local table = grafana70.panel.table;
       type: 'datasource',
     },
 
+    local clusterTemplatePrototype =
+      template.new(
+        'cluster',
+        '$datasource',
+        '',
+        hide=if config.showMultiCluster then '' else '2',
+        refresh='time',
+        label='Cluster',
+      ),
+    local clusterTemplate =
+      if platform == 'Darwin' then
+        clusterTemplatePrototype
+        { query: 'label_values(node_uname_info{%(nodeExporterSelector)s, sysname="Darwin"}, %(clusterLabel)s)' % config }
+      else
+        clusterTemplatePrototype
+        { query: 'label_values(node_uname_info{%(nodeExporterSelector)s, sysname!="Darwin"}, %(clusterLabel)s)' % config },
+
     local instanceTemplatePrototype =
       template.new(
         'instance',
@@ -38,11 +55,10 @@ local table = grafana70.panel.table;
     local instanceTemplate =
       if platform == 'Darwin' then
         instanceTemplatePrototype
-        { query: 'label_values(node_uname_info{%(nodeExporterSelector)s, sysname="Darwin"}, instance)' % config }
+        { query: 'label_values(node_uname_info{%(nodeExporterSelector)s, %(clusterLabel)s="$cluster", sysname="Darwin"}, instance)' % config }
       else
         instanceTemplatePrototype
-        { query: 'label_values(node_uname_info{%(nodeExporterSelector)s, sysname!="Darwin"}, instance)' % config },
+        { query: 'label_values(node_uname_info{%(nodeExporterSelector)s, %(clusterLabel)s="$cluster", sysname!="Darwin"}, instance)' % config },
-
 
     local idleCPU =
       graphPanel.new(
@@ -57,9 +73,9 @@ local table = grafana70.panel.table;
       .addTarget(prometheus.target(
         |||
           (
-            (1 - sum without (mode) (rate(node_cpu_seconds_total{%(nodeExporterSelector)s, mode=~"idle|iowait|steal", instance="$instance"}[$__rate_interval])))
+            (1 - sum without (mode) (rate(node_cpu_seconds_total{%(nodeExporterSelector)s, mode=~"idle|iowait|steal", instance="$instance", %(clusterLabel)s="$cluster"}[$__rate_interval])))
           / ignoring(cpu) group_left
-            count without (cpu, mode) (node_cpu_seconds_total{%(nodeExporterSelector)s, mode="idle", instance="$instance"})
+            count without (cpu, mode) (node_cpu_seconds_total{%(nodeExporterSelector)s, mode="idle", instance="$instance", %(clusterLabel)s="$cluster"})
           )
         ||| % config,
         legendFormat='{{cpu}}',
@@ -75,10 +91,10 @@ local table = grafana70.panel.table;
         min=0,
         fill=0,
       )
-      .addTarget(prometheus.target('node_load1{%(nodeExporterSelector)s, instance="$instance"}' % config, legendFormat='1m load average'))
+      .addTarget(prometheus.target('node_load1{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster"}' % config, legendFormat='1m load average'))
-      .addTarget(prometheus.target('node_load5{%(nodeExporterSelector)s, instance="$instance"}' % config, legendFormat='5m load average'))
+      .addTarget(prometheus.target('node_load5{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster"}' % config, legendFormat='5m load average'))
-      .addTarget(prometheus.target('node_load15{%(nodeExporterSelector)s, instance="$instance"}' % config, legendFormat='15m load average'))
+      .addTarget(prometheus.target('node_load15{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster"}' % config, legendFormat='15m load average'))
-      .addTarget(prometheus.target('count(node_cpu_seconds_total{%(nodeExporterSelector)s, instance="$instance", mode="idle"})' % config, legendFormat='logical cores')),
+      .addTarget(prometheus.target('count(node_cpu_seconds_total{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster", mode="idle"})' % config, legendFormat='logical cores')),
 
     local memoryGraphPanelPrototype =
       graphPanel.new(
@@ -94,44 +110,44 @@ local table = grafana70.panel.table;
         .addTarget(prometheus.target(
           |||
             (
-              node_memory_MemTotal_bytes{%(nodeExporterSelector)s, instance="$instance"}
+              node_memory_MemTotal_bytes{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster"}
             -
-              node_memory_MemFree_bytes{%(nodeExporterSelector)s, instance="$instance"}
+              node_memory_MemFree_bytes{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster"}
             -
-              node_memory_Buffers_bytes{%(nodeExporterSelector)s, instance="$instance"}
+              node_memory_Buffers_bytes{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster"}
             -
-              node_memory_Cached_bytes{%(nodeExporterSelector)s, instance="$instance"}
+              node_memory_Cached_bytes{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster"}
             )
           ||| % config,
           legendFormat='memory used'
         ))
-        .addTarget(prometheus.target('node_memory_Buffers_bytes{%(nodeExporterSelector)s, instance="$instance"}' % config, legendFormat='memory buffers'))
+        .addTarget(prometheus.target('node_memory_Buffers_bytes{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster"}' % config, legendFormat='memory buffers'))
-        .addTarget(prometheus.target('node_memory_Cached_bytes{%(nodeExporterSelector)s, instance="$instance"}' % config, legendFormat='memory cached'))
+        .addTarget(prometheus.target('node_memory_Cached_bytes{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster"}' % config, legendFormat='memory cached'))
-        .addTarget(prometheus.target('node_memory_MemFree_bytes{%(nodeExporterSelector)s, instance="$instance"}' % config, legendFormat='memory free'))
+        .addTarget(prometheus.target('node_memory_MemFree_bytes{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster"}' % config, legendFormat='memory free'))
       else if platform == 'Darwin' then
         // not useful to stack
         memoryGraphPanelPrototype { stack: false }
-        .addTarget(prometheus.target('node_memory_total_bytes{%(nodeExporterSelector)s, instance="$instance"}' % config, legendFormat='Physical Memory'))
+        .addTarget(prometheus.target('node_memory_total_bytes{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster"}' % config, legendFormat='Physical Memory'))
         .addTarget(prometheus.target(
           |||
             (
-                node_memory_internal_bytes{%(nodeExporterSelector)s, instance="$instance"} -
+                node_memory_internal_bytes{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster"} -
-                node_memory_purgeable_bytes{%(nodeExporterSelector)s, instance="$instance"} +
+                node_memory_purgeable_bytes{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster"} +
-                node_memory_wired_bytes{%(nodeExporterSelector)s, instance="$instance"} +
+                node_memory_wired_bytes{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster"} +
-                node_memory_compressed_bytes{%(nodeExporterSelector)s, instance="$instance"}
+                node_memory_compressed_bytes{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster"}
             )
           ||| % config, legendFormat='Memory Used'
         ))
         .addTarget(prometheus.target(
           |||
             (
-                node_memory_internal_bytes{%(nodeExporterSelector)s, instance="$instance"} -
+                node_memory_internal_bytes{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster"} -
-                node_memory_purgeable_bytes{%(nodeExporterSelector)s, instance="$instance"}
+                node_memory_purgeable_bytes{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster"}
             )
           ||| % config, legendFormat='App Memory'
         ))
-        .addTarget(prometheus.target('node_memory_wired_bytes{%(nodeExporterSelector)s, instance="$instance"}' % config, legendFormat='Wired Memory'))
+        .addTarget(prometheus.target('node_memory_wired_bytes{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster"}' % config, legendFormat='Wired Memory'))
-        .addTarget(prometheus.target('node_memory_compressed_bytes{%(nodeExporterSelector)s, instance="$instance"}' % config, legendFormat='Compressed')),
+        .addTarget(prometheus.target('node_memory_compressed_bytes{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster"}' % config, legendFormat='Compressed')),
 
     // NOTE: avg() is used to circumvent a label change caused by a node_exporter rollout.
     local memoryGaugePanelPrototype =
@@ -155,8 +171,8 @@ local table = grafana70.panel.table;
           |||
             100 -
             (
-              avg(node_memory_MemAvailable_bytes{%(nodeExporterSelector)s, instance="$instance"}) /
+              avg(node_memory_MemAvailable_bytes{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster"}) /
-              avg(node_memory_MemTotal_bytes{%(nodeExporterSelector)s, instance="$instance"})
+              avg(node_memory_MemTotal_bytes{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster"})
             * 100
             )
           ||| % config,
@@ -168,12 +184,12 @@ local table = grafana70.panel.table;
           |||
             (
                 (
-                  avg(node_memory_internal_bytes{%(nodeExporterSelector)s, instance="$instance"}) -
+                  avg(node_memory_internal_bytes{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster"}) -
-                  avg(node_memory_purgeable_bytes{%(nodeExporterSelector)s, instance="$instance"}) +
+                  avg(node_memory_purgeable_bytes{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster"}) +
-                  avg(node_memory_wired_bytes{%(nodeExporterSelector)s, instance="$instance"}) +
+                  avg(node_memory_wired_bytes{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster"}) +
-                  avg(node_memory_compressed_bytes{%(nodeExporterSelector)s, instance="$instance"})
+                  avg(node_memory_compressed_bytes{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster"})
                 ) /
-                avg(node_memory_total_bytes{%(nodeExporterSelector)s, instance="$instance"})
+                avg(node_memory_total_bytes{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster"})
             )
             *
             100
@@ -190,17 +206,17 @@ local table = grafana70.panel.table;
       )
       // TODO: Does it make sense to have those three in the same panel?
       .addTarget(prometheus.target(
-        'rate(node_disk_read_bytes_total{%(nodeExporterSelector)s, instance="$instance", %(diskDeviceSelector)s}[$__rate_interval])' % config,
+        'rate(node_disk_read_bytes_total{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster", %(diskDeviceSelector)s}[$__rate_interval])' % config,
         legendFormat='{{device}} read',
         intervalFactor=1,
       ))
       .addTarget(prometheus.target(
-        'rate(node_disk_written_bytes_total{%(nodeExporterSelector)s, instance="$instance", %(diskDeviceSelector)s}[$__rate_interval])' % config,
+        'rate(node_disk_written_bytes_total{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster", %(diskDeviceSelector)s}[$__rate_interval])' % config,
         legendFormat='{{device}} written',
         intervalFactor=1,
       ))
       .addTarget(prometheus.target(
-        'rate(node_disk_io_time_seconds_total{%(nodeExporterSelector)s, instance="$instance", %(diskDeviceSelector)s}[$__rate_interval])' % config,
+        'rate(node_disk_io_time_seconds_total{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster", %(diskDeviceSelector)s}[$__rate_interval])' % config,
         legendFormat='{{device}} io time',
         intervalFactor=1,
       )) +
@@ -232,7 +248,7 @@ local table = grafana70.panel.table;
       .addThresholdStep(color='red', value=0.9)
       .addTarget(prometheus.target(
         |||
-          max by (mountpoint) (node_filesystem_size_bytes{%(nodeExporterSelector)s, instance="$instance", %(fsSelector)s, %(fsMountpointSelector)s})
+          max by (mountpoint) (node_filesystem_size_bytes{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster", %(fsSelector)s, %(fsMountpointSelector)s})
         ||| % config,
         legendFormat='',
         instant=true,
@@ -240,7 +256,7 @@ local table = grafana70.panel.table;
       ))
       .addTarget(prometheus.target(
         |||
-          max by (mountpoint) (node_filesystem_avail_bytes{%(nodeExporterSelector)s, instance="$instance", %(fsSelector)s, %(fsMountpointSelector)s})
+          max by (mountpoint) (node_filesystem_avail_bytes{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster", %(fsSelector)s, %(fsMountpointSelector)s})
         ||| % config,
         legendFormat='',
         instant=true,
@@ -421,7 +437,7 @@ local table = grafana70.panel.table;
         fill=0,
       )
       .addTarget(prometheus.target(
-        'rate(node_network_receive_bytes_total{%(nodeExporterSelector)s, instance="$instance", device!="lo"}[$__rate_interval]) * 8' % config,
+        'rate(node_network_receive_bytes_total{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster", device!="lo"}[$__rate_interval]) * 8' % config,
         legendFormat='{{device}}',
         intervalFactor=1,
       )),
@@ -437,7 +453,7 @@ local table = grafana70.panel.table;
         fill=0,
       )
       .addTarget(prometheus.target(
-        'rate(node_network_transmit_bytes_total{%(nodeExporterSelector)s, instance="$instance", device!="lo"}[$__rate_interval]) * 8' % config,
+        'rate(node_network_transmit_bytes_total{%(nodeExporterSelector)s, instance="$instance", %(clusterLabel)s="$cluster", device!="lo"}[$__rate_interval]) * 8' % config,
         legendFormat='{{device}}',
         intervalFactor=1,
       )),
@@ -473,6 +489,7 @@ local table = grafana70.panel.table;
     local templates =
       [
         prometheusDatasourceTemplate,
+        clusterTemplate,
         instanceTemplate,
       ],
 

end-to-end-test.sh

@@ -40,7 +40,6 @@ enabled_collectors=$(cat << COLLECTORS
   qdisc
   rapl
   schedstat
-  selinux
   slabinfo
   sockstat
   softirqs
@@ -50,13 +49,16 @@ enabled_collectors=$(cat << COLLECTORS
   thermal_zone
   udp_queues
   vmstat
+  watchdog
   wifi
+  xfrm
   xfs
   zfs
   zoneinfo
 COLLECTORS
 )
 disabled_collectors=$(cat << COLLECTORS
+  selinux
   filesystem
   timex
   uname

go.mod

@@ -1,10 +1,10 @@
 module github.com/prometheus/node_exporter
 
-go 1.19
+go 1.20
 
 require (
-	github.com/alecthomas/kingpin/v2 v2.3.2
+	github.com/alecthomas/kingpin/v2 v2.4.0
-	github.com/beevik/ntp v1.3.0
+	github.com/beevik/ntp v1.3.1
 	github.com/coreos/go-systemd/v22 v22.5.0
 	github.com/dennwc/btrfs v0.0.0-20230312211831-a1f570bd01a1
 	github.com/ema/qdisc v1.0.0
@@ -14,7 +14,7 @@ require (
 	github.com/hodgesds/perf-utils v0.7.0
 	github.com/illumos/go-kstat v0.0.0-20210513183136-173c9b0a9973
 	github.com/josharian/native v1.1.0
-	github.com/jsimonetti/rtnetlink v1.3.5
+	github.com/jsimonetti/rtnetlink v1.4.1
 	github.com/lufia/iostat v1.2.1
 	github.com/mattn/go-xmlrpc v0.0.3
 	github.com/mdlayher/ethtool v0.1.0
@@ -22,15 +22,15 @@ require (
 	github.com/mdlayher/wifi v0.1.0
 	github.com/opencontainers/selinux v1.11.0
 	github.com/prometheus-community/go-runit v0.1.0
-	github.com/prometheus/client_golang v1.17.0
+	github.com/prometheus/client_golang v1.19.0
-	github.com/prometheus/client_model v0.5.0
+	github.com/prometheus/client_model v0.6.0
-	github.com/prometheus/common v0.45.0
+	github.com/prometheus/common v0.48.0
-	github.com/prometheus/exporter-toolkit v0.10.0
+	github.com/prometheus/exporter-toolkit v0.11.0
-	github.com/prometheus/procfs v0.12.0
+	github.com/prometheus/procfs v0.13.0
 	github.com/safchain/ethtool v0.3.0
 	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
-	golang.org/x/sys v0.13.0
+	golang.org/x/sys v0.18.0
-	howett.net/plist v1.0.0
+	howett.net/plist v1.0.1
 )
 
 require (
@@ -41,10 +41,8 @@ require (
 	github.com/dennwc/ioctl v1.0.0 // indirect
 	github.com/go-logfmt/logfmt v0.5.1 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
-	github.com/kr/text v0.2.0 // indirect
-	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
 	github.com/mdlayher/genetlink v1.3.2 // indirect
 	github.com/mdlayher/socket v0.4.1 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
@@ -52,12 +50,12 @@ require (
 	github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
-	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/crypto v0.18.0 // indirect
-	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/net v0.20.0 // indirect
-	golang.org/x/oauth2 v0.12.0 // indirect
+	golang.org/x/oauth2 v0.16.0 // indirect
-	golang.org/x/sync v0.3.0 // indirect
+	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

go.sum

@@ -1,17 +1,16 @@
-github.com/alecthomas/kingpin/v2 v2.3.2 h1:H0aULhgmSzN8xQ3nX1uxtdlTHYoPLu5AhHxWrKI6ocU=
+github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjHpqDjYY=
-github.com/alecthomas/kingpin/v2 v2.3.2/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE=
+github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
-github.com/beevik/ntp v1.3.0 h1:/w5VhpW5BGKS37vFm1p9oVk/t4HnnkKZAZIubHM6F7Q=
+github.com/beevik/ntp v1.3.1 h1:Y/srlT8L1yQr58kyPWFPZIxRL8ttx2SRIpVYJqZIlAM=
-github.com/beevik/ntp v1.3.0/go.mod h1:vD6h1um4kzXpqmLTuu0cCLcC+NfvC0IC+ltmEDA8E78=
+github.com/beevik/ntp v1.3.1/go.mod h1:fT6PylBq86Tsq23ZMEe47b7QQrZfYBFPnpzt0a9kJxw=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cilium/ebpf v0.11.0 h1:V8gS/bTCCjX9uUnkUFUpPsksM8n1lXBAvHcpiFk1X2Y=
+github.com/cilium/ebpf v0.12.3 h1:8ht6F9MquybnY97at+VDZb3eQQr8ev79RueWeVaEcG4=
 github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -33,8 +32,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/hashicorp/go-envparse v0.1.0 h1:bE++6bhIsNCPLvgDZkYqo3nA+/PFI51pkrHdmPSDFPY=
 github.com/hashicorp/go-envparse v0.1.0/go.mod h1:OHheN1GoygLlAkTlXLXvAdnXdZxy8JUweQ1rAXx1xnc=
 github.com/hodgesds/perf-utils v0.7.0 h1:7KlHGMuig4FRH5fNw68PV6xLmgTe7jKs9hgAcEAbioU=
@@ -46,17 +45,14 @@ github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtL
 github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
 github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
-github.com/jsimonetti/rtnetlink v1.3.5 h1:hVlNQNRlLDGZz31gBPicsG7Q53rnlsz1l1Ix/9XlpVA=
+github.com/jsimonetti/rtnetlink v1.4.1 h1:JfD4jthWBqZMEffc5RjgmlzpYttAVw1sdnmiNaPO3hE=
-github.com/jsimonetti/rtnetlink v1.3.5/go.mod h1:0LFedyiTkebnd43tE4YAkWGIq9jQphow4CcwxaT2Y00=
+github.com/jsimonetti/rtnetlink v1.4.1/go.mod h1:xJjT7t59UIZ62GLZbv6PLLo8VFrostJMPBAheR6OM8w=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
-github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lufia/iostat v1.2.1 h1:tnCdZBIglgxD47RyD55kfWQcJMGzO+1QBziSQfesf2k=
 github.com/lufia/iostat v1.2.1/go.mod h1:rEPNA0xXgjHQjuI5Cy05sLlS2oRcSlWHRLrvh/AQ+Pg=
 github.com/mattn/go-xmlrpc v0.0.3 h1:Y6WEMLEsqs3RviBrAa1/7qmbGB7DVD3brZIbqMbQdGY=
 github.com/mattn/go-xmlrpc v0.0.3/go.mod h1:mqc2dz7tP5x5BKlCahN/n+hs7OSZKJkS9JsHNBRlrxA=
-github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=
-github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
 github.com/mdlayher/ethtool v0.1.0 h1:XAWHsmKhyPOo42qq/yTPb0eFBGUKKTR1rE0dVrWVQ0Y=
 github.com/mdlayher/ethtool v0.1.0/go.mod h1:fBMLn2UhfRGtcH5ZFjr+6GUiHEjZsItFD7fSn7jbZVQ=
 github.com/mdlayher/genetlink v1.3.2 h1:KdrNKe+CTu+IbZnm/GVUMXSqBBLqcGpRDa0xkQy56gw=
@@ -75,109 +71,66 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus-community/go-runit v0.1.0 h1:uTWEj/Fn2RoLdfg/etSqwzgYNOYPrARx1BHUN052tGA=
 github.com/prometheus-community/go-runit v0.1.0/go.mod h1:AvJ9Jo3gAFu2lbM4+qfjdpq30FfiLDJZKbQ015u08IQ=
-github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
+github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
-github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
+github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
-github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
+github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos=
-github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
+github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8=
-github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM=
+github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
-github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY=
+github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
-github.com/prometheus/exporter-toolkit v0.10.0 h1:yOAzZTi4M22ZzVxD+fhy1URTuNRj/36uQJJ5S8IPza8=
+github.com/prometheus/exporter-toolkit v0.11.0 h1:yNTsuZ0aNCNFQ3aFTD2uhPOvr4iD7fdBvKPAEGkNf+g=
-github.com/prometheus/exporter-toolkit v0.10.0/go.mod h1:+sVFzuvV5JDyw+Ih6p3zFxZNVnKQa3x5qPmDSiPu4ZY=
+github.com/prometheus/exporter-toolkit v0.11.0/go.mod h1:BVnENhnNecpwoTLiABx7mrPB/OLRIgN74qlQbV+FK1Q=
-github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
+github.com/prometheus/procfs v0.13.0 h1:GqzLlQyfsPbaEHaQkO7tbDlriv/4o5Hudv6OXHGKX7o=
-github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
+github.com/prometheus/procfs v0.13.0/go.mod h1:cd4PFCR54QLnGKPaKGA6l+cfuNXtht43ZKY6tow0Y1g=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/safchain/ethtool v0.3.0 h1:gimQJpsI6sc1yIqP/y8GYgiXn/NjgvpM0RNoWLVVmP0=
 github.com/safchain/ethtool v0.3.0/go.mod h1:SA9BwrgyAqNo7M+uaL6IYbxpm5wk3L7Mm6ocLW+CJUs=
 github.com/siebenmann/go-kstat v0.0.0-20210513183136-173c9b0a9973 h1:GfSdC6wKfTGcgCS7BtzF5694Amne1pGCSTY252WhlEY=
 github.com/siebenmann/go-kstat v0.0.0-20210513183136-173c9b0a9973/go.mod h1:G81aIFAMS9ECrwBYR9YxhlPjWgrItd+Kje78O6+uqm8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc=
 github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU=
-github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
-golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
+golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
-golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
-golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
-golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
-golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ=
-golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o=
-golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
-golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
-golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
-golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4=
-golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
-golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211031064116-611d5d643895/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
-golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
-golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
-google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+howett.net/plist v1.0.1 h1:37GdZ8tP09Q35o9ych3ehygcsL+HqKSwzctveSlarvM=
-howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM=
+howett.net/plist v1.0.1/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=
-howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=