[FEATURE] Add fscache collector

Signed-off-by: Marek Molisch <marek.molisch@innovatrics.com>

README.md

@@ -194,6 +194,7 @@ devstat | Exposes device statistics | Dragonfly, FreeBSD
 drm | Expose GPU metrics using sysfs / DRM, `amdgpu` is the only driver which exposes this information through DRM | Linux
 drbd | Exposes Distributed Replicated Block Device statistics (to version 8.4) | Linux
 ethtool | Exposes network interface information and network driver statistics equivalent to `ethtool`, `ethtool -S`, and `ethtool -i`. | Linux
+fscache | Exposes FS-Cache statistics from `/proc/fs/fscache/stats`. | Linux
 interrupts | Exposes detailed interrupts statistics. | Linux, OpenBSD
 ksmd | Exposes kernel and system statistics from `/sys/kernel/mm/ksm`. | Linux
 lnstat | Exposes stats from `/proc/net/stat/`. | Linux

collector/fixtures/proc/fs/fscache/stats

@@ -0,0 +1,29 @@
+FS-Cache statistics
+Cookies: idx=16 dat=31970 spc=0
+Objects: alc=0 nal=0 avl=0 ded=0
+ChkAux : non=0 ok=0 upd=0 obs=0
+Pages  : mrk=0 unc=0
+Acquire: n=31998 nul=0 noc=0 ok=31986 nbf=0 oom=0
+Lookups: n=0 neg=0 pos=0 crt=0 tmo=0
+Invals : n=409 run=0
+Updates: n=0 nul=0 run=0
+Relinqs: n=31939 nul=0 wcr=0 rtr=0
+AttrChg: n=0 ok=0 nbf=0 oom=0 run=0
+Allocs : n=0 ok=0 wt=0 nbf=0 int=0
+Allocs : ops=0 owt=0 abt=0
+Retrvls: n=2551742 ok=0 wt=0 nod=0 nbf=2551742 int=0 oom=0
+Retrvls: ops=0 owt=0 abt=0
+Stores : n=0 ok=0 agn=0 nbf=0 oom=0
+Stores : ops=0 run=0 pgs=0 rxd=0 olm=0
+VmScan : nos=0 gon=0 bsy=0 can=0 wt=0
+Ops    : pend=0 run=0 enq=0 can=0 rej=0
+Ops    : ini=0 dfr=0 rel=0 gc=0
+CacheOp: alo=0 luo=0 luc=0 gro=0
+CacheOp: inv=0 upo=0 dro=0 pto=0 atc=0 syn=0
+CacheOp: rap=0 ras=0 alp=0 als=0 wrp=0 ucp=0 dsp=0
+CacheEv: nsp=0 stl=0 rtr=0 cul=0
+RdHelp : RA=0 RP=0 WB=0 WBZ=0 rr=0 sr=0
+RdHelp : ZR=0 sh=0 sk=0
+RdHelp : DL=0 ds=0 df=0 di=0
+RdHelp : RD=0 rs=0 rf=0
+RdHelp : WR=0 ws=0 wf=0

collector/fscache_linux.go

@@ -0,0 +1,264 @@
+// Copyright 2024 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !nofscache
+// +build !nofscache
+
+package collector
+
+import (
+	"fmt"
+	"log/slog"
+	"os"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/procfs"
+)
+
+const (
+	fscacheSubsystem = "fscache"
+)
+
+type fscacheCollector struct {
+	// Metrics
+	objectsAllocated *prometheus.Desc
+	objectsAvailable *prometheus.Desc
+
+	acquireAttempts *prometheus.Desc
+	acquireSuccess  *prometheus.Desc
+
+	lookupsTotal    *prometheus.Desc
+	lookupsPositive *prometheus.Desc
+	lookupsNegative *prometheus.Desc
+
+	invalidationsTotal *prometheus.Desc
+
+	updatesTotal *prometheus.Desc
+
+	relinquishesTotal *prometheus.Desc
+
+	attributeChangesTotal   *prometheus.Desc
+	attributeChangesSuccess *prometheus.Desc
+
+	allocationsTotal   *prometheus.Desc
+	allocationsSuccess *prometheus.Desc
+
+	retrievalsTotal    *prometheus.Desc
+	retrievalsSuccess  *prometheus.Desc
+	retrievalsNoBuffer *prometheus.Desc
+
+	storesTotal   *prometheus.Desc
+	storesSuccess *prometheus.Desc
+
+	cacheEventRetired *prometheus.Desc
+	cacheEventCulled  *prometheus.Desc
+
+	fs     procfs.FS
+	logger *slog.Logger
+}
+
+var _ prometheus.Collector = (*fscacheCollector)(nil)
+
+func init() {
+	registerCollector("fscache", defaultEnabled, func(logger *slog.Logger) (Collector, error) {
+		return NewFscacheCollector(logger)
+	})
+}
+
+// NewFscacheCollector returns a new Collector exposing fscache stats.
+func NewFscacheCollector(logger *slog.Logger) (*fscacheCollector, error) {
+	fs, err := procfs.NewFS(*procPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open procfs: %w", err)
+	}
+
+	return &fscacheCollector{
+		objectsAllocated: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, fscacheSubsystem, "objects_allocated_total"),
+			"Number of index cookies allocated (Cookies: idx=allocated/available/unused).",
+			nil, nil,
+		),
+		objectsAvailable: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, fscacheSubsystem, "objects_available_total"),
+			"Number of index cookies available (Cookies: idx=allocated/available/unused).",
+			nil, nil,
+		),
+		acquireAttempts: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, fscacheSubsystem, "acquire_attempts_total"),
+			"Number of acquire operations attempted (Acquire: n=attempts).",
+			nil, nil,
+		),
+		acquireSuccess: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, fscacheSubsystem, "acquire_success_total"),
+			"Number of acquire operations successful (Acquire: ok=success).",
+			nil, nil,
+		),
+		lookupsTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, fscacheSubsystem, "lookups_total"),
+			"Number of lookup operations (Lookups: n=tot).",
+			nil, nil,
+		),
+		lookupsPositive: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, fscacheSubsystem, "lookups_positive_total"),
+			"Number of positive lookup operations (Lookups: pos=positive).",
+			nil, nil,
+		),
+		lookupsNegative: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, fscacheSubsystem, "lookups_negative_total"),
+			"Number of negative lookup operations (Lookups: neg=negative).",
+			nil, nil,
+		),
+		invalidationsTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, fscacheSubsystem, "invalidations_total"),
+			"Number of invalidation operations (Invals: n=tot).",
+			nil, nil,
+		),
+		updatesTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, fscacheSubsystem, "updates_total"),
+			"Number of update operations (Updates: n=tot).",
+			nil, nil,
+		),
+		relinquishesTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, fscacheSubsystem, "relinquishes_total"),
+			"Number of relinquish operations (Relinqs: n=tot).",
+			nil, nil,
+		),
+		attributeChangesTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, fscacheSubsystem, "attribute_changes_total"),
+			"Number of attribute change operations attempted (AttrChg: n=attempts).",
+			nil, nil,
+		),
+		attributeChangesSuccess: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, fscacheSubsystem, "attribute_changes_success_total"),
+			"Number of successful attribute change operations (AttrChg: ok=success).",
+			nil, nil,
+		),
+		allocationsTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, fscacheSubsystem, "allocations_total"),
+			"Number of allocation operations attempted (Allocs: n=attempts).",
+			nil, nil,
+		),
+		allocationsSuccess: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, fscacheSubsystem, "allocations_success_total"),
+			"Number of successful allocation operations (Allocs: ok=success).",
+			nil, nil,
+		),
+		retrievalsTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, fscacheSubsystem, "retrievals_total"),
+			"Number of retrieval (read) operations attempted (Retrvls: n=attempts).",
+			nil, nil,
+		),
+		retrievalsSuccess: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, fscacheSubsystem, "retrievals_success_total"),
+			"Number of successful retrieval (read) operations (Retrvls: ok=success).",
+			nil, nil,
+		),
+		retrievalsNoBuffer: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, fscacheSubsystem, "retrievals_nobuffer_total"),
+			"Number of retrieval (read) operations failed due to no buffer (Retrvls: nbf=nobuff).",
+			nil, nil,
+		),
+		storesTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, fscacheSubsystem, "stores_total"),
+			"Number of store (write) operations attempted (Stores: n=attempts).",
+			nil, nil,
+		),
+		storesSuccess: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, fscacheSubsystem, "stores_success_total"),
+			"Number of successful store (write) operations (Stores: ok=success).",
+			nil, nil,
+		),
+		cacheEventRetired: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, fscacheSubsystem, "objects_retired_total"),
+			"Number of objects retired (CacheEv: rtr=retired).",
+			nil, nil,
+		),
+		cacheEventCulled: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, fscacheSubsystem, "objects_culled_total"),
+			"Number of objects culled (CacheEv: cul=culled).",
+			nil, nil,
+		),
+		fs:     fs,
+		logger: logger,
+	}, nil
+}
+
+// Describe implements prometheus.Collector.
+func (c *fscacheCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- c.objectsAllocated
+	ch <- c.objectsAvailable
+	ch <- c.acquireAttempts
+	ch <- c.acquireSuccess
+	ch <- c.lookupsTotal
+	ch <- c.lookupsPositive
+	ch <- c.lookupsNegative
+	ch <- c.invalidationsTotal
+	ch <- c.updatesTotal
+	ch <- c.relinquishesTotal
+	ch <- c.attributeChangesTotal
+	ch <- c.attributeChangesSuccess
+	ch <- c.allocationsTotal
+	ch <- c.allocationsSuccess
+	ch <- c.retrievalsTotal
+	ch <- c.retrievalsSuccess
+	ch <- c.retrievalsNoBuffer
+	ch <- c.storesTotal
+	ch <- c.storesSuccess
+	ch <- c.cacheEventRetired
+	ch <- c.cacheEventCulled
+}
+
+// Collect implements prometheus.Collector.
+func (c *fscacheCollector) Collect(ch chan<- prometheus.Metric) {
+	// Let the collector helper handle scrape success/failure based on Update's error.
+	if err := c.Update(ch); err != nil {
+		// Optionally log the error, but don't send invalid metrics here.
+		c.logger.Error("Error updating fscache stats", "err", err)
+	}
+}
+
+// Update gathers metrics from the fscache subsystem.
+func (c *fscacheCollector) Update(ch chan<- prometheus.Metric) error {
+	stats, err := c.fs.Fscacheinfo()
+	if err != nil {
+		if os.IsNotExist(err) {
+			c.logger.Debug("Not collecting fscache statistics, as /proc/fs/fscache/stats is not available", "err", err)
+			return ErrNoData
+		}
+		return fmt.Errorf("could not get fscache stats: %w", err)
+	}
+
+	ch <- prometheus.MustNewConstMetric(c.objectsAllocated, prometheus.CounterValue, float64(stats.IndexCookiesAllocated))
+	ch <- prometheus.MustNewConstMetric(c.objectsAvailable, prometheus.CounterValue, float64(stats.ObjectsAvailable))
+	ch <- prometheus.MustNewConstMetric(c.acquireAttempts, prometheus.CounterValue, float64(stats.AcquireCookiesRequestSeen))
+	ch <- prometheus.MustNewConstMetric(c.acquireSuccess, prometheus.CounterValue, float64(stats.AcquireRequestsSucceeded))
+	ch <- prometheus.MustNewConstMetric(c.lookupsTotal, prometheus.CounterValue, float64(stats.LookupsNumber))
+	ch <- prometheus.MustNewConstMetric(c.lookupsPositive, prometheus.CounterValue, float64(stats.LookupsPositive))
+	ch <- prometheus.MustNewConstMetric(c.lookupsNegative, prometheus.CounterValue, float64(stats.LookupsNegative))
+	ch <- prometheus.MustNewConstMetric(c.invalidationsTotal, prometheus.CounterValue, float64(stats.InvalidationsNumber))
+	ch <- prometheus.MustNewConstMetric(c.updatesTotal, prometheus.CounterValue, float64(stats.UpdateCookieRequestSeen))
+	ch <- prometheus.MustNewConstMetric(c.relinquishesTotal, prometheus.CounterValue, float64(stats.RelinquishCookiesRequestSeen))
+	ch <- prometheus.MustNewConstMetric(c.attributeChangesTotal, prometheus.CounterValue, float64(stats.AttributeChangedRequestsSeen))
+	ch <- prometheus.MustNewConstMetric(c.attributeChangesSuccess, prometheus.CounterValue, float64(stats.AttributeChangedOps))
+	ch <- prometheus.MustNewConstMetric(c.allocationsTotal, prometheus.CounterValue, float64(stats.AllocationRequestsSeen))
+	ch <- prometheus.MustNewConstMetric(c.allocationsSuccess, prometheus.CounterValue, float64(stats.AllocationOkRequests))
+	ch <- prometheus.MustNewConstMetric(c.retrievalsTotal, prometheus.CounterValue, float64(stats.RetrievalsReadRequests))
+	ch <- prometheus.MustNewConstMetric(c.retrievalsSuccess, prometheus.CounterValue, float64(stats.RetrievalsOk))
+	ch <- prometheus.MustNewConstMetric(c.retrievalsNoBuffer, prometheus.CounterValue, float64(stats.RetrievalsRejectedDueToEnobufs))
+	ch <- prometheus.MustNewConstMetric(c.storesTotal, prometheus.CounterValue, float64(stats.StoreWriteRequests))
+	ch <- prometheus.MustNewConstMetric(c.storesSuccess, prometheus.CounterValue, float64(stats.StoreSuccessfulRequests))
+	ch <- prometheus.MustNewConstMetric(c.cacheEventRetired, prometheus.CounterValue, float64(stats.CacheevRetiredWhenReliquished))
+	ch <- prometheus.MustNewConstMetric(c.cacheEventCulled, prometheus.CounterValue, float64(stats.CacheevObjectsCulled))
+
+	return nil
+}

collector/fscache_linux_test.go

@@ -0,0 +1,131 @@
+// Copyright 2024 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !nofscache
+// +build !nofscache
+
+package collector
+
+import (
+	"io"
+	"log/slog"
+	"strings"
+	"testing"
+
+	"github.com/alecthomas/kingpin/v2"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+func TestFscacheStats(t *testing.T) {
+	testcases := []struct {
+		name     string
+		procPath string
+		expected string
+		err      error
+	}{
+		{
+			name:     "stats format",
+			procPath: "fixtures/proc",
+			expected: `
+# HELP node_fscache_acquire_attempts_total Number of acquire operations attempted (Acquire: n=attempts).
+# TYPE node_fscache_acquire_attempts_total counter
+node_fscache_acquire_attempts_total 31998
+# HELP node_fscache_acquire_success_total Number of acquire operations successful (Acquire: ok=success).
+# TYPE node_fscache_acquire_success_total counter
+node_fscache_acquire_success_total 31986
+# HELP node_fscache_allocations_success_total Number of successful allocation operations (Allocs: ok=success).
+# TYPE node_fscache_allocations_success_total counter
+node_fscache_allocations_success_total 0
+# HELP node_fscache_allocations_total Number of allocation operations attempted (Allocs: n=attempts).
+# TYPE node_fscache_allocations_total counter
+node_fscache_allocations_total 0
+# HELP node_fscache_attribute_changes_success_total Number of successful attribute change operations (AttrChg: ok=success).
+# TYPE node_fscache_attribute_changes_success_total counter
+node_fscache_attribute_changes_success_total 0
+# HELP node_fscache_attribute_changes_total Number of attribute change operations attempted (AttrChg: n=attempts).
+# TYPE node_fscache_attribute_changes_total counter
+node_fscache_attribute_changes_total 0
+# HELP node_fscache_invalidations_total Number of invalidation operations (Invals: n=tot).
+# TYPE node_fscache_invalidations_total counter
+node_fscache_invalidations_total 409
+# HELP node_fscache_lookups_negative_total Number of negative lookup operations (Lookups: neg=negative).
+# TYPE node_fscache_lookups_negative_total counter
+node_fscache_lookups_negative_total 0
+# HELP node_fscache_lookups_positive_total Number of positive lookup operations (Lookups: pos=positive).
+# TYPE node_fscache_lookups_positive_total counter
+node_fscache_lookups_positive_total 0
+# HELP node_fscache_lookups_total Number of lookup operations (Lookups: n=tot).
+# TYPE node_fscache_lookups_total counter
+node_fscache_lookups_total 0
+# HELP node_fscache_objects_allocated_total Number of index cookies allocated (Cookies: idx=allocated/available/unused).
+# TYPE node_fscache_objects_allocated_total counter
+node_fscache_objects_allocated_total 16
+# HELP node_fscache_objects_available_total Number of index cookies available (Cookies: idx=allocated/available/unused).
+# TYPE node_fscache_objects_available_total counter
+node_fscache_objects_available_total 0
+# HELP node_fscache_objects_culled_total Number of objects culled (CacheEv: cul=culled).
+# TYPE node_fscache_objects_culled_total counter
+node_fscache_objects_culled_total 0
+# HELP node_fscache_objects_retired_total Number of objects retired (CacheEv: rtr=retired).
+# TYPE node_fscache_objects_retired_total counter
+node_fscache_objects_retired_total 0
+# HELP node_fscache_relinquishes_total Number of relinquish operations (Relinqs: n=tot).
+# TYPE node_fscache_relinquishes_total counter
+node_fscache_relinquishes_total 31939
+# HELP node_fscache_retrievals_nobuffer_total Number of retrieval (read) operations failed due to no buffer (Retrvls: nbf=nobuff).
+# TYPE node_fscache_retrievals_nobuffer_total counter
+node_fscache_retrievals_nobuffer_total 2551742
+# HELP node_fscache_retrievals_success_total Number of successful retrieval (read) operations (Retrvls: ok=success).
+# TYPE node_fscache_retrievals_success_total counter
+node_fscache_retrievals_success_total 0
+# HELP node_fscache_retrievals_total Number of retrieval (read) operations attempted (Retrvls: n=attempts).
+# TYPE node_fscache_retrievals_total counter
+node_fscache_retrievals_total 2551742
+# HELP node_fscache_stores_success_total Number of successful store (write) operations (Stores: ok=success).
+# TYPE node_fscache_stores_success_total counter
+node_fscache_stores_success_total 0
+# HELP node_fscache_stores_total Number of store (write) operations attempted (Stores: n=attempts).
+# TYPE node_fscache_stores_total counter
+node_fscache_stores_total 0
+# HELP node_fscache_updates_total Number of update operations (Updates: n=tot).
+# TYPE node_fscache_updates_total counter
+node_fscache_updates_total 0
+`,
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			if _, err := kingpin.CommandLine.Parse([]string{"--path.procfs", tc.procPath}); err != nil {
+				t.Fatal(err)
+			}
+
+			// Create collector
+			collector, err := NewFscacheCollector(slog.New(slog.NewTextHandler(io.Discard, nil)))
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			// Register collector
+			registry := prometheus.NewRegistry()
+			registry.MustRegister(collector)
+
+			// Compare metrics
+			err = testutil.GatherAndCompare(registry, strings.NewReader(tc.expected))
+			if err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}

end-to-end-test.sh

@@ -52,6 +52,7 @@ enabled_collectors=$(cat << COLLECTORS
   entropy
   fibrechannel
   filefd
+  fscache
   hwmon
   infiniband
   interrupts
@@ -321,7 +322,7 @@ sleep 1
 get "127.0.0.1:${port}/metrics" | grep --text -E -v "${skip_re}" > "${generated_metrics}"
 
 # The following ignore-list is only applicable to the VMs used to run E2E tests on platforms for which containerized environments are not available.
-# However, owing to this, there are some non-deterministic metrics that end up generating samples, unlike their containerized counterparts, for e.g., node_network_receive_bytes_total. 
+# However, owing to this, there are some non-deterministic metrics that end up generating samples, unlike their containerized counterparts, for e.g., node_network_receive_bytes_total.
 non_deterministic_metrics=$(cat << METRICS
   node_boot_time_seconds
   node_cpu_frequency_hertz