Add collector for PCIe devices with link information (#3339)

* Add collector for PCIe devices with link information

The link status of PCIe devices sometimes changes,
like link or speed downgrades, and devices disappear.
This patch collects PCIe devices' link infromation  to detect such failures.

As a first step, this collector exports PCIe devices'
- Device information (vendor_id, device_id, etc.)
- Parent PCIe device (e.g. PCIe bridge, PCIe switch)
- Link status (max_link_{transfers_per_second|width}, current_link_{transfers_per_second|width})

---------

Signed-off-by: Naoki MATSUMOTO <m.naoki9911@gmail.com>

README.md

@@ -201,6 +201,7 @@ logind | Exposes session counts from [logind](http://www.freedesktop.org/wiki/So
 meminfo\_numa | Exposes memory statistics from `/sys/devices/system/node/node[0-9]*/meminfo`, `/sys/devices/system/node/node[0-9]*/numastat`. | Linux
 mountstats | Exposes filesystem statistics from `/proc/self/mountstats`. Exposes detailed NFS client statistics. | Linux
 network_route | Exposes the routing table as metrics | Linux
+pcidevice | Exposes pci devices' information including their link status and parent devices. | Linux
 perf | Exposes perf based metrics (Warning: Metrics are dependent on kernel configuration and settings). | Linux
 processes | Exposes aggregate process statistics from `/proc`. | Linux
 qdisc | Exposes [queuing discipline](https://en.wikipedia.org/wiki/Network_scheduler#Linux_kernel) statistics | Linux

collector/fixtures/e2e-64k-page-output.txt

@@ -2832,6 +2832,26 @@ node_os_info{build_id="",id="ubuntu",id_like="debian",image_id="",image_version=
 # HELP node_os_version Metric containing the major.minor part of the OS version.
 # TYPE node_os_version gauge
 node_os_version{id="ubuntu",id_like="debian",name="Ubuntu"} 20.04
+# HELP node_pcidevice_current_link_transfers_per_second Value of current link's transfers per second (T/s)
+# TYPE node_pcidevice_current_link_transfers_per_second gauge
+node_pcidevice_current_link_transfers_per_second{bus="00",device="02",function="1",segment="0000"} 8e+09
+node_pcidevice_current_link_transfers_per_second{bus="01",device="00",function="0",segment="0000"} 8e+09
+# HELP node_pcidevice_current_link_width Value of current link's width (number of lanes)
+# TYPE node_pcidevice_current_link_width gauge
+node_pcidevice_current_link_width{bus="00",device="02",function="1",segment="0000"} 4
+node_pcidevice_current_link_width{bus="01",device="00",function="0",segment="0000"} 4
+# HELP node_pcidevice_info Non-numeric data from /sys/bus/pci/devices/<location>, value is always 1.
+# TYPE node_pcidevice_info gauge
+node_pcidevice_info{bus="00",class_id="0x060400",device="02",function="1",parent_bus="*",parent_device="*",parent_function="*",parent_segment="*",revision="0x00",segment="0000",subsystem_device_id="0x5095",subsystem_vendor_id="0x17aa",vendor_id="0x1634"} 1
+node_pcidevice_info{bus="01",class_id="0x010802",device="00",function="0",parent_bus="00",parent_device="02",parent_function="1",parent_segment="0000",revision="0x01",segment="0000",subsystem_device_id="0x5021",subsystem_vendor_id="0xc0a9",vendor_id="0x540a"} 1
+# HELP node_pcidevice_max_link_transfers_per_second Value of maximum link's transfers per second (T/s)
+# TYPE node_pcidevice_max_link_transfers_per_second gauge
+node_pcidevice_max_link_transfers_per_second{bus="00",device="02",function="1",segment="0000"} 8e+09
+node_pcidevice_max_link_transfers_per_second{bus="01",device="00",function="0",segment="0000"} 1.6e+10
+# HELP node_pcidevice_max_link_width Value of maximum link's width (number of lanes)
+# TYPE node_pcidevice_max_link_width gauge
+node_pcidevice_max_link_width{bus="00",device="02",function="1",segment="0000"} 8
+node_pcidevice_max_link_width{bus="01",device="00",function="0",segment="0000"} 4
 # HELP node_power_supply_capacity capacity value of /sys/class/power_supply/<power_supply>.
 # TYPE node_power_supply_capacity gauge
 node_power_supply_capacity{power_supply="BAT0"} 81
@@ -2991,6 +3011,7 @@ node_scrape_collector_success{collector="nfs"} 1
 node_scrape_collector_success{collector="nfsd"} 1
 node_scrape_collector_success{collector="nvme"} 1
 node_scrape_collector_success{collector="os"} 1
+node_scrape_collector_success{collector="pcidevice"} 1
 node_scrape_collector_success{collector="powersupplyclass"} 1
 node_scrape_collector_success{collector="pressure"} 1
 node_scrape_collector_success{collector="processes"} 1

collector/fixtures/e2e-output.txt

@@ -2854,6 +2854,26 @@ node_os_info{build_id="",id="ubuntu",id_like="debian",image_id="",image_version=
 # HELP node_os_version Metric containing the major.minor part of the OS version.
 # TYPE node_os_version gauge
 node_os_version{id="ubuntu",id_like="debian",name="Ubuntu"} 20.04
+# HELP node_pcidevice_current_link_transfers_per_second Value of current link's transfers per second (T/s)
+# TYPE node_pcidevice_current_link_transfers_per_second gauge
+node_pcidevice_current_link_transfers_per_second{bus="00",device="02",function="1",segment="0000"} 8e+09
+node_pcidevice_current_link_transfers_per_second{bus="01",device="00",function="0",segment="0000"} 8e+09
+# HELP node_pcidevice_current_link_width Value of current link's width (number of lanes)
+# TYPE node_pcidevice_current_link_width gauge
+node_pcidevice_current_link_width{bus="00",device="02",function="1",segment="0000"} 4
+node_pcidevice_current_link_width{bus="01",device="00",function="0",segment="0000"} 4
+# HELP node_pcidevice_info Non-numeric data from /sys/bus/pci/devices/<location>, value is always 1.
+# TYPE node_pcidevice_info gauge
+node_pcidevice_info{bus="00",class_id="0x060400",device="02",function="1",parent_bus="*",parent_device="*",parent_function="*",parent_segment="*",revision="0x00",segment="0000",subsystem_device_id="0x5095",subsystem_vendor_id="0x17aa",vendor_id="0x1634"} 1
+node_pcidevice_info{bus="01",class_id="0x010802",device="00",function="0",parent_bus="00",parent_device="02",parent_function="1",parent_segment="0000",revision="0x01",segment="0000",subsystem_device_id="0x5021",subsystem_vendor_id="0xc0a9",vendor_id="0x540a"} 1
+# HELP node_pcidevice_max_link_transfers_per_second Value of maximum link's transfers per second (T/s)
+# TYPE node_pcidevice_max_link_transfers_per_second gauge
+node_pcidevice_max_link_transfers_per_second{bus="00",device="02",function="1",segment="0000"} 8e+09
+node_pcidevice_max_link_transfers_per_second{bus="01",device="00",function="0",segment="0000"} 1.6e+10
+# HELP node_pcidevice_max_link_width Value of maximum link's width (number of lanes)
+# TYPE node_pcidevice_max_link_width gauge
+node_pcidevice_max_link_width{bus="00",device="02",function="1",segment="0000"} 8
+node_pcidevice_max_link_width{bus="01",device="00",function="0",segment="0000"} 4
 # HELP node_power_supply_capacity capacity value of /sys/class/power_supply/<power_supply>.
 # TYPE node_power_supply_capacity gauge
 node_power_supply_capacity{power_supply="BAT0"} 81
@@ -3013,6 +3033,7 @@ node_scrape_collector_success{collector="nfs"} 1
 node_scrape_collector_success{collector="nfsd"} 1
 node_scrape_collector_success{collector="nvme"} 1
 node_scrape_collector_success{collector="os"} 1
+node_scrape_collector_success{collector="pcidevice"} 1
 node_scrape_collector_success{collector="powersupplyclass"} 1
 node_scrape_collector_success{collector="pressure"} 1
 node_scrape_collector_success{collector="processes"} 1

collector/pcidevice_linux.go

@@ -0,0 +1,143 @@
+// Copyright 2017-2019 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !nopcidevice
+// +build !nopcidevice
+
+package collector
+
+import (
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/procfs/sysfs"
+)
+
+const (
+	pcideviceSubsystem = "pcidevice"
+)
+
+var (
+	pcideviceLabelNames = []string{"segment", "bus", "device", "function"}
+
+	pcideviceMaxLinkTSDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, pcideviceSubsystem, "max_link_transfers_per_second"),
+		"Value of maximum link's transfers per second (T/s)",
+		pcideviceLabelNames, nil,
+	)
+	pcideviceMaxLinkWidthDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, pcideviceSubsystem, "max_link_width"),
+		"Value of maximum link's width (number of lanes)",
+		pcideviceLabelNames, nil,
+	)
+
+	pcideviceCurrentLinkTSDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, pcideviceSubsystem, "current_link_transfers_per_second"),
+		"Value of current link's transfers per second (T/s)",
+		pcideviceLabelNames, nil,
+	)
+	pcideviceCurrentLinkWidthDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, pcideviceSubsystem, "current_link_width"),
+		"Value of current link's width (number of lanes)",
+		pcideviceLabelNames, nil,
+	)
+)
+
+type pcideviceCollector struct {
+	fs       sysfs.FS
+	infoDesc typedDesc
+	descs    []typedFactorDesc
+	logger   *slog.Logger
+}
+
+func init() {
+	registerCollector("pcidevice", defaultDisabled, NewPcideviceCollector)
+}
+
+// NewPcideviceCollector returns a new Collector exposing PCI devices stats.
+func NewPcideviceCollector(logger *slog.Logger) (Collector, error) {
+	fs, err := sysfs.NewFS(*sysPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open sysfs: %w", err)
+	}
+
+	c := pcideviceCollector{
+		fs:     fs,
+		logger: logger,
+		infoDesc: typedDesc{
+			desc: prometheus.NewDesc(
+				prometheus.BuildFQName(namespace, pcideviceSubsystem, "info"),
+				"Non-numeric data from /sys/bus/pci/devices/<location>, value is always 1.",
+				append(pcideviceLabelNames,
+					[]string{"parent_segment", "parent_bus", "parent_device", "parent_function",
+						"class_id", "vendor_id", "subsystem_vendor_id", "subsystem_device_id", "revision"}...),
+				nil,
+			),
+			valueType: prometheus.GaugeValue,
+		},
+		descs: []typedFactorDesc{
+			{desc: pcideviceMaxLinkTSDesc, valueType: prometheus.GaugeValue},
+			{desc: pcideviceMaxLinkWidthDesc, valueType: prometheus.GaugeValue},
+			{desc: pcideviceCurrentLinkTSDesc, valueType: prometheus.GaugeValue},
+			{desc: pcideviceCurrentLinkWidthDesc, valueType: prometheus.GaugeValue},
+		},
+	}
+
+	return &c, nil
+}
+
+func (c *pcideviceCollector) Update(ch chan<- prometheus.Metric) error {
+	devices, err := c.fs.PciDevices()
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			c.logger.Debug("PCI device not found, skipping")
+			return ErrNoData
+		}
+		return fmt.Errorf("error obtaining PCI device info: %w", err)
+	}
+
+	for _, device := range devices {
+		// The device location is represented in separated format.
+		values := device.Location.Strings()
+		if device.ParentLocation != nil {
+			values = append(values, device.ParentLocation.Strings()...)
+		} else {
+			values = append(values, []string{"*", "*", "*", "*"}...)
+		}
+		values = append(values, fmt.Sprintf("0x%06x", device.Class))
+		values = append(values, fmt.Sprintf("0x%04x", device.Device))
+		values = append(values, fmt.Sprintf("0x%04x", device.SubsystemVendor))
+		values = append(values, fmt.Sprintf("0x%04x", device.SubsystemDevice))
+		values = append(values, fmt.Sprintf("0x%02x", device.Revision))
+
+		ch <- c.infoDesc.mustNewConstMetric(1.0, values...)
+
+		// MaxLinkSpeed and CurrentLinkSpeed are represnted in GT/s
+		maxLinkSpeedTS := float64(int64(*device.MaxLinkSpeed * 1e9))
+		currentLinkSpeedTS := float64(int64(*device.CurrentLinkSpeed * 1e9))
+
+		for i, val := range []float64{
+			maxLinkSpeedTS,
+			float64(*device.MaxLinkWidth),
+			currentLinkSpeedTS,
+			float64(*device.CurrentLinkWidth),
+		} {
+			ch <- c.descs[i].mustNewConstMetric(val, device.Location.Strings()...)
+		}
+	}
+
+	return nil
+}

end-to-end-test.sh

@@ -67,6 +67,7 @@ enabled_collectors=$(cat << COLLECTORS
   netstat
   nfs
   nfsd
+  pcidevice
   pressure
   processes
   qdisc