Add multipath collector for NVMe-oF subsystem path health

Add a new disabled-by-default collector (--collector.multipath) that
exposes NVMe over Fabrics connectivity metrics by reading
/sys/class/nvme-subsystem/:

- node_multipath_nvme_subsystem_info: subsystem metadata (NQN, model, etc.)
- node_multipath_nvme_subsystem_paths_total: total controller paths
- node_multipath_nvme_subsystem_paths_live: live controller paths
- node_multipath_nvme_path_state: per-controller state (live/dead/etc.)

This fills a monitoring gap for storage connectivity — the existing NVMe
collector reports hardware health but is blind to fabric path failures.
No special permissions required; sysfs is world-readable.

Signed-off-by: Shirly Radco <sradco@redhat.com>
Co-authored-by: AI Assistant <noreply@cursor.com>

Author: sradco

README.md

@@ -201,6 +201,7 @@ lnstat | Exposes stats from `/proc/net/stat/`. | Linux
 logind | Exposes session counts from [logind](http://www.freedesktop.org/wiki/Software/systemd/logind/). | Linux
 meminfo\_numa | Exposes memory statistics from `/sys/devices/system/node/node[0-9]*/meminfo`, `/sys/devices/system/node/node[0-9]*/numastat`. | Linux
 mountstats | Exposes filesystem statistics from `/proc/self/mountstats`. Exposes detailed NFS client statistics. | Linux
+multipath | Exposes NVMe-oF subsystem path health from `/sys/class/nvme-subsystem/`. | Linux
 network_route | Exposes the routing table as metrics | Linux
 pcidevice | Exposes pci devices' information including their link status and parent devices. | Linux
 perf | Exposes perf based metrics (Warning: Metrics are dependent on kernel configuration and settings). | Linux
@@ -339,6 +340,18 @@ echo 'role{role="application_server"} 1' > /path/to/directory/role.prom.$$
 mv /path/to/directory/role.prom.$$ /path/to/directory/role.prom
 ```
 
+### Multipath Collector
+
+The `multipath` collector exposes NVMe-oF (NVMe over Fabrics) subsystem path
+health by reading `/sys/class/nvme-subsystem/`. It provides connectivity-layer
+visibility that the standard `nvme` collector does not cover — specifically,
+per-controller path state and path redundancy counts at the subsystem level.
+
+This enables alerting on fabric path failures (e.g. disconnected FC cables or
+failed switches) before they cause total storage loss.
+
+No special permissions are required — the sysfs files are world-readable.
+
 ### Filtering enabled collectors
 
 The `node_exporter` will expose all metrics from enabled collectors by default.  This is the recommended way to collect metrics to avoid errors when comparing metrics of different families.

collector/fixtures/sys.ttar

@@ -2219,40 +2219,138 @@ Lines: 1
 Samsung SSD 970 PRO 512GB
 Mode: 444
 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-Path: sys/class/nvme/nvme0/serial
+Directory: sys/class/nvme/nvme0/nvme0c0n0
+Mode: 755
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/nvme/nvme0/nvme0c0n0/ana_state
 Lines: 1
-S680HF8N190894I
+optimized
 Mode: 444
 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-Path: sys/class/nvme/nvme0/state
+Path: sys/class/nvme/nvme0/nvme0c0n0/nuse
 Lines: 1
-live
+488281250
 Mode: 444
 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-Directory: sys/class/nvme/nvme0/nvme0c0n0
+Directory: sys/class/nvme/nvme0/nvme0c0n0/queue
 Mode: 755
 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-Path: sys/class/nvme/nvme0/nvme0c0n0/ana_state
+Path: sys/class/nvme/nvme0/nvme0c0n0/queue/logical_block_size
 Lines: 1
-optimized
-Mode: 444
+4096
+Mode: 644
 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 Path: sys/class/nvme/nvme0/nvme0c0n0/size
 Lines: 1
 3906250000
 Mode: 444
 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-Path: sys/class/nvme/nvme0/nvme0c0n0/nuse
+Path: sys/class/nvme/nvme0/serial
 Lines: 1
-488281250
+S680HF8N190894I
 Mode: 444
 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-Directory: sys/class/nvme/nvme0/nvme0c0n0/queue
+Path: sys/class/nvme/nvme0/state
+Lines: 1
+live
+Mode: 444
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Directory: sys/class/nvme-subsystem
 Mode: 755
 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-Path: sys/class/nvme/nvme0/nvme0c0n0/queue/logical_block_size
+Directory: sys/class/nvme-subsystem/nvme-subsys0
+Mode: 755
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/nvme-subsystem/nvme-subsys0/iopolicy
 Lines: 1
-4096
+round-robinEOF
+Mode: 644
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/nvme-subsystem/nvme-subsys0/model
+Lines: 1
+Dell PowerStoreEOF
+Mode: 644
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Directory: sys/class/nvme-subsystem/nvme-subsys0/nvme0
+Mode: 755
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/nvme-subsystem/nvme-subsys0/nvme0/address
+Lines: 1
+nn-0x200000109b123456:pn-0x100000109b123456EOF
+Mode: 644
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/nvme-subsystem/nvme-subsys0/nvme0/state
+Lines: 1
+liveEOF
+Mode: 644
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/nvme-subsystem/nvme-subsys0/nvme0/transport
+Lines: 1
+fcEOF
+Mode: 644
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Directory: sys/class/nvme-subsystem/nvme-subsys0/nvme1
+Mode: 755
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/nvme-subsystem/nvme-subsys0/nvme1/address
+Lines: 1
+nn-0x200000109b123457:pn-0x100000109b123457EOF
+Mode: 644
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/nvme-subsystem/nvme-subsys0/nvme1/state
+Lines: 1
+liveEOF
+Mode: 644
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/nvme-subsystem/nvme-subsys0/nvme1/transport
+Lines: 1
+fcEOF
+Mode: 644
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Directory: sys/class/nvme-subsystem/nvme-subsys0/nvme2
+Mode: 755
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/nvme-subsystem/nvme-subsys0/nvme2/address
+Lines: 1
+nn-0x200000109b123458:pn-0x100000109b123458EOF
+Mode: 644
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/nvme-subsystem/nvme-subsys0/nvme2/state
+Lines: 1
+liveEOF
+Mode: 644
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/nvme-subsystem/nvme-subsys0/nvme2/transport
+Lines: 1
+fcEOF
+Mode: 644
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Directory: sys/class/nvme-subsystem/nvme-subsys0/nvme3
+Mode: 755
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/nvme-subsystem/nvme-subsys0/nvme3/address
+Lines: 1
+nn-0x200000109b123459:pn-0x100000109b123459EOF
+Mode: 644
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/nvme-subsystem/nvme-subsys0/nvme3/state
+Lines: 1
+deadEOF
+Mode: 644
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/nvme-subsystem/nvme-subsys0/nvme3/transport
+Lines: 1
+fcEOF
+Mode: 644
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/nvme-subsystem/nvme-subsys0/serial
+Lines: 1
+SN12345678EOF
+Mode: 644
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/class/nvme-subsystem/nvme-subsys0/subsysnqn
+Lines: 1
+nqn.2014-08.org.nvmexpress:uuid:a34c4f3a-0d6f-5cec-dead-beefcafebabeEOF
 Mode: 644
 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 Directory: sys/class/power_supply

collector/multipath_linux.go

@@ -0,0 +1,86 @@
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !nomultipath
+
+package collector
+
+import (
+	"fmt"
+	"log/slog"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type multipathCollector struct {
+	logger             *slog.Logger
+	scanNVMeSubsystems func() ([]nvmeSubsystem, error)
+
+	nvmeSubsystemInfo       *prometheus.Desc
+	nvmeSubsystemPathsTotal *prometheus.Desc
+	nvmeSubsystemPathsLive  *prometheus.Desc
+	nvmePathState           *prometheus.Desc
+}
+
+func init() {
+	registerCollector("multipath", defaultDisabled, NewMultipathCollector)
+}
+
+// NewMultipathCollector returns a new Collector exposing multipath storage
+// connectivity metrics from /sys/class/nvme-subsystem/.
+func NewMultipathCollector(logger *slog.Logger) (Collector, error) {
+	const subsystem = "multipath"
+
+	c := &multipathCollector{
+		logger: logger,
+		nvmeSubsystemInfo: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "nvme_subsystem_info"),
+			"Non-numeric information about an NVMe subsystem.",
+			[]string{"subsystem", "nqn", "model", "serial", "iopolicy"}, nil,
+		),
+		nvmeSubsystemPathsTotal: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "nvme_subsystem_paths_total"),
+			"Total number of controller paths for an NVMe subsystem.",
+			[]string{"subsystem"}, nil,
+		),
+		nvmeSubsystemPathsLive: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "nvme_subsystem_paths_live"),
+			"Number of controller paths in live state for an NVMe subsystem.",
+			[]string{"subsystem"}, nil,
+		),
+		nvmePathState: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "nvme_path_state"),
+			"Current NVMe controller path state (1 for the current state, 0 for all others).",
+			[]string{"subsystem", "controller", "transport", "state"}, nil,
+		),
+	}
+
+	c.scanNVMeSubsystems = func() ([]nvmeSubsystem, error) {
+		return scanNVMeSubsystems(*sysPath)
+	}
+
+	return c, nil
+}
+
+func (c *multipathCollector) Update(ch chan<- prometheus.Metric) error {
+	subsystems, err := c.scanNVMeSubsystems()
+	if err != nil {
+		return fmt.Errorf("failed to scan NVMe subsystems: %w", err)
+	}
+
+	if len(subsystems) > 0 {
+		c.emitNVMeSubsystemMetrics(ch, subsystems)
+	}
+
+	return nil
+}