diff --git a/collector/fixtures/e2e-64k-page-output.txt b/collector/fixtures/e2e-64k-page-output.txt index 5475fcfe72..55eb967964 100644 --- a/collector/fixtures/e2e-64k-page-output.txt +++ b/collector/fixtures/e2e-64k-page-output.txt @@ -2210,6 +2210,7 @@ node_md_blocks{device="md101"} 322560 node_md_blocks{device="md11"} 4.190208e+06 node_md_blocks{device="md12"} 3.886394368e+09 node_md_blocks{device="md120"} 2.095104e+06 +node_md_blocks{device="md125"} 614336 node_md_blocks{device="md126"} 1.855870976e+09 node_md_blocks{device="md127"} 3.12319552e+08 node_md_blocks{device="md201"} 1.993728e+06 @@ -2229,6 +2230,7 @@ node_md_blocks_synced{device="md101"} 322560 node_md_blocks_synced{device="md11"} 0 node_md_blocks_synced{device="md12"} 3.886394368e+09 node_md_blocks_synced{device="md120"} 2.095104e+06 +node_md_blocks_synced{device="md125"} 614336 node_md_blocks_synced{device="md126"} 1.855870976e+09 node_md_blocks_synced{device="md127"} 3.12319552e+08 node_md_blocks_synced{device="md201"} 114176 @@ -2247,58 +2249,79 @@ node_md_degraded{device="md10"} 0 node_md_degraded{device="md4"} 0 node_md_degraded{device="md5"} 1 node_md_degraded{device="md6"} 1 -# HELP node_md_disks Number of active/failed/spare disks of device. +# HELP node_md_disks Number of active/failed/spare/removed disks of device. # TYPE node_md_disks gauge node_md_disks{device="md0",state="active"} 2 node_md_disks{device="md0",state="failed"} 0 +node_md_disks{device="md0",state="removed"} 0 node_md_disks{device="md0",state="spare"} 0 node_md_disks{device="md00",state="active"} 1 node_md_disks{device="md00",state="failed"} 0 +node_md_disks{device="md00",state="removed"} 0 node_md_disks{device="md00",state="spare"} 0 node_md_disks{device="md10",state="active"} 2 node_md_disks{device="md10",state="failed"} 0 +node_md_disks{device="md10",state="removed"} 0 node_md_disks{device="md10",state="spare"} 0 node_md_disks{device="md101",state="active"} 3 node_md_disks{device="md101",state="failed"} 0 +node_md_disks{device="md101",state="removed"} 0 node_md_disks{device="md101",state="spare"} 0 node_md_disks{device="md11",state="active"} 2 node_md_disks{device="md11",state="failed"} 1 +node_md_disks{device="md11",state="removed"} 0 node_md_disks{device="md11",state="spare"} 2 node_md_disks{device="md12",state="active"} 2 node_md_disks{device="md12",state="failed"} 0 +node_md_disks{device="md12",state="removed"} 0 node_md_disks{device="md12",state="spare"} 0 node_md_disks{device="md120",state="active"} 2 node_md_disks{device="md120",state="failed"} 0 +node_md_disks{device="md120",state="removed"} 0 node_md_disks{device="md120",state="spare"} 0 +node_md_disks{device="md125",state="active"} 1 +node_md_disks{device="md125",state="failed"} 0 +node_md_disks{device="md125",state="removed"} 1 +node_md_disks{device="md125",state="spare"} 0 node_md_disks{device="md126",state="active"} 2 node_md_disks{device="md126",state="failed"} 0 +node_md_disks{device="md126",state="removed"} 0 node_md_disks{device="md126",state="spare"} 0 node_md_disks{device="md127",state="active"} 2 node_md_disks{device="md127",state="failed"} 0 +node_md_disks{device="md127",state="removed"} 0 node_md_disks{device="md127",state="spare"} 0 node_md_disks{device="md201",state="active"} 2 node_md_disks{device="md201",state="failed"} 0 +node_md_disks{device="md201",state="removed"} 0 node_md_disks{device="md201",state="spare"} 0 node_md_disks{device="md219",state="active"} 0 node_md_disks{device="md219",state="failed"} 0 +node_md_disks{device="md219",state="removed"} 0 node_md_disks{device="md219",state="spare"} 3 node_md_disks{device="md3",state="active"} 8 node_md_disks{device="md3",state="failed"} 0 +node_md_disks{device="md3",state="removed"} 0 node_md_disks{device="md3",state="spare"} 2 node_md_disks{device="md4",state="active"} 0 node_md_disks{device="md4",state="failed"} 1 +node_md_disks{device="md4",state="removed"} 0 node_md_disks{device="md4",state="spare"} 1 node_md_disks{device="md6",state="active"} 1 node_md_disks{device="md6",state="failed"} 1 +node_md_disks{device="md6",state="removed"} 1 node_md_disks{device="md6",state="spare"} 1 node_md_disks{device="md7",state="active"} 3 node_md_disks{device="md7",state="failed"} 1 +node_md_disks{device="md7",state="removed"} 1 node_md_disks{device="md7",state="spare"} 0 node_md_disks{device="md8",state="active"} 2 node_md_disks{device="md8",state="failed"} 0 +node_md_disks{device="md8",state="removed"} 0 node_md_disks{device="md8",state="spare"} 2 node_md_disks{device="md9",state="active"} 4 node_md_disks{device="md9",state="failed"} 2 +node_md_disks{device="md9",state="removed"} 0 node_md_disks{device="md9",state="spare"} 1 # HELP node_md_disks_required Total number of disks of device. # TYPE node_md_disks_required gauge @@ -2309,6 +2332,7 @@ node_md_disks_required{device="md101"} 3 node_md_disks_required{device="md11"} 2 node_md_disks_required{device="md12"} 2 node_md_disks_required{device="md120"} 2 +node_md_disks_required{device="md125"} 2 node_md_disks_required{device="md126"} 2 node_md_disks_required{device="md127"} 2 node_md_disks_required{device="md201"} 2 @@ -2364,6 +2388,11 @@ node_md_state{device="md120",state="check"} 0 node_md_state{device="md120",state="inactive"} 0 node_md_state{device="md120",state="recovering"} 0 node_md_state{device="md120",state="resync"} 0 +node_md_state{device="md125",state="active"} 1 +node_md_state{device="md125",state="check"} 0 +node_md_state{device="md125",state="inactive"} 0 +node_md_state{device="md125",state="recovering"} 0 +node_md_state{device="md125",state="resync"} 0 node_md_state{device="md126",state="active"} 1 node_md_state{device="md126",state="check"} 0 node_md_state{device="md126",state="inactive"} 0 diff --git a/collector/fixtures/e2e-output.txt b/collector/fixtures/e2e-output.txt index 141cb76c3a..872ddfb1b7 100644 --- a/collector/fixtures/e2e-output.txt +++ b/collector/fixtures/e2e-output.txt @@ -2242,6 +2242,7 @@ node_md_blocks{device="md101"} 322560 node_md_blocks{device="md11"} 4.190208e+06 node_md_blocks{device="md12"} 3.886394368e+09 node_md_blocks{device="md120"} 2.095104e+06 +node_md_blocks{device="md125"} 614336 node_md_blocks{device="md126"} 1.855870976e+09 node_md_blocks{device="md127"} 3.12319552e+08 node_md_blocks{device="md201"} 1.993728e+06 @@ -2261,6 +2262,7 @@ node_md_blocks_synced{device="md101"} 322560 node_md_blocks_synced{device="md11"} 0 node_md_blocks_synced{device="md12"} 3.886394368e+09 node_md_blocks_synced{device="md120"} 2.095104e+06 +node_md_blocks_synced{device="md125"} 614336 node_md_blocks_synced{device="md126"} 1.855870976e+09 node_md_blocks_synced{device="md127"} 3.12319552e+08 node_md_blocks_synced{device="md201"} 114176 @@ -2279,58 +2281,79 @@ node_md_degraded{device="md10"} 0 node_md_degraded{device="md4"} 0 node_md_degraded{device="md5"} 1 node_md_degraded{device="md6"} 1 -# HELP node_md_disks Number of active/failed/spare disks of device. +# HELP node_md_disks Number of active/failed/spare/removed disks of device. # TYPE node_md_disks gauge node_md_disks{device="md0",state="active"} 2 node_md_disks{device="md0",state="failed"} 0 +node_md_disks{device="md0",state="removed"} 0 node_md_disks{device="md0",state="spare"} 0 node_md_disks{device="md00",state="active"} 1 node_md_disks{device="md00",state="failed"} 0 +node_md_disks{device="md00",state="removed"} 0 node_md_disks{device="md00",state="spare"} 0 node_md_disks{device="md10",state="active"} 2 node_md_disks{device="md10",state="failed"} 0 +node_md_disks{device="md10",state="removed"} 0 node_md_disks{device="md10",state="spare"} 0 node_md_disks{device="md101",state="active"} 3 node_md_disks{device="md101",state="failed"} 0 +node_md_disks{device="md101",state="removed"} 0 node_md_disks{device="md101",state="spare"} 0 node_md_disks{device="md11",state="active"} 2 node_md_disks{device="md11",state="failed"} 1 +node_md_disks{device="md11",state="removed"} 0 node_md_disks{device="md11",state="spare"} 2 node_md_disks{device="md12",state="active"} 2 node_md_disks{device="md12",state="failed"} 0 +node_md_disks{device="md12",state="removed"} 0 node_md_disks{device="md12",state="spare"} 0 node_md_disks{device="md120",state="active"} 2 node_md_disks{device="md120",state="failed"} 0 +node_md_disks{device="md120",state="removed"} 0 node_md_disks{device="md120",state="spare"} 0 +node_md_disks{device="md125",state="active"} 1 +node_md_disks{device="md125",state="failed"} 0 +node_md_disks{device="md125",state="removed"} 1 +node_md_disks{device="md125",state="spare"} 0 node_md_disks{device="md126",state="active"} 2 node_md_disks{device="md126",state="failed"} 0 +node_md_disks{device="md126",state="removed"} 0 node_md_disks{device="md126",state="spare"} 0 node_md_disks{device="md127",state="active"} 2 node_md_disks{device="md127",state="failed"} 0 +node_md_disks{device="md127",state="removed"} 0 node_md_disks{device="md127",state="spare"} 0 node_md_disks{device="md201",state="active"} 2 node_md_disks{device="md201",state="failed"} 0 +node_md_disks{device="md201",state="removed"} 0 node_md_disks{device="md201",state="spare"} 0 node_md_disks{device="md219",state="active"} 0 node_md_disks{device="md219",state="failed"} 0 +node_md_disks{device="md219",state="removed"} 0 node_md_disks{device="md219",state="spare"} 3 node_md_disks{device="md3",state="active"} 8 node_md_disks{device="md3",state="failed"} 0 +node_md_disks{device="md3",state="removed"} 0 node_md_disks{device="md3",state="spare"} 2 node_md_disks{device="md4",state="active"} 0 node_md_disks{device="md4",state="failed"} 1 +node_md_disks{device="md4",state="removed"} 0 node_md_disks{device="md4",state="spare"} 1 node_md_disks{device="md6",state="active"} 1 node_md_disks{device="md6",state="failed"} 1 +node_md_disks{device="md6",state="removed"} 1 node_md_disks{device="md6",state="spare"} 1 node_md_disks{device="md7",state="active"} 3 node_md_disks{device="md7",state="failed"} 1 +node_md_disks{device="md7",state="removed"} 1 node_md_disks{device="md7",state="spare"} 0 node_md_disks{device="md8",state="active"} 2 node_md_disks{device="md8",state="failed"} 0 +node_md_disks{device="md8",state="removed"} 0 node_md_disks{device="md8",state="spare"} 2 node_md_disks{device="md9",state="active"} 4 node_md_disks{device="md9",state="failed"} 2 +node_md_disks{device="md9",state="removed"} 0 node_md_disks{device="md9",state="spare"} 1 # HELP node_md_disks_required Total number of disks of device. # TYPE node_md_disks_required gauge @@ -2341,6 +2364,7 @@ node_md_disks_required{device="md101"} 3 node_md_disks_required{device="md11"} 2 node_md_disks_required{device="md12"} 2 node_md_disks_required{device="md120"} 2 +node_md_disks_required{device="md125"} 2 node_md_disks_required{device="md126"} 2 node_md_disks_required{device="md127"} 2 node_md_disks_required{device="md201"} 2 @@ -2396,6 +2420,11 @@ node_md_state{device="md120",state="check"} 0 node_md_state{device="md120",state="inactive"} 0 node_md_state{device="md120",state="recovering"} 0 node_md_state{device="md120",state="resync"} 0 +node_md_state{device="md125",state="active"} 1 +node_md_state{device="md125",state="check"} 0 +node_md_state{device="md125",state="inactive"} 0 +node_md_state{device="md125",state="recovering"} 0 +node_md_state{device="md125",state="resync"} 0 node_md_state{device="md126",state="active"} 1 node_md_state{device="md126",state="check"} 0 node_md_state{device="md126",state="inactive"} 0 diff --git a/collector/fixtures/proc/mdstat b/collector/fixtures/proc/mdstat index a19bf5e3f0..d6a5eac146 100644 --- a/collector/fixtures/proc/mdstat +++ b/collector/fixtures/proc/mdstat @@ -57,4 +57,7 @@ md120 : active linear sda1[1] sdb1[0] md101 : active (read-only) raid0 sdb[2] sdd[1] sdc[0] 322560 blocks super 1.2 512k chunks +md125 : active raid1 sdb2[1] + 614336 blocks super 1.0 [2/1] [U_] + unused devices: diff --git a/collector/mdadm_linux.go b/collector/mdadm_linux.go index 433e25c56b..71de4ab92a 100644 --- a/collector/mdadm_linux.go +++ b/collector/mdadm_linux.go @@ -74,7 +74,7 @@ var ( disksDesc = prometheus.NewDesc( prometheus.BuildFQName(namespace, "md", "disks"), - "Number of active/failed/spare disks of device.", + "Number of active/failed/spare/removed disks of device.", []string{"device", "state"}, nil, ) @@ -159,6 +159,14 @@ func (c *mdadmCollector) Update(ch chan<- prometheus.Metric) error { mdStat.Name, "failed", ) + // DisksDown is the count of '_' slots in mdstat (no device / removed). + ch <- prometheus.MustNewConstMetric( + disksDesc, + prometheus.GaugeValue, + float64(mdStat.DisksDown), + mdStat.Name, + "removed", + ) ch <- prometheus.MustNewConstMetric( disksDesc, prometheus.GaugeValue, diff --git a/collector/mdadm_linux_test.go b/collector/mdadm_linux_test.go index 6aa216232e..907ccceb6f 100644 --- a/collector/mdadm_linux_test.go +++ b/collector/mdadm_linux_test.go @@ -57,6 +57,7 @@ func TestMdadmStats(t *testing.T) { node_md_blocks{device="md11"} 4.190208e+06 node_md_blocks{device="md12"} 3.886394368e+09 node_md_blocks{device="md120"} 2.095104e+06 + node_md_blocks{device="md125"} 614336 node_md_blocks{device="md126"} 1.855870976e+09 node_md_blocks{device="md127"} 3.12319552e+08 node_md_blocks{device="md201"} 1.993728e+06 @@ -76,6 +77,7 @@ func TestMdadmStats(t *testing.T) { node_md_blocks_synced{device="md11"} 0 node_md_blocks_synced{device="md12"} 3.886394368e+09 node_md_blocks_synced{device="md120"} 2.095104e+06 + node_md_blocks_synced{device="md125"} 614336 node_md_blocks_synced{device="md126"} 1.855870976e+09 node_md_blocks_synced{device="md127"} 3.12319552e+08 node_md_blocks_synced{device="md201"} 114176 @@ -94,58 +96,79 @@ func TestMdadmStats(t *testing.T) { node_md_degraded{device="md4"} 0 node_md_degraded{device="md5"} 1 node_md_degraded{device="md6"} 1 - # HELP node_md_disks Number of active/failed/spare disks of device. + # HELP node_md_disks Number of active/failed/spare/removed disks of device. # TYPE node_md_disks gauge node_md_disks{device="md0",state="active"} 2 node_md_disks{device="md0",state="failed"} 0 + node_md_disks{device="md0",state="removed"} 0 node_md_disks{device="md0",state="spare"} 0 node_md_disks{device="md00",state="active"} 1 node_md_disks{device="md00",state="failed"} 0 + node_md_disks{device="md00",state="removed"} 0 node_md_disks{device="md00",state="spare"} 0 node_md_disks{device="md10",state="active"} 2 node_md_disks{device="md10",state="failed"} 0 + node_md_disks{device="md10",state="removed"} 0 node_md_disks{device="md10",state="spare"} 0 node_md_disks{device="md101",state="active"} 3 node_md_disks{device="md101",state="failed"} 0 + node_md_disks{device="md101",state="removed"} 0 node_md_disks{device="md101",state="spare"} 0 node_md_disks{device="md11",state="active"} 2 node_md_disks{device="md11",state="failed"} 1 + node_md_disks{device="md11",state="removed"} 0 node_md_disks{device="md11",state="spare"} 2 node_md_disks{device="md12",state="active"} 2 node_md_disks{device="md12",state="failed"} 0 + node_md_disks{device="md12",state="removed"} 0 node_md_disks{device="md12",state="spare"} 0 node_md_disks{device="md120",state="active"} 2 node_md_disks{device="md120",state="failed"} 0 + node_md_disks{device="md120",state="removed"} 0 node_md_disks{device="md120",state="spare"} 0 + node_md_disks{device="md125",state="active"} 1 + node_md_disks{device="md125",state="failed"} 0 + node_md_disks{device="md125",state="removed"} 1 + node_md_disks{device="md125",state="spare"} 0 node_md_disks{device="md126",state="active"} 2 node_md_disks{device="md126",state="failed"} 0 + node_md_disks{device="md126",state="removed"} 0 node_md_disks{device="md126",state="spare"} 0 node_md_disks{device="md127",state="active"} 2 node_md_disks{device="md127",state="failed"} 0 + node_md_disks{device="md127",state="removed"} 0 node_md_disks{device="md127",state="spare"} 0 node_md_disks{device="md201",state="active"} 2 node_md_disks{device="md201",state="failed"} 0 + node_md_disks{device="md201",state="removed"} 0 node_md_disks{device="md201",state="spare"} 0 node_md_disks{device="md219",state="active"} 0 node_md_disks{device="md219",state="failed"} 0 + node_md_disks{device="md219",state="removed"} 0 node_md_disks{device="md219",state="spare"} 3 node_md_disks{device="md3",state="active"} 8 node_md_disks{device="md3",state="failed"} 0 + node_md_disks{device="md3",state="removed"} 0 node_md_disks{device="md3",state="spare"} 2 node_md_disks{device="md4",state="active"} 0 node_md_disks{device="md4",state="failed"} 1 + node_md_disks{device="md4",state="removed"} 0 node_md_disks{device="md4",state="spare"} 1 node_md_disks{device="md6",state="active"} 1 node_md_disks{device="md6",state="failed"} 1 + node_md_disks{device="md6",state="removed"} 1 node_md_disks{device="md6",state="spare"} 1 node_md_disks{device="md7",state="active"} 3 node_md_disks{device="md7",state="failed"} 1 + node_md_disks{device="md7",state="removed"} 1 node_md_disks{device="md7",state="spare"} 0 node_md_disks{device="md8",state="active"} 2 node_md_disks{device="md8",state="failed"} 0 + node_md_disks{device="md8",state="removed"} 0 node_md_disks{device="md8",state="spare"} 2 node_md_disks{device="md9",state="active"} 4 node_md_disks{device="md9",state="failed"} 2 + node_md_disks{device="md9",state="removed"} 0 node_md_disks{device="md9",state="spare"} 1 # HELP node_md_disks_required Total number of disks of device. # TYPE node_md_disks_required gauge @@ -156,6 +179,7 @@ func TestMdadmStats(t *testing.T) { node_md_disks_required{device="md11"} 2 node_md_disks_required{device="md12"} 2 node_md_disks_required{device="md120"} 2 + node_md_disks_required{device="md125"} 2 node_md_disks_required{device="md126"} 2 node_md_disks_required{device="md127"} 2 node_md_disks_required{device="md201"} 2 @@ -211,6 +235,11 @@ func TestMdadmStats(t *testing.T) { node_md_state{device="md120",state="inactive"} 0 node_md_state{device="md120",state="recovering"} 0 node_md_state{device="md120",state="resync"} 0 + node_md_state{device="md125",state="active"} 1 + node_md_state{device="md125",state="check"} 0 + node_md_state{device="md125",state="inactive"} 0 + node_md_state{device="md125",state="recovering"} 0 + node_md_state{device="md125",state="resync"} 0 node_md_state{device="md126",state="active"} 1 node_md_state{device="md126",state="check"} 0 node_md_state{device="md126",state="inactive"} 0