From 6ddc19c99cf33aab3551ad64a451a5a5ac45727b Mon Sep 17 00:00:00 2001 From: EunJiJung Date: Sun, 2 Aug 2026 02:11:46 +0900 Subject: [PATCH] collector/conntrack: make procfs statistics optional Signed-off-by: EunJiJung --- README.md | 2 +- collector/conntrack_linux.go | 45 ++++++-- collector/conntrack_linux_test.go | 167 ++++++++++++++++++++++++++++++ 3 files changed, 203 insertions(+), 11 deletions(-) create mode 100644 collector/conntrack_linux_test.go diff --git a/README.md b/README.md index 8abf2cdf0f..2f97ce5c98 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ bcache | Exposes bcache statistics from `/sys/fs/bcache/`. | Linux bonding | Exposes the number of configured and active slaves of Linux bonding interfaces. | Linux btrfs | Exposes btrfs statistics | Linux boottime | Exposes system boot time derived from the `kern.boottime` sysctl. | Darwin, Dragonfly, FreeBSD, NetBSD, OpenBSD, Solaris -conntrack | Shows conntrack statistics (does nothing if no `/proc/sys/net/netfilter/` present). | Linux +conntrack | Shows conntrack statistics (does nothing if no `/proc/sys/net/netfilter/` present). The `nf_conntrack_stat_*` metrics additionally need a kernel built with `CONFIG_NF_CONNTRACK_PROCFS`, which recent kernels no longer enable by default; use `--no-collector.conntrack.stats` to skip them. | Linux cpu | Exposes CPU statistics | Darwin, Dragonfly, FreeBSD, Linux, Solaris, OpenBSD cpufreq | Exposes CPU frequency statistics | Linux, Solaris diskstats | Exposes disk I/O statistics. | Darwin, Linux, OpenBSD diff --git a/collector/conntrack_linux.go b/collector/conntrack_linux.go index eeeb7d4bd9..89bd3faf57 100644 --- a/collector/conntrack_linux.go +++ b/collector/conntrack_linux.go @@ -20,13 +20,20 @@ import ( "fmt" "log/slog" "os" + "sync" + "github.com/alecthomas/kingpin/v2" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/procfs" ) +var conntrackStats = kingpin.Flag("collector.conntrack.stats", "Collect conntrack statistics from /proc/net/stat/nf_conntrack, which requires a kernel built with CONFIG_NF_CONNTRACK_PROCFS.").Default("true").Bool() + type conntrackCollector struct { logger *slog.Logger + // warnOnce keeps the missing procfs interface from being logged on + // every scrape. + warnOnce sync.Once } type conntrackStatistics struct { @@ -119,30 +126,48 @@ func (c *conntrackCollector) Update(ch chan<- prometheus.Metric) error { ch <- prometheus.MustNewConstMetric( conntrackLimit, prometheus.GaugeValue, float64(value)) - conntrackStats, err := getConntrackStatistics() + // The statistics below come from /proc/net/stat/nf_conntrack, which only + // exists on kernels built with CONFIG_NF_CONNTRACK_PROCFS. The two + // metrics above are read from sysctl and stay available either way. + if !*conntrackStats { + return nil + } + + stats, err := getConntrackStatistics() if err != nil { - return c.handleErr(err) + if errors.Is(err, os.ErrNotExist) { + c.warnOnce.Do(func() { + c.logger.Warn("conntrack statistics unavailable", + "file", procFilePath("net/stat/nf_conntrack"), + "reason", "kernel built without CONFIG_NF_CONNTRACK_PROCFS", + "hint", "pass --no-collector.conntrack.stats to stop collecting them") + }) + return ErrNoData + } + return fmt.Errorf("failed to retrieve conntrack stats: %w", err) } ch <- prometheus.MustNewConstMetric( - conntrackFound, prometheus.GaugeValue, float64(conntrackStats.found)) + conntrackFound, prometheus.GaugeValue, float64(stats.found)) ch <- prometheus.MustNewConstMetric( - conntrackInvalid, prometheus.GaugeValue, float64(conntrackStats.invalid)) + conntrackInvalid, prometheus.GaugeValue, float64(stats.invalid)) ch <- prometheus.MustNewConstMetric( - conntrackIgnore, prometheus.GaugeValue, float64(conntrackStats.ignore)) + conntrackIgnore, prometheus.GaugeValue, float64(stats.ignore)) ch <- prometheus.MustNewConstMetric( - conntrackInsert, prometheus.GaugeValue, float64(conntrackStats.insert)) + conntrackInsert, prometheus.GaugeValue, float64(stats.insert)) ch <- prometheus.MustNewConstMetric( - conntrackInsertFailed, prometheus.GaugeValue, float64(conntrackStats.insertFailed)) + conntrackInsertFailed, prometheus.GaugeValue, float64(stats.insertFailed)) ch <- prometheus.MustNewConstMetric( - conntrackDrop, prometheus.GaugeValue, float64(conntrackStats.drop)) + conntrackDrop, prometheus.GaugeValue, float64(stats.drop)) ch <- prometheus.MustNewConstMetric( - conntrackEarlyDrop, prometheus.GaugeValue, float64(conntrackStats.earlyDrop)) + conntrackEarlyDrop, prometheus.GaugeValue, float64(stats.earlyDrop)) ch <- prometheus.MustNewConstMetric( - conntrackSearchRestart, prometheus.GaugeValue, float64(conntrackStats.searchRestart)) + conntrackSearchRestart, prometheus.GaugeValue, float64(stats.searchRestart)) return nil } +// handleErr covers the sysctl entries, which are absent when the nf_conntrack +// module is not loaded. func (c *conntrackCollector) handleErr(err error) error { if errors.Is(err, os.ErrNotExist) { c.logger.Debug("conntrack probably not loaded") diff --git a/collector/conntrack_linux_test.go b/collector/conntrack_linux_test.go new file mode 100644 index 0000000000..be07781b9f --- /dev/null +++ b/collector/conntrack_linux_test.go @@ -0,0 +1,167 @@ +// Copyright 2026 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !noconntrack + +package collector + +import ( + "bytes" + "log/slog" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/alecthomas/kingpin/v2" + "github.com/prometheus/client_golang/prometheus" +) + +// procWithoutConntrackStat builds a minimal procfs tree that has the conntrack +// sysctl entries but no net/stat/nf_conntrack, mimicking a kernel built +// without CONFIG_NF_CONNTRACK_PROCFS. +func procWithoutConntrackStat(t *testing.T) string { + t.Helper() + + root := t.TempDir() + sysctl := filepath.Join(root, "sys", "net", "netfilter") + if err := os.MkdirAll(sysctl, 0755); err != nil { + t.Fatal(err) + } + for name, value := range map[string]string{ + "nf_conntrack_count": "123\n", + "nf_conntrack_max": "65536\n", + } { + if err := os.WriteFile(filepath.Join(sysctl, name), []byte(value), 0644); err != nil { + t.Fatal(err) + } + } + // net/stat exists but holds no nf_conntrack file. + if err := os.MkdirAll(filepath.Join(root, "net", "stat"), 0755); err != nil { + t.Fatal(err) + } + + return root +} + +// collectConntrack runs one scrape and returns the fully qualified names of the +// metrics that were emitted, along with the error the collector returned. +func collectConntrack(t *testing.T, c Collector) ([]string, error) { + t.Helper() + + ch := make(chan prometheus.Metric, 32) + err := c.Update(ch) + close(ch) + + var names []string + for metric := range ch { + desc := metric.Desc().String() + _, rest, found := strings.Cut(desc, `fqName: "`) + if !found { + t.Fatalf("cannot parse metric description: %s", desc) + } + name, _, found := strings.Cut(rest, `"`) + if !found { + t.Fatalf("cannot parse metric description: %s", desc) + } + names = append(names, name) + } + + return names, err +} + +func newConntrackCollectorForTest(t *testing.T, logger *slog.Logger, args ...string) Collector { + t.Helper() + + if _, err := kingpin.CommandLine.Parse(args); err != nil { + t.Fatal(err) + } + c, err := NewConntrackCollector(logger) + if err != nil { + t.Fatal(err) + } + + return c +} + +func discardLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})) +} + +func TestConntrackStatsCollectedWhenProcfsAvailable(t *testing.T) { + c := newConntrackCollectorForTest(t, discardLogger(), "--path.procfs", "fixtures/proc") + + names, err := collectConntrack(t, c) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got, want := len(names), 10; got != want { + t.Fatalf("expected %d metrics, got %d: %v", want, got, names) + } + if !slices.Contains(names, "node_nf_conntrack_stat_insert_failed") { + t.Fatalf("expected stat metrics to be present, got: %v", names) + } +} + +func TestConntrackStatsMissingProcfsKeepsSysctlMetrics(t *testing.T) { + c := newConntrackCollectorForTest(t, discardLogger(), "--path.procfs", procWithoutConntrackStat(t)) + + names, err := collectConntrack(t, c) + if err != ErrNoData { + t.Fatalf("expected ErrNoData, got: %v", err) + } + want := []string{"node_nf_conntrack_entries", "node_nf_conntrack_entries_limit"} + if len(names) != len(want) { + t.Fatalf("expected only %v, got: %v", want, names) + } + for _, name := range want { + if !slices.Contains(names, name) { + t.Fatalf("expected %s to be present, got: %v", name, names) + } + } +} + +func TestConntrackStatsDisabledByFlag(t *testing.T) { + c := newConntrackCollectorForTest(t, discardLogger(), + "--path.procfs", "fixtures/proc", "--no-collector.conntrack.stats") + + names, err := collectConntrack(t, c) + if err != nil { + t.Fatalf("expected no error when stats are disabled, got: %v", err) + } + if got, want := len(names), 2; got != want { + t.Fatalf("expected %d metrics, got %d: %v", want, got, names) + } + for _, name := range names { + if strings.HasPrefix(name, "node_nf_conntrack_stat_") { + t.Fatalf("stat metrics should not be collected when disabled, got: %v", names) + } + } +} + +func TestConntrackStatsMissingProcfsWarnsOnce(t *testing.T) { + var buf bytes.Buffer + logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelWarn})) + c := newConntrackCollectorForTest(t, logger, "--path.procfs", procWithoutConntrackStat(t)) + + for i := range 3 { + if _, err := collectConntrack(t, c); err != ErrNoData { + t.Fatalf("scrape %d: expected ErrNoData, got: %v", i, err) + } + } + + if got := strings.Count(buf.String(), "conntrack statistics unavailable"); got != 1 { + t.Fatalf("expected exactly one warning across three scrapes, got %d:\n%s", got, buf.String()) + } +}