From ea787338fabf4c61555a22ae8f67e074f988f469 Mon Sep 17 00:00:00 2001 From: Veer Singh <8453348+digitalveer@users.noreply.github.com> Date: Wed, 8 Jul 2026 12:32:56 -0700 Subject: [PATCH] test/e2e: inject synthetic ext4 kmsg lines instead of a real fault The Ext4FilesystemError problem-maker wrote to /sys/fs/ext4/sda1/trigger_fs_error, a real fault on the boot disk. On kernel 6.5+ a real ext4 error shuts down the whole filesystem instead of remounting it read-only, so sshd dies and the e2e test loses its ssh connection. The ext4 metric assertions have been skipped because of this (issue 970). NPD only reads /dev/kmsg, so it does not need a real fault. OOMKill and DockerHung already inject their log lines into kmsg. Do the same for ext4. - write the two ext4 error lines to /dev/kmsg instead of triggering a fault - un-skip the ext4 metrics test; drop the "stays healthy" check (with no real fault there is nothing to survive, and the metric scrape already proves NPD is alive) - update the problem-maker README Signed-off-by: Veer Singh <8453348+digitalveer@users.noreply.github.com> --- test/e2e/metriconly/metrics_test.go | 9 --------- test/e2e/problemmaker/README.md | 4 ++-- test/e2e/problemmaker/makers/filesystem.go | 15 +++------------ 3 files changed, 5 insertions(+), 23 deletions(-) diff --git a/test/e2e/metriconly/metrics_test.go b/test/e2e/metriconly/metrics_test.go index cf0db18f5..6500ef99a 100644 --- a/test/e2e/metriconly/metrics_test.go +++ b/test/e2e/metriconly/metrics_test.go @@ -125,12 +125,10 @@ var _ = ginkgo.Describe("NPD should export Prometheus metrics.", func() { ginkgo.BeforeEach(func() { err := npd.WaitForNPD(instance, []string{"problem_gauge"}, 120) Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Expect NPD to become ready in 120s, but hit error: %v", err)) - // This will trigger a ext4 error on the boot disk, causing the boot disk mounted as read-only and systemd-journald crashing. instance.RunCommandOrFail("sudo /home/kubernetes/bin/problem-maker --problem Ext4FilesystemError") }) ginkgo.It("NPD should update problem_counter{reason:Ext4Error} and problem_gauge{type:ReadonlyFilesystem}", func() { - ginkgo.Skip("Writing to /sys/fs/ext4/sda1/trigger_fs_error breaks SSH: https://github.com/kubernetes/node-problem-detector/issues/970") time.Sleep(5 * time.Second) assertMetricValueAtLeast(instance, "problem_counter", map[string]string{"reason": "Ext4Error"}, @@ -139,13 +137,6 @@ var _ = ginkgo.Describe("NPD should export Prometheus metrics.", func() { "problem_gauge", map[string]string{"reason": "FilesystemIsReadOnly", "type": "ReadonlyFilesystem"}, 1.0, 1.0) }) - - ginkgo.It("NPD should remain healthy", func() { - ginkgo.Skip("Writing to /sys/fs/ext4/sda1/trigger_fs_error breaks SSH: https://github.com/kubernetes/node-problem-detector/issues/970") - npdStates := instance.RunCommandOrFail("sudo systemctl show node-problem-detector -p ActiveState -p SubState") - Expect(npdStates.Stdout).To(ContainSubstring("ActiveState=active"), "NPD is no longer active: %v", npdStates) - Expect(npdStates.Stdout).To(ContainSubstring("SubState=running"), "NPD is no longer running: %v", npdStates) - }) }) ginkgo.Context("When OOM kills and docker hung happen", func() { diff --git a/test/e2e/problemmaker/README.md b/test/e2e/problemmaker/README.md index ab1d15e8b..5cf8b45b9 100644 --- a/test/e2e/problemmaker/README.md +++ b/test/e2e/problemmaker/README.md @@ -4,7 +4,7 @@ Problem maker is a program to generate/simulate various kinds of node problems. 1. NPD should report the problems correctly. 2. NPD should survive the problems as much as possible. -**Problem maker is NOT intended to be used in any other places. And please do NOT run this directly on your workstation, as it can cause real OS failures.** For example, running `sudo problem-maker --problem Ext4FilesystemError` will cause an ext4 file system error, which could result in the boot disk being mounted as readonly, requiring a reboot to recover from the failure. +**Problem maker is NOT intended to be used in any other places. And please do NOT run this directly on your workstation.** Problem makers can cause real OS failures, and the current ones write fake kernel messages to `/dev/kmsg` as root, which pollutes the kernel log and can trigger any real monitoring watching it. You shouldn't need to run it anyways. If you want to test NPD, it's best to run NPD e2e test. @@ -17,4 +17,4 @@ sudo problem-maker --problem DockerHung sudo problem-maker --problem Ext4FilesystemError ``` -Problem maker tries to generate real node problems, and can cause real node failures. And when we do not have a good way to generate the problems, we instruct problem maker to simulate problems by injecting logs. In most (if not all) scenarios, generating real problems is preferred over injecting logs. This is because when kernel is upgraded, log patterns can change. NPD e2e tests is supposed to verify whether NPD can correctly understand the tested kernel. +Problem maker tries to generate real node problems, and can cause real node failures. And when we do not have a good way to generate the problems, we instruct problem maker to simulate problems by injecting logs. Generating real problems is preferred over injecting logs when the node can survive them. This is because when kernel is upgraded, log patterns can change. NPD e2e tests is supposed to verify whether NPD can correctly understand the tested kernel. diff --git a/test/e2e/problemmaker/makers/filesystem.go b/test/e2e/problemmaker/makers/filesystem.go index bf6d303f4..f8af51395 100644 --- a/test/e2e/problemmaker/makers/filesystem.go +++ b/test/e2e/problemmaker/makers/filesystem.go @@ -16,22 +16,13 @@ limitations under the License. package makers -import ( - "os" - - "k8s.io/klog/v2" -) +const ext4ErrorPattern = `EXT4-fs error (device sda1): fake filesystem error from problem-maker +EXT4-fs (sda1): Remounting filesystem read-only` func init() { ProblemGenerators["Ext4FilesystemError"] = makeFilesystemError } -const ext4ErrorTrigger = "/sys/fs/ext4/sda1/trigger_fs_error" - func makeFilesystemError() { - msg := []byte("fake filesystem error from problem-maker") - err := os.WriteFile(ext4ErrorTrigger, msg, 0200) - if err != nil { - klog.Fatalf("Failed writing log to %q: %v", ext4ErrorTrigger, err) - } + writeKernelMessageOrDie(ext4ErrorPattern) }