diff --git a/test/e2e/metriconly/metrics_test.go b/test/e2e/metriconly/metrics_test.go index cf0db18f5..6500ef99a 100644 --- a/test/e2e/metriconly/metrics_test.go +++ b/test/e2e/metriconly/metrics_test.go @@ -125,12 +125,10 @@ var _ = ginkgo.Describe("NPD should export Prometheus metrics.", func() { ginkgo.BeforeEach(func() { err := npd.WaitForNPD(instance, []string{"problem_gauge"}, 120) Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Expect NPD to become ready in 120s, but hit error: %v", err)) - // This will trigger a ext4 error on the boot disk, causing the boot disk mounted as read-only and systemd-journald crashing. instance.RunCommandOrFail("sudo /home/kubernetes/bin/problem-maker --problem Ext4FilesystemError") }) ginkgo.It("NPD should update problem_counter{reason:Ext4Error} and problem_gauge{type:ReadonlyFilesystem}", func() { - ginkgo.Skip("Writing to /sys/fs/ext4/sda1/trigger_fs_error breaks SSH: https://github.com/kubernetes/node-problem-detector/issues/970") time.Sleep(5 * time.Second) assertMetricValueAtLeast(instance, "problem_counter", map[string]string{"reason": "Ext4Error"}, @@ -139,13 +137,6 @@ var _ = ginkgo.Describe("NPD should export Prometheus metrics.", func() { "problem_gauge", map[string]string{"reason": "FilesystemIsReadOnly", "type": "ReadonlyFilesystem"}, 1.0, 1.0) }) - - ginkgo.It("NPD should remain healthy", func() { - ginkgo.Skip("Writing to /sys/fs/ext4/sda1/trigger_fs_error breaks SSH: https://github.com/kubernetes/node-problem-detector/issues/970") - npdStates := instance.RunCommandOrFail("sudo systemctl show node-problem-detector -p ActiveState -p SubState") - Expect(npdStates.Stdout).To(ContainSubstring("ActiveState=active"), "NPD is no longer active: %v", npdStates) - Expect(npdStates.Stdout).To(ContainSubstring("SubState=running"), "NPD is no longer running: %v", npdStates) - }) }) ginkgo.Context("When OOM kills and docker hung happen", func() { diff --git a/test/e2e/problemmaker/README.md b/test/e2e/problemmaker/README.md index ab1d15e8b..5cf8b45b9 100644 --- a/test/e2e/problemmaker/README.md +++ b/test/e2e/problemmaker/README.md @@ -4,7 +4,7 @@ Problem maker is a program to generate/simulate various kinds of node problems. 1. NPD should report the problems correctly. 2. NPD should survive the problems as much as possible. -**Problem maker is NOT intended to be used in any other places. And please do NOT run this directly on your workstation, as it can cause real OS failures.** For example, running `sudo problem-maker --problem Ext4FilesystemError` will cause an ext4 file system error, which could result in the boot disk being mounted as readonly, requiring a reboot to recover from the failure. +**Problem maker is NOT intended to be used in any other places. And please do NOT run this directly on your workstation.** Problem makers can cause real OS failures, and the current ones write fake kernel messages to `/dev/kmsg` as root, which pollutes the kernel log and can trigger any real monitoring watching it. You shouldn't need to run it anyways. If you want to test NPD, it's best to run NPD e2e test. @@ -17,4 +17,4 @@ sudo problem-maker --problem DockerHung sudo problem-maker --problem Ext4FilesystemError ``` -Problem maker tries to generate real node problems, and can cause real node failures. And when we do not have a good way to generate the problems, we instruct problem maker to simulate problems by injecting logs. In most (if not all) scenarios, generating real problems is preferred over injecting logs. This is because when kernel is upgraded, log patterns can change. NPD e2e tests is supposed to verify whether NPD can correctly understand the tested kernel. +Problem maker tries to generate real node problems, and can cause real node failures. And when we do not have a good way to generate the problems, we instruct problem maker to simulate problems by injecting logs. Generating real problems is preferred over injecting logs when the node can survive them. This is because when kernel is upgraded, log patterns can change. NPD e2e tests is supposed to verify whether NPD can correctly understand the tested kernel. diff --git a/test/e2e/problemmaker/makers/filesystem.go b/test/e2e/problemmaker/makers/filesystem.go index bf6d303f4..f8af51395 100644 --- a/test/e2e/problemmaker/makers/filesystem.go +++ b/test/e2e/problemmaker/makers/filesystem.go @@ -16,22 +16,13 @@ limitations under the License. package makers -import ( - "os" - - "k8s.io/klog/v2" -) +const ext4ErrorPattern = `EXT4-fs error (device sda1): fake filesystem error from problem-maker +EXT4-fs (sda1): Remounting filesystem read-only` func init() { ProblemGenerators["Ext4FilesystemError"] = makeFilesystemError } -const ext4ErrorTrigger = "/sys/fs/ext4/sda1/trigger_fs_error" - func makeFilesystemError() { - msg := []byte("fake filesystem error from problem-maker") - err := os.WriteFile(ext4ErrorTrigger, msg, 0200) - if err != nil { - klog.Fatalf("Failed writing log to %q: %v", ext4ErrorTrigger, err) - } + writeKernelMessageOrDie(ext4ErrorPattern) }