Skip to content

Commit 4504214

Browse files
BhargaviGudiBhargaviGudi
committed
Manual cherry-pick node swap feature testcases to 4.21
condition.Status changes (True→False) for 4.21
1 parent ac4798a commit 4504214

11 files changed

Lines changed: 2643 additions & 0 deletions

‎test/extended/node/node_swap.go‎

Lines changed: 223 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,223 @@
1+
package node
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"strings"
7+
"time"
8+
9+
g "github.com/onsi/ginkgo/v2"
10+
o "github.com/onsi/gomega"
11+
ote "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo"
12+
13+
corev1 "k8s.io/api/core/v1"
14+
apierrors "k8s.io/apimachinery/pkg/api/errors"
15+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
16+
"k8s.io/apimachinery/pkg/runtime"
17+
"k8s.io/apimachinery/pkg/util/wait"
18+
"k8s.io/kubernetes/test/e2e/framework"
19+
20+
configv1 "github.com/openshift/api/config/v1"
21+
machineconfigv1 "github.com/openshift/api/machineconfiguration/v1"
22+
mcclient "github.com/openshift/client-go/machineconfiguration/clientset/versioned"
23+
exutil "github.com/openshift/origin/test/extended/util"
24+
)
25+
26+
const (
27+
workerGeneratedKubeletMC = "99-worker-generated-kubelet"
28+
)
29+
30+
var _ = g.Describe("[Jira:Node][sig-node] Node non-cnv swap configuration", func() {
31+
defer g.GinkgoRecover()
32+
33+
var oc = exutil.NewCLI("node-swap")
34+
35+
g.BeforeEach(func(ctx context.Context) {
36+
// Skip all tests on MicroShift clusters
37+
isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient())
38+
o.Expect(err).NotTo(o.HaveOccurred())
39+
if isMicroShift {
40+
g.Skip("Skipping test on MicroShift cluster")
41+
}
42+
})
43+
44+
// This test validates that:
45+
// - Worker nodes have failSwapOn=false to allow kubelet to start even if swap is present at OS level
46+
// - Control plane nodes have failSwapOn=true to prevent kubelet from starting if swap is enabled
47+
// - All nodes have swapBehavior=NoSwap to ensure kubelet does not utilize swap even if available at OS level
48+
// The swapBehavior=NoSwap configuration ensures that even if swap is manually enabled on a worker node,
49+
// the kubelet will not use it for memory management, maintaining consistent behavior across the cluster.
50+
g.It("should have correct default kubelet swap settings with worker nodes failSwapOn=false, control plane nodes failSwapOn=true, and both swapBehavior=NoSwap [OCP-86394]", ote.Informing(), func(ctx context.Context) {
51+
g.By("Getting worker nodes")
52+
allWorkerNodes, err := getNodesByLabel(ctx, oc, "node-role.kubernetes.io/worker")
53+
o.Expect(err).NotTo(o.HaveOccurred())
54+
o.Expect(len(allWorkerNodes)).Should(o.BeNumerically(">", 0), "Expected at least one worker node")
55+
56+
// Filter out nodes that are also control plane (e.g., SNO)
57+
workerNodes := getPureWorkerNodes(allWorkerNodes)
58+
59+
g.By("Validating kubelet configuration on each worker node")
60+
for _, node := range workerNodes {
61+
config, err := getKubeletConfigFromNode(ctx, oc, node.Name)
62+
o.Expect(err).NotTo(o.HaveOccurred(), "Failed to get kubelet config for worker node %s", node.Name)
63+
64+
g.By(fmt.Sprintf("Checking failSwapOn=false on worker node %s", node.Name))
65+
o.Expect(config.FailSwapOn).NotTo(o.BeNil(), "failSwapOn should be set on worker node %s", node.Name)
66+
o.Expect(*config.FailSwapOn).To(o.BeFalse(), "failSwapOn should be false on worker node %s", node.Name)
67+
framework.Logf("Worker node %s: failSwapOn=%v ✓", node.Name, *config.FailSwapOn)
68+
69+
g.By(fmt.Sprintf("Checking swapBehavior=NoSwap on worker node %s", node.Name))
70+
o.Expect(config.MemorySwap).NotTo(o.BeNil(), "memorySwap should be set on worker node %s", node.Name)
71+
o.Expect(config.MemorySwap.SwapBehavior).To(o.Equal("NoSwap"), "swapBehavior should be NoSwap on worker node %s", node.Name)
72+
framework.Logf("Worker node %s: swapBehavior=%s ✓", node.Name, config.MemorySwap.SwapBehavior)
73+
}
74+
75+
// Skip control plane validation on Hypershift (control plane is external)
76+
controlPlaneTopology, err := exutil.GetControlPlaneTopology(oc)
77+
o.Expect(err).NotTo(o.HaveOccurred())
78+
79+
if *controlPlaneTopology != configv1.ExternalTopologyMode {
80+
g.By("Getting control plane nodes")
81+
controlPlaneNodes, err := getControlPlaneNodes(ctx, oc)
82+
o.Expect(err).NotTo(o.HaveOccurred())
83+
o.Expect(len(controlPlaneNodes)).Should(o.BeNumerically(">", 0), "Expected at least one control plane node")
84+
85+
g.By("Validating kubelet configuration on each control plane node")
86+
for _, node := range controlPlaneNodes {
87+
config, err := getKubeletConfigFromNode(ctx, oc, node.Name)
88+
o.Expect(err).NotTo(o.HaveOccurred(), "Failed to get kubelet config for control plane node %s", node.Name)
89+
90+
g.By(fmt.Sprintf("Checking failSwapOn=true on control plane node %s", node.Name))
91+
o.Expect(config.FailSwapOn).NotTo(o.BeNil(), "failSwapOn should be set on control plane node %s", node.Name)
92+
o.Expect(*config.FailSwapOn).To(o.BeTrue(), "failSwapOn should be true on control plane node %s", node.Name)
93+
framework.Logf("Control plane node %s: failSwapOn=%v ✓", node.Name, *config.FailSwapOn)
94+
95+
g.By(fmt.Sprintf("Checking swapBehavior=NoSwap on control plane node %s", node.Name))
96+
o.Expect(config.MemorySwap).NotTo(o.BeNil(), "memorySwap should be set on control plane node %s", node.Name)
97+
o.Expect(config.MemorySwap.SwapBehavior).To(o.Equal("NoSwap"), "swapBehavior should be NoSwap on control plane node %s", node.Name)
98+
framework.Logf("Control plane node %s: swapBehavior=%s ✓", node.Name, config.MemorySwap.SwapBehavior)
99+
}
100+
} else {
101+
framework.Logf("Skipping control plane validation on Hypershift (external control plane)")
102+
}
103+
framework.Logf("Test PASSED: All nodes have correct default swap settings")
104+
})
105+
106+
g.It("should reject user override of swap settings via KubeletConfig API [OCP-86395]", ote.Informing(), func(ctx context.Context) {
107+
// Skip on Hypershift - MachineConfig API is not available
108+
controlPlaneTopology, err := exutil.GetControlPlaneTopology(oc)
109+
o.Expect(err).NotTo(o.HaveOccurred())
110+
if *controlPlaneTopology == configv1.ExternalTopologyMode {
111+
g.Skip("Skipping test on Hypershift - MachineConfig API not available")
112+
}
113+
114+
g.By("Creating machine config client")
115+
mcClient, err := mcclient.NewForConfig(oc.KubeFramework().ClientConfig())
116+
o.Expect(err).NotTo(o.HaveOccurred(), "Failed to create machine config client")
117+
118+
g.By("Getting initial machine config resourceVersion")
119+
// Get the initial resourceVersion of the worker machine config before creating KubeletConfig
120+
workerMC, err := mcClient.MachineconfigurationV1().MachineConfigs().Get(ctx, workerGeneratedKubeletMC, metav1.GetOptions{})
121+
initialResourceVersion := ""
122+
if err == nil {
123+
initialResourceVersion = workerMC.ResourceVersion
124+
framework.Logf("Initial %s resourceVersion: %s", workerGeneratedKubeletMC, initialResourceVersion)
125+
}
126+
127+
g.By("Creating a KubeletConfig with swap settings")
128+
kcName := fmt.Sprintf("test-swap-override-%d", time.Now().UnixNano())
129+
kubeletConfig := &machineconfigv1.KubeletConfig{
130+
ObjectMeta: metav1.ObjectMeta{
131+
Name: kcName,
132+
},
133+
Spec: machineconfigv1.KubeletConfigSpec{
134+
KubeletConfig: &runtime.RawExtension{
135+
Raw: []byte(`{
136+
"failSwapOn": true,
137+
"memorySwap": {
138+
"swapBehavior": "LimitedSwap"
139+
}
140+
}`),
141+
},
142+
},
143+
}
144+
145+
g.By("Attempting to apply the KubeletConfig")
146+
defer func() {
147+
if err := mcClient.MachineconfigurationV1().KubeletConfigs().Delete(ctx, kcName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
148+
framework.Logf("cleanup failed for KubeletConfig %s: %v", kcName, err)
149+
}
150+
}()
151+
framework.Logf("Creating KubeletConfig with failSwapOn=true and swapBehavior=LimitedSwap")
152+
_, err = mcClient.MachineconfigurationV1().KubeletConfigs().Create(ctx, kubeletConfig, metav1.CreateOptions{})
153+
o.Expect(err).NotTo(o.HaveOccurred(), "Failed to create KubeletConfig")
154+
155+
g.By("Checking KubeletConfig status for expected error message")
156+
err = wait.Poll(2*time.Second, 30*time.Second, func() (bool, error) {
157+
kc, err := mcClient.MachineconfigurationV1().KubeletConfigs().Get(ctx, kcName, metav1.GetOptions{})
158+
if err != nil {
159+
return false, err
160+
}
161+
162+
if kc.Status.ObservedGeneration != kc.Generation {
163+
framework.Logf("Waiting for controller to process generation %d (current: %d)", kc.Generation, kc.Status.ObservedGeneration)
164+
return false, nil
165+
}
166+
167+
// Fail fast if KubeletConfig was unexpectedly accepted
168+
for _, condition := range kc.Status.Conditions {
169+
if condition.Type == machineconfigv1.KubeletConfigSuccess && condition.Status == corev1.ConditionFalse {
170+
return false, fmt.Errorf("KubeletConfig was unexpectedly accepted")
171+
}
172+
}
173+
174+
// Check for Failure condition with the expected error message
175+
for _, condition := range kc.Status.Conditions {
176+
if condition.Type == machineconfigv1.KubeletConfigFailure && condition.Status == corev1.ConditionFalse {
177+
framework.Logf("Found Failure condition: %s", condition.Message)
178+
if strings.Contains(condition.Message, "failSwapOn is not allowed to be set") {
179+
return true, nil
180+
}
181+
}
182+
}
183+
return false, nil
184+
})
185+
o.Expect(err).NotTo(o.HaveOccurred(), "Expected to find error message about failSwapOn not being allowed in KubeletConfig status")
186+
187+
g.By("Verifying machine config was not created or updated")
188+
// Wait a bit to ensure no update happens
189+
time.Sleep(5 * time.Second)
190+
191+
// Check if the machine config was created or updated (compare to initial resourceVersion captured earlier)
192+
workerMC, err = mcClient.MachineconfigurationV1().MachineConfigs().Get(ctx, workerGeneratedKubeletMC, metav1.GetOptions{})
193+
if err == nil {
194+
o.Expect(workerMC.ResourceVersion).To(o.Equal(initialResourceVersion), "Machine config %s should not be updated when failSwapOn is rejected", workerGeneratedKubeletMC)
195+
framework.Logf("Verified: %s was not updated (resourceVersion: %s)", workerGeneratedKubeletMC, workerMC.ResourceVersion)
196+
}
197+
198+
g.By("Verifying worker nodes still have correct swap settings")
199+
allWorkerNodes, err := getNodesByLabel(ctx, oc, "node-role.kubernetes.io/worker")
200+
o.Expect(err).NotTo(o.HaveOccurred())
201+
o.Expect(len(allWorkerNodes)).Should(o.BeNumerically(">", 0), "Expected at least one worker node")
202+
203+
// Filter out nodes that are also control plane (e.g., SNO)
204+
workerNodes := getPureWorkerNodes(allWorkerNodes)
205+
206+
for _, node := range workerNodes {
207+
config, err := getKubeletConfigFromNode(ctx, oc, node.Name)
208+
o.Expect(err).NotTo(o.HaveOccurred(), "Failed to get kubelet config for worker node %s", node.Name)
209+
210+
g.By(fmt.Sprintf("Verifying failSwapOn=false remains unchanged on worker node %s", node.Name))
211+
o.Expect(config.FailSwapOn).NotTo(o.BeNil(), "failSwapOn should be set on worker node %s", node.Name)
212+
o.Expect(*config.FailSwapOn).To(o.BeFalse(), "failSwapOn should still be false on worker node %s after rejection", node.Name)
213+
framework.Logf("Worker node %s: failSwapOn=%v (unchanged) ✓", node.Name, *config.FailSwapOn)
214+
215+
g.By(fmt.Sprintf("Verifying swapBehavior=NoSwap remains unchanged on worker node %s", node.Name))
216+
o.Expect(config.MemorySwap).NotTo(o.BeNil(), "memorySwap should be set on worker node %s", node.Name)
217+
o.Expect(config.MemorySwap.SwapBehavior).To(o.Equal("NoSwap"), "swapBehavior should still be NoSwap on worker node %s after rejection", node.Name)
218+
framework.Logf("Worker node %s: swapBehavior=%s (unchanged) ✓", node.Name, config.MemorySwap.SwapBehavior)
219+
}
220+
221+
framework.Logf("Test PASSED: KubeletConfig with failSwapOn was properly rejected")
222+
})
223+
})

0 commit comments

Comments
 (0)