From d398e5b4cafef8e43b194d98f737ce7bd43a94d0 Mon Sep 17 00:00:00 2001 From: xueqzhan Date: Thu, 9 Jul 2026 16:08:07 -0400 Subject: [PATCH] Create more test bucket to debug high cpu issues --- pkg/test/ginkgo/cmd_runsuite.go | 58 ++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/pkg/test/ginkgo/cmd_runsuite.go b/pkg/test/ginkgo/cmd_runsuite.go index 7d011e9c725e..3e2b3a3c0c69 100644 --- a/pkg/test/ginkgo/cmd_runsuite.go +++ b/pkg/test/ginkgo/cmd_runsuite.go @@ -583,6 +583,18 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc return strings.Contains(t.name, "OrderedNamespaceDeletion") }) + draTests, kubeTests := splitTests(kubeTests, func(t *testCase) bool { + return strings.Contains(t.name, "[DRA]") + }) + + inPlaceResizeTests, kubeTests := splitTests(kubeTests, func(t *testCase) bool { + return strings.Contains(t.name, "InPlace Resize") + }) + + probingTests, kubeTests := splitTests(kubeTests, func(t *testCase) bool { + return strings.Contains(t.name, "Probing container") || strings.Contains(t.name, "Probing restartable init container") + }) + networkTests, openshiftTests := splitTests(openshiftTests, func(t *testCase) bool { return strings.Contains(t.name, "[sig-network]") }) @@ -600,6 +612,10 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc return strings.Contains(t.name, "[sig-cli] oc adm must-gather") }) + deploymentConfigTests, openshiftTests := splitTests(openshiftTests, func(t *testCase) bool { + return strings.Contains(t.name, "[Feature:DeploymentConfig]") + }) + logrus.Infof("Found %d openshift tests", len(openshiftTests)) logrus.Infof("Found %d kubernetes tests", len(kubeTests)) logrus.Infof("Found %d storage tests", len(storageTests)) @@ -609,6 +625,10 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc logrus.Infof("Found %d netpol tests", len(netpolTests)) logrus.Infof("Found %d builds tests", len(buildsTests)) logrus.Infof("Found %d must-gather tests", len(mustGatherTests)) + logrus.Infof("Found %d DeploymentConfig tests", len(deploymentConfigTests)) + logrus.Infof("Found %d DRA tests", len(draTests)) + logrus.Infof("Found %d InPlace Resize tests", len(inPlaceResizeTests)) + logrus.Infof("Found %d Probing tests", len(probingTests)) // If user specifies a count, duplicate the kube and openshift tests that many times. expectedTestCount := len(early) + len(late) @@ -622,6 +642,10 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc originalNetpol := netpolTests originalBuilds := buildsTests originalMustGather := mustGatherTests + originalDRA := draTests + originalInPlaceResize := inPlaceResizeTests + originalProbing := probingTests + originalDeploymentConfig := deploymentConfigTests for i := 1; i < count; i++ { kubeTests = append(kubeTests, copyTests(originalKube)...) @@ -633,9 +657,13 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc netpolTests = append(netpolTests, copyTests(originalNetpol)...) buildsTests = append(buildsTests, copyTests(originalBuilds)...) mustGatherTests = append(mustGatherTests, copyTests(originalMustGather)...) + draTests = append(draTests, copyTests(originalDRA)...) + inPlaceResizeTests = append(inPlaceResizeTests, copyTests(originalInPlaceResize)...) + probingTests = append(probingTests, copyTests(originalProbing)...) + deploymentConfigTests = append(deploymentConfigTests, copyTests(originalDeploymentConfig)...) } } - expectedTestCount += len(openshiftTests) + len(kubeTests) + len(storageTests) + len(networkK8sTests) + len(orderedNamespaceDeletionTests) + len(networkTests) + len(netpolTests) + len(buildsTests) + len(mustGatherTests) + expectedTestCount += len(openshiftTests) + len(kubeTests) + len(storageTests) + len(networkK8sTests) + len(orderedNamespaceDeletionTests) + len(networkTests) + len(netpolTests) + len(buildsTests) + len(mustGatherTests) + len(draTests) + len(inPlaceResizeTests) + len(probingTests) + len(deploymentConfigTests) abortFn := neverAbort testCtx := ctx @@ -667,6 +695,27 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc logrus.Infof("Completed Kubernetes test bucket in %v", time.Since(kubeStartTime)) tests = append(tests, kubeTestsCopy...) + draTestsCopy := copyTests(draTests) + draIntervalID, draStartTime := recordTestBucketInterval(monitorEventRecorder, "DRA") + q.Execute(testCtx, draTestsCopy, parallelism, testOutputConfig, abortFn) + monitorEventRecorder.EndInterval(draIntervalID, time.Now()) + logrus.Infof("Completed DRA test bucket in %v", time.Since(draStartTime)) + tests = append(tests, draTestsCopy...) + + inPlaceResizeTestsCopy := copyTests(inPlaceResizeTests) + inPlaceResizeIntervalID, inPlaceResizeStartTime := recordTestBucketInterval(monitorEventRecorder, "InPlaceResize") + q.Execute(testCtx, inPlaceResizeTestsCopy, parallelism, testOutputConfig, abortFn) + monitorEventRecorder.EndInterval(inPlaceResizeIntervalID, time.Now()) + logrus.Infof("Completed InPlaceResize test bucket in %v", time.Since(inPlaceResizeStartTime)) + tests = append(tests, inPlaceResizeTestsCopy...) + + probingTestsCopy := copyTests(probingTests) + probingIntervalID, probingStartTime := recordTestBucketInterval(monitorEventRecorder, "Probing") + q.Execute(testCtx, probingTestsCopy, parallelism, testOutputConfig, abortFn) + monitorEventRecorder.EndInterval(probingIntervalID, time.Now()) + logrus.Infof("Completed Probing test bucket in %v", time.Since(probingStartTime)) + tests = append(tests, probingTestsCopy...) + // I thought about randomizing the order of the kube, storage, and openshift tests, but storage dominates our e2e runs, so it doesn't help much. storageTestsCopy := copyTests(storageTests) storageIntervalID, storageStartTime := recordTestBucketInterval(monitorEventRecorder, "Storage") @@ -721,6 +770,13 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc logrus.Infof("Completed OpenShift test bucket in %v", time.Since(openshiftStartTime)) tests = append(tests, openshiftTestsCopy...) + deploymentConfigTestsCopy := copyTests(deploymentConfigTests) + deploymentConfigIntervalID, deploymentConfigStartTime := recordTestBucketInterval(monitorEventRecorder, "DeploymentConfig") + q.Execute(testCtx, deploymentConfigTestsCopy, parallelism, testOutputConfig, abortFn) + monitorEventRecorder.EndInterval(deploymentConfigIntervalID, time.Now()) + logrus.Infof("Completed DeploymentConfig test bucket in %v", time.Since(deploymentConfigStartTime)) + tests = append(tests, deploymentConfigTestsCopy...) + // run the must-gather tests after parallel tests to reduce resource contention mustGatherTestsCopy := copyTests(mustGatherTests) mustGatherIntervalID, mustGatherStartTime := recordTestBucketInterval(monitorEventRecorder, "MustGather")