WIP: cts-lab cleanup

cts/cts-attrd.in

@@ -7,7 +7,7 @@
 # pacemaker imports need to come after we modify sys.path, which pylint will complain about.
 # pylint: disable=wrong-import-position
 
-__copyright__ = "Copyright 2023-2025 the Pacemaker project contributors"
+__copyright__ = "Copyright 2023-2026 the Pacemaker project contributors"
 __license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"
 
 import argparse
@@ -81,7 +81,7 @@ class AttributeTests(Tests):
     def setup_environment(self, use_corosync):
         """Prepare the host before executing any tests."""
         if use_corosync:
-            self._corosync.start(kill_first=True)
+            self._corosync.start()
 
     def cleanup_environment(self, use_corosync):
         """Clean up the host after executing desired tests."""

cts/cts-exec.in

@@ -7,7 +7,7 @@
 # pacemaker imports need to come after we modify sys.path, which pylint will complain about.
 # pylint: disable=wrong-import-position
 
-__copyright__ = "Copyright 2012-2025 the Pacemaker project contributors"
+__copyright__ = "Copyright 2012-2026 the Pacemaker project contributors"
 __license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"
 
 import argparse
@@ -87,7 +87,7 @@ class ExecTest(Test):
 
     def _start_daemons(self):
         if self._corosync:
-            self._corosync.start(kill_first=True)
+            self._corosync.start()
             # pylint: disable=consider-using-with
             self._fencer = subprocess.Popen(["pacemaker-fenced", "-s"])
 

cts/cts-fencing.in

@@ -7,7 +7,7 @@
 # pacemaker imports need to come after we modify sys.path, which pylint will complain about.
 # pylint: disable=wrong-import-position
 
-__copyright__ = "Copyright 2012-2025 the Pacemaker project contributors"
+__copyright__ = "Copyright 2012-2026 the Pacemaker project contributors"
 __license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"
 
 import argparse
@@ -819,7 +819,7 @@ class FenceTests(Tests):
 
     def setup_environment(self):
         """Prepare the host before executing any tests."""
-        self._corosync.start(kill_first=True)
+        self._corosync.start()
         subprocess.call(["cts-support", "install"])
 
     def cleanup_environment(self):

cts/cts-lab.in

@@ -9,6 +9,7 @@
 __copyright__ = "Copyright 2001-2026 the Pacemaker project contributors"
 __license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"
 
+from functools import partial
 import signal
 import sys
 
@@ -20,42 +21,29 @@ from pacemaker._cts import logging
 from pacemaker._cts.scenarios import AllOnce, Boot, BootCluster, LeaveBooted, RandomTests, Sequence
 from pacemaker._cts.tests import test_list
 
-# These are globals so they can be used by the signal handler.
-scenario = None
-logging.add_stderr()
 
-
-def sig_handler(signum, _frame):
-    """Handle the given signal number."""
+def term_handler(scenario, signum, _frame):
+    """Handle a SIGTERM by exiting gracefully."""
     logging.log(f"Interrupted by signal {signum}")
 
     if scenario:
         scenario.summarize()
+        scenario.teardown()
 
-    if signum == 15:
-        if scenario:
-            scenario.teardown()
-
-        sys.exit(1)
+    sys.exit(1)
 
 
 def plural_s(n):
     """Return a string suffix depending on whether or not n is > 1."""
-    if n == 1:
-        return ""
-
-    return "S"
+    return "" if n == 1 else "S"
 
 
 if __name__ == '__main__':
+    logging.add_stderr()
+
     env = Environment(sys.argv[1:])
     lab = CtsLab(env)
     iters = lab["iterations"]
-    tests = []
-
-    # Set the signal handler
-    signal.signal(15, sig_handler)
-    signal.signal(10, sig_handler)
 
     # Create the Cluster Manager object.
     # Currently Corosync2 is the only available cluster manager.
@@ -71,33 +59,26 @@ if __name__ == '__main__':
                 outputfile.truncate(0)
 
     audits = audit_list(cm)
+    tests = test_list(cm, env, audits)
 
     if lab["ListTests"]:
-        tests = test_list(cm, env, audits)
         logging.log(f"Total {len(tests)} tests")
 
         for test in tests:
             logging.log(test.name)
 
         sys.exit(0)
 
-    elif len(lab["tests"]) == 0:
-        tests = test_list(cm, env, audits)
+    if lab["tests"]:
+        # Get tests corresponding to the names in lab["tests"]
+        test_map = {test.name: test for test in tests}
 
-    else:
-        chosen = lab["tests"]
-        for test_case in chosen:
-            match = None
+        try:
+            tests = [test_map[test] for test in lab["tests"]]
 
-            for test in test_list(cm, env, audits):
-                if test.name == test_case:
-                    match = test
-
-            if not match:
-                logging.log("--choose: No applicable/valid tests chosen")
-                sys.exit(1)
-            else:
-                tests.append(match)
+        except KeyError as e:
+            logging.log(f"--choose: Test {e} is invalid or not applicable")
+            sys.exit(1)
 
     # Scenario selection
     if lab["scenario"] == "all-once":
@@ -115,8 +96,10 @@ if __name__ == '__main__':
     logging.log(f"Scenario:               {scenario.__doc__}")
     logging.log(f"CTS Exerciser:          {lab['cts-exerciser']}")
     logging.log(f"CTS Logfile:            {lab['OutputFile']}")
+
     if "syslogd" in lab:
         logging.log(f"Syslog variant:         {lab['syslogd']}")
+
     logging.log(f"System log files:       {lab['LogFileName']}")
 
     if "IPBase" in lab:
@@ -125,6 +108,10 @@ if __name__ == '__main__':
     logging.log(f"Cluster starts at boot: {lab['at-boot']}")
 
     lab.dump()
+
+    # On SIGTERM during lab run, print a summary and do teardown
+    signal.signal(15, partial(term_handler, scenario))
+
     rc = lab.run(scenario, iters)
     sys.exit(rc)
 

python/pacemaker/_cts/audits.py

@@ -47,10 +47,6 @@ def is_applicable(self):
         """
         raise NotImplementedError
 
-    def log(self, args):
-        """Log a message."""
-        logging.log(f"audit: {args}")
-
     def debug(self, args):
         """Log a debug message."""
         logging.debug(f"audit: {args}")
@@ -74,14 +70,11 @@ def __init__(self, cm):
         ClusterAudit.__init__(self, cm)
         self.name = "LogAudit"
 
-    def _restart_cluster_logging(self, nodes=None):
-        """Restart logging on the given nodes, or all if none are given."""
-        if not nodes:
-            nodes = self._cm.env["nodes"]
-
-        logging.debug(f"Restarting logging on: {nodes!r}")
+    def _restart_cluster_logging(self):
+        """Restart logging on all nodes."""
+        logging.debug("Restarting logging on all nodes")
 
-        for node in nodes:
+        for node in self._cm.env["nodes"]:
             if self._cm.env["have_systemd"]:
                 (rc, _) = self._cm.rsh.call(node, "systemctl stop systemd-journald.socket")
                 if rc != 0:
@@ -283,8 +276,10 @@ def _find_core_with_coredumpctl(self, node):
         (_, lsout) = self._cm.rsh.call(node, "coredumpctl --no-legend --no-pager")
         return self._output_has_core(lsout, node)
 
-    def _find_core_on_fs(self, node, paths):
-        """Check for core dumps on the given node, under any of the given paths."""
+    def _find_core_on_fs(self, node):
+        """Check for Pacemaker and Corosync core dumps on the given node."""
+        paths = ["/var/lib/pacemaker/cores/*", "/var/lib/corosync"]
+
         (_, lsout) = self._cm.rsh.call(node, f"ls -al {' '.join(paths)} | grep core.[0-9]",
                                        verbose=1)
         return self._output_has_core(lsout, node)
@@ -311,8 +306,7 @@ def __call__(self):
             #
             # To handle the last two cases, check the other filesystem locations.
             if not found:
-                found = self._find_core_on_fs(node, ["/var/lib/pacemaker/cores/*",
-                                                     "/var/lib/corosync"])
+                found = self._find_core_on_fs(node)
                 if found:
                     passed = False
 
@@ -449,43 +443,49 @@ def __init__(self, cm):
 
     def _audit_resource(self, resource, quorum):
         """Perform the audit of a single resource."""
-        rc = True
         active = self._cm.resource_location(resource.id)
 
         if len(active) == 1:
             if quorum:
                 self.debug(f"Resource {resource.id} active on {active!r}")
+                return True
 
-            elif resource.needs_quorum == 1:
+            if resource.needs_quorum == 1:
                 logging.log(f"Resource {resource.id} active without quorum: {active!r}")
-                rc = False
+                return False
 
-        elif not resource.managed:
+            return True
+
+        if not resource.managed:
             logging.log(f"Resource {resource.id} not managed. Active on {active!r}")
+            return True
 
-        elif not resource.unique:
+        if not resource.unique:
             # TODO: Figure out a clever way to actually audit these resource types
             if len(active) > 1:
                 self.debug(f"Non-unique resource {resource.id} is active on: {active!r}")
             else:
                 self.debug(f"Non-unique resource {resource.id} is not active")
 
-        elif len(active) > 1:
+            return True
+
+        if len(active) > 1:
             logging.log(f"Resource {resource.id} is active multiple times: {active!r}")
-            rc = False
+            return False
 
-        elif resource.orphan:
+        if resource.orphan:
             self.debug(f"Resource {resource.id} is an inactive orphan")
+            return True
 
-        elif not self._inactive_nodes:
+        if not self._inactive_nodes:
             logging.log(f"WARN: Resource {resource.id} not served anywhere")
-            rc = False
+            return False
 
-        elif quorum or not resource.needs_quorum:
+        if quorum or not resource.needs_quorum:
             self.debug(f"Resource {resource.id} not served anywhere "
                        f"(Inactive nodes: {self._inactive_nodes!r})")
 
-        return rc
+        return True
 
     def _setup(self):
         """
@@ -530,7 +530,7 @@ def __call__(self):
             return passed
 
         primitives = [r for r in self._resources if r.type == "primitive"]
-        quorum = self._cm.has_quorum(None)
+        quorum = self._cm.has_quorum()
 
         for primitive in primitives:
             if not self._audit_resource(primitive, quorum):