From 6fbba90c22c61d1b9b179e6d3bd5f8e98e37a145 Mon Sep 17 00:00:00 2001 From: Reid Wahl Date: Sun, 9 Aug 2026 11:37:38 -0700 Subject: [PATCH 01/25] Refactor: cts: Simplify a couple of if/else blocks in cts-lab.in Signed-off-by: Reid Wahl --- cts/cts-lab.in | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/cts/cts-lab.in b/cts/cts-lab.in index 2031c172db3..952b0e1f289 100644 --- a/cts/cts-lab.in +++ b/cts/cts-lab.in @@ -20,8 +20,9 @@ from pacemaker._cts import logging from pacemaker._cts.scenarios import AllOnce, Boot, BootCluster, LeaveBooted, RandomTests, Sequence from pacemaker._cts.tests import test_list -# These are globals so they can be used by the signal handler. +# This is a global so it can be used by the signal handler scenario = None + logging.add_stderr() @@ -41,10 +42,7 @@ def sig_handler(signum, _frame): def plural_s(n): """Return a string suffix depending on whether or not n is > 1.""" - if n == 1: - return "" - - return "S" + return "" if n == 1 else "S" if __name__ == '__main__': @@ -81,10 +79,7 @@ if __name__ == '__main__': sys.exit(0) - elif len(lab["tests"]) == 0: - tests = test_list(cm, env, audits) - - else: + if lab["tests"]: chosen = lab["tests"] for test_case in chosen: match = None @@ -96,8 +91,11 @@ if __name__ == '__main__': if not match: logging.log("--choose: No applicable/valid tests chosen") sys.exit(1) - else: - tests.append(match) + + tests.append(match) + + else: + tests = test_list(cm, env, audits) # Scenario selection if lab["scenario"] == "all-once": From e3a5a48616daf20ea5eaea7366e841681f84e608 Mon Sep 17 00:00:00 2001 From: Reid Wahl Date: Sun, 9 Aug 2026 12:15:56 -0700 Subject: [PATCH 02/25] Low: cts: Call test_list() only once When --choose is given many tests, we call test_list() for each of those tests. That adds a significant delay for all the redundant work, which requires SSHing to all the systems in the test bed. Lately I've been running with --choose set to 15 instances of RemoteMigrate, and there's a delay of a couple of minutes for all the test_list() work. The change in scenarios.py keeps the stats correct. Previously each test in self.tests was a separate instance that came from a separate test_list() call, so each one had its own stats. It made sense to sum up the stats for all instances of the same test. However, now if there are multiple tests with the same name in self.tests, each of them is the same CTSTest object. So summing them up give incorrect results. At a glance, the debug-level stats for multiple tests with the same name may have been wrong before this commit. Signed-off-by: Reid Wahl --- cts/cts-lab.in | 24 ++++++++---------------- python/pacemaker/_cts/scenarios.py | 22 +++++++++------------- 2 files changed, 17 insertions(+), 29 deletions(-) diff --git a/cts/cts-lab.in b/cts/cts-lab.in index 952b0e1f289..6caf34fcf68 100644 --- a/cts/cts-lab.in +++ b/cts/cts-lab.in @@ -49,7 +49,6 @@ if __name__ == '__main__': env = Environment(sys.argv[1:]) lab = CtsLab(env) iters = lab["iterations"] - tests = [] # Set the signal handler signal.signal(15, sig_handler) @@ -69,9 +68,9 @@ if __name__ == '__main__': outputfile.truncate(0) audits = audit_list(cm) + tests = test_list(cm, env, audits) if lab["ListTests"]: - tests = test_list(cm, env, audits) logging.log(f"Total {len(tests)} tests") for test in tests: @@ -80,22 +79,15 @@ if __name__ == '__main__': sys.exit(0) if lab["tests"]: - chosen = lab["tests"] - for test_case in chosen: - match = None + # Get tests corresponding to the names in lab["tests"] + test_map = {test.name: test for test in tests} - for test in test_list(cm, env, audits): - if test.name == test_case: - match = test + try: + tests = [test_map[test] for test in lab["tests"]] - if not match: - logging.log("--choose: No applicable/valid tests chosen") - sys.exit(1) - - tests.append(match) - - else: - tests = test_list(cm, env, audits) + except KeyError as e: + logging.log(f"--choose: Test {e} is invalid or not applicable") + sys.exit(1) # Scenario selection if lab["scenario"] == "all-once": diff --git a/python/pacemaker/_cts/scenarios.py b/python/pacemaker/_cts/scenarios.py index 13b00b060ec..cdd22600c90 100644 --- a/python/pacemaker/_cts/scenarios.py +++ b/python/pacemaker/_cts/scenarios.py @@ -246,27 +246,23 @@ def run_test(self, test, testcount): def summarize(self): """Output scenario results.""" + # This dict removes duplicates in self.tests while preserving order + tests = {test.name: test for test in self.tests} + + summary_keys = ["calls", "failure", "skipped", "auditfail"] + logging.log("****************") logging.log("Overall Results:%r" % self.stats) logging.log("****************") - stat_summary = {} - summary_keys = ["calls", "failure", "skipped", "auditfail"] - logging.log("Test Summary") - for test in self.tests: - if test.name not in stat_summary: - stat_summary[test.name] = {key: 0 for key in summary_keys} - - for key in summary_keys: - stat_summary[test.name][key] += test.stats[key] - - for (name, summary) in stat_summary.items(): + for (name, test) in tests.items(): + summary = {key: test.stats[key] for key in summary_keys} logging.log(f"{f'Test {name}':<25} {summary!r}") logging.debug("Detailed Results") - for test in self.tests: - logging.debug(f"{f'Test {test.name}: ':<25} {test.stats!r}") + for (name, test) in tests.items(): + logging.debug(f"{f'Test {name}: ':<25} {test.stats!r}") logging.log("<<<<<<<<<<<<<<<< TESTS COMPLETED") From bde313b2eef5429d72098c8814eda4fd1a26e056 Mon Sep 17 00:00:00 2001 From: Reid Wahl Date: Sun, 9 Aug 2026 12:40:05 -0700 Subject: [PATCH 03/25] Refactor: cts: Don't set signal handler for SIGUSR1 We started handling SIGUSR1 in 2005 with commit 57e4d8f3. The handler just prints a summary. I don't see any reason why we need a summary until the cts-lab run is finished. The normal output and debug logging give plenty of info. Signed-off-by: Reid Wahl --- cts/cts-lab.in | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/cts/cts-lab.in b/cts/cts-lab.in index 6caf34fcf68..1bb285be159 100644 --- a/cts/cts-lab.in +++ b/cts/cts-lab.in @@ -26,18 +26,15 @@ scenario = None logging.add_stderr() -def sig_handler(signum, _frame): - """Handle the given signal number.""" +def term_handler(signum, _frame): + """Handle a SIGTERM by exiting gracefully.""" logging.log(f"Interrupted by signal {signum}") if scenario: scenario.summarize() + scenario.teardown() - if signum == 15: - if scenario: - scenario.teardown() - - sys.exit(1) + sys.exit(1) def plural_s(n): @@ -51,8 +48,7 @@ if __name__ == '__main__': iters = lab["iterations"] # Set the signal handler - signal.signal(15, sig_handler) - signal.signal(10, sig_handler) + signal.signal(15, term_handler) # Create the Cluster Manager object. # Currently Corosync2 is the only available cluster manager. From 2d7297aa65969062ee76e49a06ca9df1308819ce Mon Sep 17 00:00:00 2001 From: Reid Wahl Date: Sun, 9 Aug 2026 12:54:36 -0700 Subject: [PATCH 04/25] Refactor: cts: Drop global scenario variable Note that the signal handler isn't really useful until we start the lab run. Signed-off-by: Reid Wahl --- cts/cts-lab.in | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/cts/cts-lab.in b/cts/cts-lab.in index 1bb285be159..1aaf0ddd7dd 100644 --- a/cts/cts-lab.in +++ b/cts/cts-lab.in @@ -9,6 +9,7 @@ __copyright__ = "Copyright 2001-2026 the Pacemaker project contributors" __license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY" +from functools import partial import signal import sys @@ -20,13 +21,10 @@ from pacemaker._cts import logging from pacemaker._cts.scenarios import AllOnce, Boot, BootCluster, LeaveBooted, RandomTests, Sequence from pacemaker._cts.tests import test_list -# This is a global so it can be used by the signal handler -scenario = None - logging.add_stderr() -def term_handler(signum, _frame): +def term_handler(scenario, signum, _frame): """Handle a SIGTERM by exiting gracefully.""" logging.log(f"Interrupted by signal {signum}") @@ -47,9 +45,6 @@ if __name__ == '__main__': lab = CtsLab(env) iters = lab["iterations"] - # Set the signal handler - signal.signal(15, term_handler) - # Create the Cluster Manager object. # Currently Corosync2 is the only available cluster manager. cm = Corosync2(env) @@ -101,8 +96,10 @@ if __name__ == '__main__': logging.log(f"Scenario: {scenario.__doc__}") logging.log(f"CTS Exerciser: {lab['cts-exerciser']}") logging.log(f"CTS Logfile: {lab['OutputFile']}") + if "syslogd" in lab: logging.log(f"Syslog variant: {lab['syslogd']}") + logging.log(f"System log files: {lab['LogFileName']}") if "IPBase" in lab: @@ -111,6 +108,10 @@ if __name__ == '__main__': logging.log(f"Cluster starts at boot: {lab['at-boot']}") lab.dump() + + # On SIGTERM during lab run, print a summary and do teardown + signal.signal(15, partial(term_handler, scenario)) + rc = lab.run(scenario, iters) sys.exit(rc) From a7298e83229e089897d0ca301d1f931798c790d3 Mon Sep 17 00:00:00 2001 From: Reid Wahl Date: Sun, 9 Aug 2026 12:59:38 -0700 Subject: [PATCH 05/25] Refactor: cts: Move add_stderr() call to the '__main__' block Signed-off-by: Reid Wahl --- cts/cts-lab.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cts/cts-lab.in b/cts/cts-lab.in index 1aaf0ddd7dd..98a353d88b1 100644 --- a/cts/cts-lab.in +++ b/cts/cts-lab.in @@ -21,8 +21,6 @@ from pacemaker._cts import logging from pacemaker._cts.scenarios import AllOnce, Boot, BootCluster, LeaveBooted, RandomTests, Sequence from pacemaker._cts.tests import test_list -logging.add_stderr() - def term_handler(scenario, signum, _frame): """Handle a SIGTERM by exiting gracefully.""" @@ -41,6 +39,8 @@ def plural_s(n): if __name__ == '__main__': + logging.add_stderr() + env = Environment(sys.argv[1:]) lab = CtsLab(env) iters = lab["iterations"] From 9b2b84463129d0f87b7bab7ef2f118ffebf24be0 Mon Sep 17 00:00:00 2001 From: Reid Wahl Date: Sun, 9 Aug 2026 13:09:17 -0700 Subject: [PATCH 06/25] Refactor: cts: Drop ClusterManager.startall() nodelist argument The sole caller passes the default. Signed-off-by: Reid Wahl --- python/pacemaker/_cts/clustermanager.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/python/pacemaker/_cts/clustermanager.py b/python/pacemaker/_cts/clustermanager.py index 4794e5e4840..2c6a9c42c1e 100644 --- a/python/pacemaker/_cts/clustermanager.py +++ b/python/pacemaker/_cts/clustermanager.py @@ -332,10 +332,9 @@ def stop_cm_async(self, node): self.rsh.call_async(node, self.templates["StopCmd"]) self.expected_status[node] = "down" - def startall(self, nodelist=None, verbose=False, quick=False): - """Start the cluster manager on every node in the cluster, or on every node in nodelist.""" - if not nodelist: - nodelist = self.env["nodes"] + def startall(self, verbose=False, quick=False): + """Start the cluster manager on every node in the cluster.""" + nodelist = self.env["nodes"] for node in nodelist: if self.expected_status[node] == "down": From b78ff8fb5159548f497afd302a4aee97303d5bcf Mon Sep 17 00:00:00 2001 From: Reid Wahl Date: Sun, 9 Aug 2026 13:02:17 -0700 Subject: [PATCH 07/25] Refactor: cts: Drop ClusterManager.startall() verbose argument The sole caller passes True. Signed-off-by: Reid Wahl --- python/pacemaker/_cts/clustermanager.py | 8 ++++---- python/pacemaker/_cts/scenarios.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/pacemaker/_cts/clustermanager.py b/python/pacemaker/_cts/clustermanager.py index 2c6a9c42c1e..e61e5395507 100644 --- a/python/pacemaker/_cts/clustermanager.py +++ b/python/pacemaker/_cts/clustermanager.py @@ -332,7 +332,7 @@ def stop_cm_async(self, node): self.rsh.call_async(node, self.templates["StopCmd"]) self.expected_status[node] = "down" - def startall(self, verbose=False, quick=False): + def startall(self, quick=False): """Start the cluster manager on every node in the cluster.""" nodelist = self.env["nodes"] @@ -342,7 +342,7 @@ def startall(self, verbose=False, quick=False): if not quick: # This is used for "basic sanity checks", so only start one node ... - return self.start_cm(nodelist[0], verbose=verbose) + return self.start_cm(nodelist[0], verbose=True) # Approximation of SimulStartList for --boot watchpats = [ @@ -362,11 +362,11 @@ def startall(self, verbose=False, quick=False): self.env["dead_time"] + 10) watch.set_watch() - if not self.start_cm(nodelist[0], verbose=verbose): + if not self.start_cm(nodelist[0], verbose=True): return False for node in nodelist: - self.start_cm_async(node, verbose=verbose) + self.start_cm_async(node, verbose=True) watch.look_for_all() if watch.unmatched: diff --git a/python/pacemaker/_cts/scenarios.py b/python/pacemaker/_cts/scenarios.py index cdd22600c90..dd1284bf79f 100644 --- a/python/pacemaker/_cts/scenarios.py +++ b/python/pacemaker/_cts/scenarios.py @@ -389,7 +389,7 @@ def setup(self): # Now start the Cluster Manager on all the nodes. logging.log("Starting Cluster Manager on all nodes.") - return self._cm.startall(verbose=True, quick=True) + return self._cm.startall(quick=True) def teardown(self): """Tear down the component.""" From 93a6d394e512c95077653909f2e045e9f448da15 Mon Sep 17 00:00:00 2001 From: Reid Wahl Date: Sun, 9 Aug 2026 13:11:32 -0700 Subject: [PATCH 08/25] Refactor: cts: Drop ClusterManager.startall() quick argument The sole caller passes True. Signed-off-by: Reid Wahl --- python/pacemaker/_cts/clustermanager.py | 6 +----- python/pacemaker/_cts/scenarios.py | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/python/pacemaker/_cts/clustermanager.py b/python/pacemaker/_cts/clustermanager.py index e61e5395507..792b22b856f 100644 --- a/python/pacemaker/_cts/clustermanager.py +++ b/python/pacemaker/_cts/clustermanager.py @@ -332,7 +332,7 @@ def stop_cm_async(self, node): self.rsh.call_async(node, self.templates["StopCmd"]) self.expected_status[node] = "down" - def startall(self, quick=False): + def startall(self): """Start the cluster manager on every node in the cluster.""" nodelist = self.env["nodes"] @@ -340,10 +340,6 @@ def startall(self, quick=False): if self.expected_status[node] == "down": self.ns.wait_for_all_nodes(nodelist, 300) - if not quick: - # This is used for "basic sanity checks", so only start one node ... - return self.start_cm(nodelist[0], verbose=True) - # Approximation of SimulStartList for --boot watchpats = [ self.templates["Pat:DC_IDLE"], diff --git a/python/pacemaker/_cts/scenarios.py b/python/pacemaker/_cts/scenarios.py index dd1284bf79f..d99d4851328 100644 --- a/python/pacemaker/_cts/scenarios.py +++ b/python/pacemaker/_cts/scenarios.py @@ -389,7 +389,7 @@ def setup(self): # Now start the Cluster Manager on all the nodes. logging.log("Starting Cluster Manager on all nodes.") - return self._cm.startall(quick=True) + return self._cm.startall() def teardown(self): """Tear down the component.""" From 44912282c5f2a1805a3de07537d242f2c2222ebd Mon Sep 17 00:00:00 2001 From: Reid Wahl Date: Sun, 9 Aug 2026 13:12:28 -0700 Subject: [PATCH 09/25] Refactor: cts: Drop ClusterManager:stopall() nodelist argument The sole caller passes the default. Signed-off-by: Reid Wahl --- python/pacemaker/_cts/clustermanager.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/python/pacemaker/_cts/clustermanager.py b/python/pacemaker/_cts/clustermanager.py index 792b22b856f..01200a3cfb5 100644 --- a/python/pacemaker/_cts/clustermanager.py +++ b/python/pacemaker/_cts/clustermanager.py @@ -375,13 +375,10 @@ def startall(self): return True - def stopall(self, nodelist=None, verbose=False, force=False): - """Stop the cluster manager on every node in the cluster, or on every node in nodelist.""" + def stopall(self, verbose=False, force=False): + """Stop the cluster manager on every node in the cluster.""" ret = True - if not nodelist: - nodelist = self.env["nodes"] - for node in self.env["nodes"]: if self.expected_status[node] == "up" or force: if not self.stop_cm(node, verbose=verbose, force=force): From 138a59ee1cd1d2de1d46a6bda30ee987a9964a8b Mon Sep 17 00:00:00 2001 From: Reid Wahl Date: Sun, 9 Aug 2026 13:14:10 -0700 Subject: [PATCH 10/25] Refactor: cts: Drop ClusterManager.stopall() verbose argument Both callers pass True. Signed-off-by: Reid Wahl --- python/pacemaker/_cts/clustermanager.py | 4 ++-- python/pacemaker/_cts/scenarios.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/pacemaker/_cts/clustermanager.py b/python/pacemaker/_cts/clustermanager.py index 01200a3cfb5..5f32599cf81 100644 --- a/python/pacemaker/_cts/clustermanager.py +++ b/python/pacemaker/_cts/clustermanager.py @@ -375,13 +375,13 @@ def startall(self): return True - def stopall(self, verbose=False, force=False): + def stopall(self, force=False): """Stop the cluster manager on every node in the cluster.""" ret = True for node in self.env["nodes"]: if self.expected_status[node] == "up" or force: - if not self.stop_cm(node, verbose=verbose, force=force): + if not self.stop_cm(node, verbose=True, force=force): ret = False return ret diff --git a/python/pacemaker/_cts/scenarios.py b/python/pacemaker/_cts/scenarios.py index d99d4851328..abaf3098d9c 100644 --- a/python/pacemaker/_cts/scenarios.py +++ b/python/pacemaker/_cts/scenarios.py @@ -385,7 +385,7 @@ def setup(self): self._cm.prepare() # Clear out the cobwebs ;-) - self._cm.stopall(verbose=True, force=True) + self._cm.stopall(force=True) # Now start the Cluster Manager on all the nodes. logging.log("Starting Cluster Manager on all nodes.") @@ -394,7 +394,7 @@ def setup(self): def teardown(self): """Tear down the component.""" logging.log("Stopping Cluster Manager on all nodes") - self._cm.stopall(verbose=True, force=False) + self._cm.stopall(force=False) class LeaveBooted(BootCluster): From ce2b4c4664d0db84338d37a701f0d0574d769242 Mon Sep 17 00:00:00 2001 From: Reid Wahl Date: Sun, 9 Aug 2026 13:24:33 -0700 Subject: [PATCH 11/25] Refactor: cts: Drop ClusterManager:statall() nodelist argument The sole caller passes the default. Signed-off-by: Reid Wahl --- python/pacemaker/_cts/clustermanager.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/python/pacemaker/_cts/clustermanager.py b/python/pacemaker/_cts/clustermanager.py index 5f32599cf81..d6adb0ba50b 100644 --- a/python/pacemaker/_cts/clustermanager.py +++ b/python/pacemaker/_cts/clustermanager.py @@ -386,14 +386,11 @@ def stopall(self, force=False): return ret - def statall(self, nodelist=None): - """Return the status of the cluster manager on every node in the cluster, or on every node in nodelist.""" + def statall(self): + """Return the status of the cluster manager on every node in the cluster.""" result = {} - if not nodelist: - nodelist = self.env["nodes"] - - for node in nodelist: + for node in self.env["nodes"]: if self.stat_cm(node): result[node] = "up" else: From 0c40e0eb898b1ca3a044e400a37409ef4a576ba9 Mon Sep 17 00:00:00 2001 From: Reid Wahl Date: Sun, 9 Aug 2026 13:17:20 -0700 Subject: [PATCH 12/25] Refactor: cts: Drop nodes default in ClusterManager:{un,}isolate_node() The callers pass truthy values. Signed-off-by: Reid Wahl --- python/pacemaker/_cts/clustermanager.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/python/pacemaker/_cts/clustermanager.py b/python/pacemaker/_cts/clustermanager.py index d6adb0ba50b..9075c611414 100644 --- a/python/pacemaker/_cts/clustermanager.py +++ b/python/pacemaker/_cts/clustermanager.py @@ -398,11 +398,8 @@ def statall(self): return result - def isolate_node(self, target, nodes=None): - """Break communication between the target node and all other nodes in the cluster, or nodes.""" - if not nodes: - nodes = self.env["nodes"] - + def isolate_node(self, target, nodes): + """Break communication between the target node the given other nodes.""" for node in nodes: if node == target: continue @@ -416,11 +413,8 @@ def isolate_node(self, target, nodes=None): return True - def unisolate_node(self, target, nodes=None): - """Re-establish communication between the target node and all other nodes in the cluster, or nodes.""" - if not nodes: - nodes = self.env["nodes"] - + def unisolate_node(self, target, nodes): + """Re-establish communication between the target node and the given other nodes.""" for node in nodes: if node == target: continue From dbcdd6c77aeddf3a56688db58315535d5ac96a82 Mon Sep 17 00:00:00 2001 From: Reid Wahl Date: Sun, 9 Aug 2026 13:19:16 -0700 Subject: [PATCH 13/25] Refactor: cts: Drop timeout default in ClusterManager:_partition_stable The callers pass values. Signed-off-by: Reid Wahl --- python/pacemaker/_cts/clustermanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pacemaker/_cts/clustermanager.py b/python/pacemaker/_cts/clustermanager.py index 9075c611414..c6ba3e9ab64 100644 --- a/python/pacemaker/_cts/clustermanager.py +++ b/python/pacemaker/_cts/clustermanager.py @@ -519,7 +519,7 @@ def node_stable(self, node): logging.log(f"Warn: Node {node} not stable") return False - def _partition_stable(self, nodes, timeout=None): + def _partition_stable(self, nodes, timeout): """Return whether or not all nodes in the given partition are stable.""" watchpats = [ "Current ping state: S_IDLE", From e361ae4702529ec532b92a2c6e608f08e8f9b3f5 Mon Sep 17 00:00:00 2001 From: Reid Wahl Date: Sun, 9 Aug 2026 13:30:17 -0700 Subject: [PATCH 14/25] Refactor: cts: Drop ClusterManager.has_quorum node_list argument All callers pass None. Signed-off-by: Reid Wahl --- python/pacemaker/_cts/audits.py | 2 +- python/pacemaker/_cts/clustermanager.py | 15 ++++----------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/python/pacemaker/_cts/audits.py b/python/pacemaker/_cts/audits.py index 2634d6a0878..419443d41d9 100644 --- a/python/pacemaker/_cts/audits.py +++ b/python/pacemaker/_cts/audits.py @@ -530,7 +530,7 @@ def __call__(self): return passed primitives = [r for r in self._resources if r.type == "primitive"] - quorum = self._cm.has_quorum(None) + quorum = self._cm.has_quorum() for primitive in primitives: if not self._audit_resource(primitive, quorum): diff --git a/python/pacemaker/_cts/clustermanager.py b/python/pacemaker/_cts/clustermanager.py index c6ba3e9ab64..7d6cc1b30a6 100644 --- a/python/pacemaker/_cts/clustermanager.py +++ b/python/pacemaker/_cts/clustermanager.py @@ -97,7 +97,7 @@ def prepare_fencing_watcher(self): """Return a LogWatcher object that watches for fencing log messages.""" # If we don't have quorum now but get it as a result of starting this node, # then a bunch of nodes might get fenced - if self.has_quorum(None): + if self.has_quorum(): logging.debug("Have quorum") return None @@ -137,7 +137,7 @@ def fencing_cleanup(self, node, stonith): logging.debug("Nothing to do") return peer_list - q = self.has_quorum(None) + q = self.has_quorum() if not q and len(self.env["nodes"]) > 2: # We didn't gain quorum - we shouldn't have shot anyone logging.debug(f"Quorum: {q} Len: {len(self.env['nodes'])}") @@ -688,16 +688,9 @@ def find_partitions(self): logging.debug(f"Found partitions: {ccm_partitions!r}") return ccm_partitions - def has_quorum(self, node_list): + def has_quorum(self): """Return whether or not the cluster has quorum.""" - # If we are auditing a partition, then one side will - # have quorum and the other not. - # So the caller needs to tell us which we are checking - # If no value for node_list is specified... assume all nodes - if not node_list: - node_list = self.env["nodes"] - - for node in node_list: + for node in self.env["nodes"]: if self.expected_status[node] != "up": continue From 7c60ec9212e639c5158a3601b47d8d7386359089 Mon Sep 17 00:00:00 2001 From: Reid Wahl Date: Sun, 9 Aug 2026 13:34:09 -0700 Subject: [PATCH 15/25] Refactor: cts: Drop ClusterAudit.log() Nothing uses it. Signed-off-by: Reid Wahl --- python/pacemaker/_cts/audits.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/python/pacemaker/_cts/audits.py b/python/pacemaker/_cts/audits.py index 419443d41d9..63ca4d84687 100644 --- a/python/pacemaker/_cts/audits.py +++ b/python/pacemaker/_cts/audits.py @@ -47,10 +47,6 @@ def is_applicable(self): """ raise NotImplementedError - def log(self, args): - """Log a message.""" - logging.log(f"audit: {args}") - def debug(self, args): """Log a debug message.""" logging.debug(f"audit: {args}") From 7312f1760c05f2a36878eb8019d0f1c3c6ac80c1 Mon Sep 17 00:00:00 2001 From: Reid Wahl Date: Sun, 9 Aug 2026 13:36:47 -0700 Subject: [PATCH 16/25] Refactor: cts: Drop the _restart_cluster_logging() nodes argument The sole caller passes the default. Signed-off-by: Reid Wahl --- python/pacemaker/_cts/audits.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/python/pacemaker/_cts/audits.py b/python/pacemaker/_cts/audits.py index 63ca4d84687..65d61132e9d 100644 --- a/python/pacemaker/_cts/audits.py +++ b/python/pacemaker/_cts/audits.py @@ -70,14 +70,11 @@ def __init__(self, cm): ClusterAudit.__init__(self, cm) self.name = "LogAudit" - def _restart_cluster_logging(self, nodes=None): - """Restart logging on the given nodes, or all if none are given.""" - if not nodes: - nodes = self._cm.env["nodes"] + def _restart_cluster_logging(self): + """Restart logging on all nodes.""" + logging.debug("Restarting logging on all nodes") - logging.debug(f"Restarting logging on: {nodes!r}") - - for node in nodes: + for node in self._cm.env["nodes"]: if self._cm.env["have_systemd"]: (rc, _) = self._cm.rsh.call(node, "systemctl stop systemd-journald.socket") if rc != 0: From 3164e4a73f7d39b90425d67d855e1e4d079f25ca Mon Sep 17 00:00:00 2001 From: Reid Wahl Date: Sun, 9 Aug 2026 13:50:44 -0700 Subject: [PATCH 17/25] Refactor: cts: Drop _find_core_on_fs() paths argument Signed-off-by: Reid Wahl --- python/pacemaker/_cts/audits.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/python/pacemaker/_cts/audits.py b/python/pacemaker/_cts/audits.py index 65d61132e9d..82049657537 100644 --- a/python/pacemaker/_cts/audits.py +++ b/python/pacemaker/_cts/audits.py @@ -276,8 +276,10 @@ def _find_core_with_coredumpctl(self, node): (_, lsout) = self._cm.rsh.call(node, "coredumpctl --no-legend --no-pager") return self._output_has_core(lsout, node) - def _find_core_on_fs(self, node, paths): - """Check for core dumps on the given node, under any of the given paths.""" + def _find_core_on_fs(self, node): + """Check for Pacemaker and Corosync core dumps on the given node.""" + paths = ["/var/lib/pacemaker/cores/*", "/var/lib/corosync"] + (_, lsout) = self._cm.rsh.call(node, f"ls -al {' '.join(paths)} | grep core.[0-9]", verbose=1) return self._output_has_core(lsout, node) @@ -304,8 +306,7 @@ def __call__(self): # # To handle the last two cases, check the other filesystem locations. if not found: - found = self._find_core_on_fs(node, ["/var/lib/pacemaker/cores/*", - "/var/lib/corosync"]) + found = self._find_core_on_fs(node) if found: passed = False From fa5475097cc3b34f2f7b8e40cf6d80eb1b9c5449 Mon Sep 17 00:00:00 2001 From: Reid Wahl Date: Sun, 9 Aug 2026 13:54:22 -0700 Subject: [PATCH 18/25] Refactor: cts: Drop _audit_resource() rc variable This gets rid of some elses. Signed-off-by: Reid Wahl --- python/pacemaker/_cts/audits.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/python/pacemaker/_cts/audits.py b/python/pacemaker/_cts/audits.py index 82049657537..b2e98834548 100644 --- a/python/pacemaker/_cts/audits.py +++ b/python/pacemaker/_cts/audits.py @@ -443,43 +443,49 @@ def __init__(self, cm): def _audit_resource(self, resource, quorum): """Perform the audit of a single resource.""" - rc = True active = self._cm.resource_location(resource.id) if len(active) == 1: if quorum: self.debug(f"Resource {resource.id} active on {active!r}") + return True - elif resource.needs_quorum == 1: + if resource.needs_quorum == 1: logging.log(f"Resource {resource.id} active without quorum: {active!r}") - rc = False + return False + + return True - elif not resource.managed: + if not resource.managed: logging.log(f"Resource {resource.id} not managed. Active on {active!r}") + return True - elif not resource.unique: + if not resource.unique: # TODO: Figure out a clever way to actually audit these resource types if len(active) > 1: self.debug(f"Non-unique resource {resource.id} is active on: {active!r}") else: self.debug(f"Non-unique resource {resource.id} is not active") - elif len(active) > 1: + return True + + if len(active) > 1: logging.log(f"Resource {resource.id} is active multiple times: {active!r}") - rc = False + return False - elif resource.orphan: + if resource.orphan: self.debug(f"Resource {resource.id} is an inactive orphan") + return True - elif not self._inactive_nodes: + if not self._inactive_nodes: logging.log(f"WARN: Resource {resource.id} not served anywhere") - rc = False + return False - elif quorum or not resource.needs_quorum: + if quorum or not resource.needs_quorum: self.debug(f"Resource {resource.id} not served anywhere " f"(Inactive nodes: {self._inactive_nodes!r})") - return rc + return True def _setup(self): """ From 2c5bcc99a61d45c5e2774b0d925f29e73b9f4b14 Mon Sep 17 00:00:00 2001 From: Reid Wahl Date: Sun, 9 Aug 2026 14:03:19 -0700 Subject: [PATCH 19/25] Refactor: cts: Drop Corosync.start() kill_first argument All callers pass True. Signed-off-by: Reid Wahl --- cts/cts-attrd.in | 4 ++-- cts/cts-exec.in | 4 ++-- cts/cts-fencing.in | 4 ++-- python/pacemaker/_cts/corosync.py | 9 +++------ 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/cts/cts-attrd.in b/cts/cts-attrd.in index a4e24587e9f..879c51ba6b2 100644 --- a/cts/cts-attrd.in +++ b/cts/cts-attrd.in @@ -7,7 +7,7 @@ # pacemaker imports need to come after we modify sys.path, which pylint will complain about. # pylint: disable=wrong-import-position -__copyright__ = "Copyright 2023-2025 the Pacemaker project contributors" +__copyright__ = "Copyright 2023-2026 the Pacemaker project contributors" __license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY" import argparse @@ -81,7 +81,7 @@ class AttributeTests(Tests): def setup_environment(self, use_corosync): """Prepare the host before executing any tests.""" if use_corosync: - self._corosync.start(kill_first=True) + self._corosync.start() def cleanup_environment(self, use_corosync): """Clean up the host after executing desired tests.""" diff --git a/cts/cts-exec.in b/cts/cts-exec.in index c423ff154a4..0286b1c29c7 100644 --- a/cts/cts-exec.in +++ b/cts/cts-exec.in @@ -7,7 +7,7 @@ # pacemaker imports need to come after we modify sys.path, which pylint will complain about. # pylint: disable=wrong-import-position -__copyright__ = "Copyright 2012-2025 the Pacemaker project contributors" +__copyright__ = "Copyright 2012-2026 the Pacemaker project contributors" __license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY" import argparse @@ -87,7 +87,7 @@ class ExecTest(Test): def _start_daemons(self): if self._corosync: - self._corosync.start(kill_first=True) + self._corosync.start() # pylint: disable=consider-using-with self._fencer = subprocess.Popen(["pacemaker-fenced", "-s"]) diff --git a/cts/cts-fencing.in b/cts/cts-fencing.in index 1fe4a635683..df403cb0f9e 100644 --- a/cts/cts-fencing.in +++ b/cts/cts-fencing.in @@ -7,7 +7,7 @@ # pacemaker imports need to come after we modify sys.path, which pylint will complain about. # pylint: disable=wrong-import-position -__copyright__ = "Copyright 2012-2025 the Pacemaker project contributors" +__copyright__ = "Copyright 2012-2026 the Pacemaker project contributors" __license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY" import argparse @@ -819,7 +819,7 @@ class FenceTests(Tests): def setup_environment(self): """Prepare the host before executing any tests.""" - self._corosync.start(kill_first=True) + self._corosync.start() subprocess.call(["cts-support", "install"]) def cleanup_environment(self): diff --git a/python/pacemaker/_cts/corosync.py b/python/pacemaker/_cts/corosync.py index de4e00b7871..14f2aa61a90 100644 --- a/python/pacemaker/_cts/corosync.py +++ b/python/pacemaker/_cts/corosync.py @@ -159,18 +159,15 @@ def _stop(self): else: killall(["corosync"]) - def start(self, kill_first=False, timeout=10): + def start(self, timeout=10): """ - Start the corosync process. + Start the corosync process, stopping any existing ones first. Arguments: - kill_first -- Whether to kill any pre-existing corosync processes before - starting a new one timeout -- If corosync does not start within this many seconds, raise TimeoutError """ - if kill_first: - self._stop() + self._stop() self._existing_cfg_file = generate_corosync_cfg(self.logdir, self.cluster_name, localname()) From 2d27c33cc8c0dbe5cf1cb79a7dd66cd5e1b9a43c Mon Sep 17 00:00:00 2001 From: Reid Wahl Date: Sun, 9 Aug 2026 14:07:04 -0700 Subject: [PATCH 20/25] Refactor: cts: Drop Corosync.start() timeout argument All callers pass the default. Signed-off-by: Reid Wahl --- python/pacemaker/_cts/corosync.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/python/pacemaker/_cts/corosync.py b/python/pacemaker/_cts/corosync.py index 14f2aa61a90..8ae4992a4a6 100644 --- a/python/pacemaker/_cts/corosync.py +++ b/python/pacemaker/_cts/corosync.py @@ -159,14 +159,8 @@ def _stop(self): else: killall(["corosync"]) - def start(self, timeout=10): - """ - Start the corosync process, stopping any existing ones first. - - Arguments: - timeout -- If corosync does not start within this many seconds, raise - TimeoutError - """ + def start(self): + """Start the corosync process, stopping any existing ones first.""" self._stop() self._existing_cfg_file = generate_corosync_cfg(self.logdir, @@ -176,7 +170,7 @@ def start(self, timeout=10): self._start() # Wait for corosync to be ready before returning - self._ready(logfile, timeout=timeout) + self._ready(logfile, timeout=10) def stop(self): """Stop the corosync process.""" From 95128071cdca5368e9610c79beedf2da9454788e Mon Sep 17 00:00:00 2001 From: Reid Wahl Date: Sun, 9 Aug 2026 14:10:26 -0700 Subject: [PATCH 21/25] Refactor: cts: Drop Corosync._ready() timeout argument Signed-off-by: Reid Wahl --- python/pacemaker/_cts/corosync.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/pacemaker/_cts/corosync.py b/python/pacemaker/_cts/corosync.py index 8ae4992a4a6..d1bf3390508 100644 --- a/python/pacemaker/_cts/corosync.py +++ b/python/pacemaker/_cts/corosync.py @@ -118,11 +118,11 @@ def __init__(self, verbose, logdir, cluster_name): self._env = Environment(["--nodes", "localhost"]) self._existing_cfg_file = None - def _ready(self, logfile, timeout=10): + def _ready(self, logfile): """Return whether corosync is ready.""" i = 0 - while i < timeout: + while i < 10: with open(logfile, "r", encoding="utf-8") as corosync_log: for line in corosync_log.readlines(): if line.endswith("ready to provide service.\n"): @@ -170,7 +170,7 @@ def start(self): self._start() # Wait for corosync to be ready before returning - self._ready(logfile, timeout=10) + self._ready(logfile) def stop(self): """Stop the corosync process.""" From b406de3af3ab2954af7b7764b9af2c8630d58b71 Mon Sep 17 00:00:00 2001 From: Reid Wahl Date: Sun, 9 Aug 2026 14:15:59 -0700 Subject: [PATCH 22/25] Refactor: cts: Drop killal() terminate argument All callers pass the default. Signed-off-by: Reid Wahl --- python/pacemaker/_cts/process.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/python/pacemaker/_cts/process.py b/python/pacemaker/_cts/process.py index 8d652037e0c..279643e6285 100644 --- a/python/pacemaker/_cts/process.py +++ b/python/pacemaker/_cts/process.py @@ -12,7 +12,7 @@ from pacemaker.exitstatus import ExitStatus -def killall(process_names, terminate=False): +def killall(process_names): """Kill all instances of every process in a list.""" if not process_names: return @@ -20,19 +20,9 @@ def killall(process_names, terminate=False): if not isinstance(process_names, list): process_names = [process_names] - procs = [] for proc in psutil.process_iter(["name"]): if proc.info["name"] in process_names: - procs.append(proc) - - if terminate: - for proc in procs: - proc.terminate() - _, alive = psutil.wait_procs(procs, timeout=3) - procs = alive - - for proc in procs: - proc.kill() + proc.kill() def is_proc_running(process_name): From 987bb9937528a5787a8183d004a9ba5e3ea7ecdf Mon Sep 17 00:00:00 2001 From: Reid Wahl Date: Sun, 9 Aug 2026 14:18:40 -0700 Subject: [PATCH 23/25] Refactor: cts: Simplify killall() All callers pass a nonempty list, and an empty list would be fine anyway. Signed-off-by: Reid Wahl --- python/pacemaker/_cts/process.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/python/pacemaker/_cts/process.py b/python/pacemaker/_cts/process.py index 279643e6285..3e7b87347f3 100644 --- a/python/pacemaker/_cts/process.py +++ b/python/pacemaker/_cts/process.py @@ -14,12 +14,6 @@ def killall(process_names): """Kill all instances of every process in a list.""" - if not process_names: - return - - if not isinstance(process_names, list): - process_names = [process_names] - for proc in psutil.process_iter(["name"]): if proc.info["name"] in process_names: proc.kill() From 14a34732c8022a3c14659ecb5c50f6a240a8921e Mon Sep 17 00:00:00 2001 From: Reid Wahl Date: Sun, 9 Aug 2026 14:26:00 -0700 Subject: [PATCH 24/25] Refactor: cts: Drop AsyncCmd.__init__() proc argument This should have been done with beb9468d. Signed-off-by: Reid Wahl --- python/pacemaker/_cts/remote.py | 39 ++++++++++++++------------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/python/pacemaker/_cts/remote.py b/python/pacemaker/_cts/remote.py index 9d05f7819a5..aeff5069ed6 100644 --- a/python/pacemaker/_cts/remote.py +++ b/python/pacemaker/_cts/remote.py @@ -16,23 +16,19 @@ class AsyncCmd(Thread): """A class for doing the hard work of running a command on another machine.""" - def __init__(self, node, command, proc=None, delegate=None): + def __init__(self, node, command, delegate=None): """ Create a new AsyncCmd instance. Arguments: node -- The remote machine to run on command -- The ssh command string to use for remote execution - proc -- If not None, a process object previously created with Popen. - Instead of spawning a new process, we will then wait on - this process to finish and handle its output. delegate -- When the command completes, call the async_complete method on this object """ self._command = command self._delegate = delegate self._node = node - self._proc = proc Thread.__init__(self) @@ -41,33 +37,32 @@ def run(self): out = None err = None - if not self._proc: - # pylint: disable=consider-using-with - self._proc = subprocess.Popen(self._command, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, close_fds=True, - shell=True, universal_newlines=True) + # pylint: disable=consider-using-with + proc = subprocess.Popen(self._command, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, close_fds=True, + shell=True, universal_newlines=True) - logging.debug(f"cmd: async: target={self._node}, pid={self._proc.pid}: {self._command}") - self._proc.wait() + logging.debug(f"cmd: async: target={self._node}, pid={proc.pid}: {self._command}") + proc.wait() if self._delegate: - logging.debug(f"cmd: pid {self._proc.pid} returned {self._proc.returncode} to {self._delegate!r}") + logging.debug(f"cmd: pid {proc.pid} returned {proc.returncode} to {self._delegate!r}") else: - logging.debug(f"cmd: pid {self._proc.pid} returned {self._proc.returncode}") + logging.debug(f"cmd: pid {proc.pid} returned {proc.returncode}") - if self._proc.stderr: - err = self._proc.stderr.readlines() - self._proc.stderr.close() + if proc.stderr: + err = proc.stderr.readlines() + proc.stderr.close() for line in err: - logging.debug(f"cmd: stderr[{self._proc.pid}]: {line}") + logging.debug(f"cmd: stderr[{proc.pid}]: {line}") - if self._proc.stdout: - out = self._proc.stdout.readlines() - self._proc.stdout.close() + if proc.stdout: + out = proc.stdout.readlines() + proc.stdout.close() if self._delegate: - self._delegate.async_complete(self._proc.pid, self._proc.returncode, out, err) + self._delegate.async_complete(proc.pid, proc.returncode, out, err) class RemoteExec: From 6f2434310f7a82d1a4ef8bad040563b4511f1cd0 Mon Sep 17 00:00:00 2001 From: Reid Wahl Date: Sun, 9 Aug 2026 23:04:41 -0700 Subject: [PATCH 25/25] Refactor: cts: Use dict.get() in incr() methods Signed-off-by: Reid Wahl --- python/pacemaker/_cts/scenarios.py | 5 +---- python/pacemaker/_cts/tests/ctstest.py | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/python/pacemaker/_cts/scenarios.py b/python/pacemaker/_cts/scenarios.py index abaf3098d9c..7a246553543 100644 --- a/python/pacemaker/_cts/scenarios.py +++ b/python/pacemaker/_cts/scenarios.py @@ -174,10 +174,7 @@ def teardown(self, n_components=None): def incr(self, name): """Increment the given stats key.""" - if name not in self.stats: - self.stats[name] = 0 - - self.stats[name] += 1 + self.stats[name] = self.stats.get(name, 0) + 1 def run(self, iterations): """Run all the tests the given number of times.""" diff --git a/python/pacemaker/_cts/tests/ctstest.py b/python/pacemaker/_cts/tests/ctstest.py index e2d0bbd7731..9e1f64c2ca0 100644 --- a/python/pacemaker/_cts/tests/ctstest.py +++ b/python/pacemaker/_cts/tests/ctstest.py @@ -83,10 +83,7 @@ def log_timer(self, key="test"): def incr(self, name): """Increment the given stats key.""" - if name not in self.stats: - self.stats[name] = 0 - - self.stats[name] += 1 + self.stats[name] = self.stats.get(name, 0) + 1 # Reset the test passed boolean if name == "calls":