diff --git a/cts/cts-attrd.in b/cts/cts-attrd.in index a4e24587e9f..879c51ba6b2 100644 --- a/cts/cts-attrd.in +++ b/cts/cts-attrd.in @@ -7,7 +7,7 @@ # pacemaker imports need to come after we modify sys.path, which pylint will complain about. # pylint: disable=wrong-import-position -__copyright__ = "Copyright 2023-2025 the Pacemaker project contributors" +__copyright__ = "Copyright 2023-2026 the Pacemaker project contributors" __license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY" import argparse @@ -81,7 +81,7 @@ class AttributeTests(Tests): def setup_environment(self, use_corosync): """Prepare the host before executing any tests.""" if use_corosync: - self._corosync.start(kill_first=True) + self._corosync.start() def cleanup_environment(self, use_corosync): """Clean up the host after executing desired tests.""" diff --git a/cts/cts-exec.in b/cts/cts-exec.in index c423ff154a4..0286b1c29c7 100644 --- a/cts/cts-exec.in +++ b/cts/cts-exec.in @@ -7,7 +7,7 @@ # pacemaker imports need to come after we modify sys.path, which pylint will complain about. # pylint: disable=wrong-import-position -__copyright__ = "Copyright 2012-2025 the Pacemaker project contributors" +__copyright__ = "Copyright 2012-2026 the Pacemaker project contributors" __license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY" import argparse @@ -87,7 +87,7 @@ class ExecTest(Test): def _start_daemons(self): if self._corosync: - self._corosync.start(kill_first=True) + self._corosync.start() # pylint: disable=consider-using-with self._fencer = subprocess.Popen(["pacemaker-fenced", "-s"]) diff --git a/cts/cts-fencing.in b/cts/cts-fencing.in index 1fe4a635683..df403cb0f9e 100644 --- a/cts/cts-fencing.in +++ b/cts/cts-fencing.in @@ -7,7 +7,7 @@ # pacemaker imports need to come after we modify sys.path, which pylint will complain about. # pylint: disable=wrong-import-position -__copyright__ = "Copyright 2012-2025 the Pacemaker project contributors" +__copyright__ = "Copyright 2012-2026 the Pacemaker project contributors" __license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY" import argparse @@ -819,7 +819,7 @@ class FenceTests(Tests): def setup_environment(self): """Prepare the host before executing any tests.""" - self._corosync.start(kill_first=True) + self._corosync.start() subprocess.call(["cts-support", "install"]) def cleanup_environment(self): diff --git a/cts/cts-lab.in b/cts/cts-lab.in index 2031c172db3..98a353d88b1 100644 --- a/cts/cts-lab.in +++ b/cts/cts-lab.in @@ -9,6 +9,7 @@ __copyright__ = "Copyright 2001-2026 the Pacemaker project contributors" __license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY" +from functools import partial import signal import sys @@ -20,42 +21,29 @@ from pacemaker._cts import logging from pacemaker._cts.scenarios import AllOnce, Boot, BootCluster, LeaveBooted, RandomTests, Sequence from pacemaker._cts.tests import test_list -# These are globals so they can be used by the signal handler. -scenario = None -logging.add_stderr() - -def sig_handler(signum, _frame): - """Handle the given signal number.""" +def term_handler(scenario, signum, _frame): + """Handle a SIGTERM by exiting gracefully.""" logging.log(f"Interrupted by signal {signum}") if scenario: scenario.summarize() + scenario.teardown() - if signum == 15: - if scenario: - scenario.teardown() - - sys.exit(1) + sys.exit(1) def plural_s(n): """Return a string suffix depending on whether or not n is > 1.""" - if n == 1: - return "" - - return "S" + return "" if n == 1 else "S" if __name__ == '__main__': + logging.add_stderr() + env = Environment(sys.argv[1:]) lab = CtsLab(env) iters = lab["iterations"] - tests = [] - - # Set the signal handler - signal.signal(15, sig_handler) - signal.signal(10, sig_handler) # Create the Cluster Manager object. # Currently Corosync2 is the only available cluster manager. @@ -71,9 +59,9 @@ if __name__ == '__main__': outputfile.truncate(0) audits = audit_list(cm) + tests = test_list(cm, env, audits) if lab["ListTests"]: - tests = test_list(cm, env, audits) logging.log(f"Total {len(tests)} tests") for test in tests: @@ -81,23 +69,16 @@ if __name__ == '__main__': sys.exit(0) - elif len(lab["tests"]) == 0: - tests = test_list(cm, env, audits) + if lab["tests"]: + # Get tests corresponding to the names in lab["tests"] + test_map = {test.name: test for test in tests} - else: - chosen = lab["tests"] - for test_case in chosen: - match = None + try: + tests = [test_map[test] for test in lab["tests"]] - for test in test_list(cm, env, audits): - if test.name == test_case: - match = test - - if not match: - logging.log("--choose: No applicable/valid tests chosen") - sys.exit(1) - else: - tests.append(match) + except KeyError as e: + logging.log(f"--choose: Test {e} is invalid or not applicable") + sys.exit(1) # Scenario selection if lab["scenario"] == "all-once": @@ -115,8 +96,10 @@ if __name__ == '__main__': logging.log(f"Scenario: {scenario.__doc__}") logging.log(f"CTS Exerciser: {lab['cts-exerciser']}") logging.log(f"CTS Logfile: {lab['OutputFile']}") + if "syslogd" in lab: logging.log(f"Syslog variant: {lab['syslogd']}") + logging.log(f"System log files: {lab['LogFileName']}") if "IPBase" in lab: @@ -125,6 +108,10 @@ if __name__ == '__main__': logging.log(f"Cluster starts at boot: {lab['at-boot']}") lab.dump() + + # On SIGTERM during lab run, print a summary and do teardown + signal.signal(15, partial(term_handler, scenario)) + rc = lab.run(scenario, iters) sys.exit(rc) diff --git a/python/pacemaker/_cts/audits.py b/python/pacemaker/_cts/audits.py index 2634d6a0878..b2e98834548 100644 --- a/python/pacemaker/_cts/audits.py +++ b/python/pacemaker/_cts/audits.py @@ -47,10 +47,6 @@ def is_applicable(self): """ raise NotImplementedError - def log(self, args): - """Log a message.""" - logging.log(f"audit: {args}") - def debug(self, args): """Log a debug message.""" logging.debug(f"audit: {args}") @@ -74,14 +70,11 @@ def __init__(self, cm): ClusterAudit.__init__(self, cm) self.name = "LogAudit" - def _restart_cluster_logging(self, nodes=None): - """Restart logging on the given nodes, or all if none are given.""" - if not nodes: - nodes = self._cm.env["nodes"] - - logging.debug(f"Restarting logging on: {nodes!r}") + def _restart_cluster_logging(self): + """Restart logging on all nodes.""" + logging.debug("Restarting logging on all nodes") - for node in nodes: + for node in self._cm.env["nodes"]: if self._cm.env["have_systemd"]: (rc, _) = self._cm.rsh.call(node, "systemctl stop systemd-journald.socket") if rc != 0: @@ -283,8 +276,10 @@ def _find_core_with_coredumpctl(self, node): (_, lsout) = self._cm.rsh.call(node, "coredumpctl --no-legend --no-pager") return self._output_has_core(lsout, node) - def _find_core_on_fs(self, node, paths): - """Check for core dumps on the given node, under any of the given paths.""" + def _find_core_on_fs(self, node): + """Check for Pacemaker and Corosync core dumps on the given node.""" + paths = ["/var/lib/pacemaker/cores/*", "/var/lib/corosync"] + (_, lsout) = self._cm.rsh.call(node, f"ls -al {' '.join(paths)} | grep core.[0-9]", verbose=1) return self._output_has_core(lsout, node) @@ -311,8 +306,7 @@ def __call__(self): # # To handle the last two cases, check the other filesystem locations. if not found: - found = self._find_core_on_fs(node, ["/var/lib/pacemaker/cores/*", - "/var/lib/corosync"]) + found = self._find_core_on_fs(node) if found: passed = False @@ -449,43 +443,49 @@ def __init__(self, cm): def _audit_resource(self, resource, quorum): """Perform the audit of a single resource.""" - rc = True active = self._cm.resource_location(resource.id) if len(active) == 1: if quorum: self.debug(f"Resource {resource.id} active on {active!r}") + return True - elif resource.needs_quorum == 1: + if resource.needs_quorum == 1: logging.log(f"Resource {resource.id} active without quorum: {active!r}") - rc = False + return False - elif not resource.managed: + return True + + if not resource.managed: logging.log(f"Resource {resource.id} not managed. Active on {active!r}") + return True - elif not resource.unique: + if not resource.unique: # TODO: Figure out a clever way to actually audit these resource types if len(active) > 1: self.debug(f"Non-unique resource {resource.id} is active on: {active!r}") else: self.debug(f"Non-unique resource {resource.id} is not active") - elif len(active) > 1: + return True + + if len(active) > 1: logging.log(f"Resource {resource.id} is active multiple times: {active!r}") - rc = False + return False - elif resource.orphan: + if resource.orphan: self.debug(f"Resource {resource.id} is an inactive orphan") + return True - elif not self._inactive_nodes: + if not self._inactive_nodes: logging.log(f"WARN: Resource {resource.id} not served anywhere") - rc = False + return False - elif quorum or not resource.needs_quorum: + if quorum or not resource.needs_quorum: self.debug(f"Resource {resource.id} not served anywhere " f"(Inactive nodes: {self._inactive_nodes!r})") - return rc + return True def _setup(self): """ @@ -530,7 +530,7 @@ def __call__(self): return passed primitives = [r for r in self._resources if r.type == "primitive"] - quorum = self._cm.has_quorum(None) + quorum = self._cm.has_quorum() for primitive in primitives: if not self._audit_resource(primitive, quorum): diff --git a/python/pacemaker/_cts/clustermanager.py b/python/pacemaker/_cts/clustermanager.py index 4794e5e4840..7d6cc1b30a6 100644 --- a/python/pacemaker/_cts/clustermanager.py +++ b/python/pacemaker/_cts/clustermanager.py @@ -97,7 +97,7 @@ def prepare_fencing_watcher(self): """Return a LogWatcher object that watches for fencing log messages.""" # If we don't have quorum now but get it as a result of starting this node, # then a bunch of nodes might get fenced - if self.has_quorum(None): + if self.has_quorum(): logging.debug("Have quorum") return None @@ -137,7 +137,7 @@ def fencing_cleanup(self, node, stonith): logging.debug("Nothing to do") return peer_list - q = self.has_quorum(None) + q = self.has_quorum() if not q and len(self.env["nodes"]) > 2: # We didn't gain quorum - we shouldn't have shot anyone logging.debug(f"Quorum: {q} Len: {len(self.env['nodes'])}") @@ -332,19 +332,14 @@ def stop_cm_async(self, node): self.rsh.call_async(node, self.templates["StopCmd"]) self.expected_status[node] = "down" - def startall(self, nodelist=None, verbose=False, quick=False): - """Start the cluster manager on every node in the cluster, or on every node in nodelist.""" - if not nodelist: - nodelist = self.env["nodes"] + def startall(self): + """Start the cluster manager on every node in the cluster.""" + nodelist = self.env["nodes"] for node in nodelist: if self.expected_status[node] == "down": self.ns.wait_for_all_nodes(nodelist, 300) - if not quick: - # This is used for "basic sanity checks", so only start one node ... - return self.start_cm(nodelist[0], verbose=verbose) - # Approximation of SimulStartList for --boot watchpats = [ self.templates["Pat:DC_IDLE"], @@ -363,11 +358,11 @@ def startall(self, nodelist=None, verbose=False, quick=False): self.env["dead_time"] + 10) watch.set_watch() - if not self.start_cm(nodelist[0], verbose=verbose): + if not self.start_cm(nodelist[0], verbose=True): return False for node in nodelist: - self.start_cm_async(node, verbose=verbose) + self.start_cm_async(node, verbose=True) watch.look_for_all() if watch.unmatched: @@ -380,28 +375,22 @@ def startall(self, nodelist=None, verbose=False, quick=False): return True - def stopall(self, nodelist=None, verbose=False, force=False): - """Stop the cluster manager on every node in the cluster, or on every node in nodelist.""" + def stopall(self, force=False): + """Stop the cluster manager on every node in the cluster.""" ret = True - if not nodelist: - nodelist = self.env["nodes"] - for node in self.env["nodes"]: if self.expected_status[node] == "up" or force: - if not self.stop_cm(node, verbose=verbose, force=force): + if not self.stop_cm(node, verbose=True, force=force): ret = False return ret - def statall(self, nodelist=None): - """Return the status of the cluster manager on every node in the cluster, or on every node in nodelist.""" + def statall(self): + """Return the status of the cluster manager on every node in the cluster.""" result = {} - if not nodelist: - nodelist = self.env["nodes"] - - for node in nodelist: + for node in self.env["nodes"]: if self.stat_cm(node): result[node] = "up" else: @@ -409,11 +398,8 @@ def statall(self, nodelist=None): return result - def isolate_node(self, target, nodes=None): - """Break communication between the target node and all other nodes in the cluster, or nodes.""" - if not nodes: - nodes = self.env["nodes"] - + def isolate_node(self, target, nodes): + """Break communication between the target node the given other nodes.""" for node in nodes: if node == target: continue @@ -427,11 +413,8 @@ def isolate_node(self, target, nodes=None): return True - def unisolate_node(self, target, nodes=None): - """Re-establish communication between the target node and all other nodes in the cluster, or nodes.""" - if not nodes: - nodes = self.env["nodes"] - + def unisolate_node(self, target, nodes): + """Re-establish communication between the target node and the given other nodes.""" for node in nodes: if node == target: continue @@ -536,7 +519,7 @@ def node_stable(self, node): logging.log(f"Warn: Node {node} not stable") return False - def _partition_stable(self, nodes, timeout=None): + def _partition_stable(self, nodes, timeout): """Return whether or not all nodes in the given partition are stable.""" watchpats = [ "Current ping state: S_IDLE", @@ -705,16 +688,9 @@ def find_partitions(self): logging.debug(f"Found partitions: {ccm_partitions!r}") return ccm_partitions - def has_quorum(self, node_list): + def has_quorum(self): """Return whether or not the cluster has quorum.""" - # If we are auditing a partition, then one side will - # have quorum and the other not. - # So the caller needs to tell us which we are checking - # If no value for node_list is specified... assume all nodes - if not node_list: - node_list = self.env["nodes"] - - for node in node_list: + for node in self.env["nodes"]: if self.expected_status[node] != "up": continue diff --git a/python/pacemaker/_cts/corosync.py b/python/pacemaker/_cts/corosync.py index de4e00b7871..d1bf3390508 100644 --- a/python/pacemaker/_cts/corosync.py +++ b/python/pacemaker/_cts/corosync.py @@ -118,11 +118,11 @@ def __init__(self, verbose, logdir, cluster_name): self._env = Environment(["--nodes", "localhost"]) self._existing_cfg_file = None - def _ready(self, logfile, timeout=10): + def _ready(self, logfile): """Return whether corosync is ready.""" i = 0 - while i < timeout: + while i < 10: with open(logfile, "r", encoding="utf-8") as corosync_log: for line in corosync_log.readlines(): if line.endswith("ready to provide service.\n"): @@ -159,18 +159,9 @@ def _stop(self): else: killall(["corosync"]) - def start(self, kill_first=False, timeout=10): - """ - Start the corosync process. - - Arguments: - kill_first -- Whether to kill any pre-existing corosync processes before - starting a new one - timeout -- If corosync does not start within this many seconds, raise - TimeoutError - """ - if kill_first: - self._stop() + def start(self): + """Start the corosync process, stopping any existing ones first.""" + self._stop() self._existing_cfg_file = generate_corosync_cfg(self.logdir, self.cluster_name, localname()) @@ -179,7 +170,7 @@ def start(self, kill_first=False, timeout=10): self._start() # Wait for corosync to be ready before returning - self._ready(logfile, timeout=timeout) + self._ready(logfile) def stop(self): """Stop the corosync process.""" diff --git a/python/pacemaker/_cts/process.py b/python/pacemaker/_cts/process.py index 8d652037e0c..3e7b87347f3 100644 --- a/python/pacemaker/_cts/process.py +++ b/python/pacemaker/_cts/process.py @@ -12,27 +12,11 @@ from pacemaker.exitstatus import ExitStatus -def killall(process_names, terminate=False): +def killall(process_names): """Kill all instances of every process in a list.""" - if not process_names: - return - - if not isinstance(process_names, list): - process_names = [process_names] - - procs = [] for proc in psutil.process_iter(["name"]): if proc.info["name"] in process_names: - procs.append(proc) - - if terminate: - for proc in procs: - proc.terminate() - _, alive = psutil.wait_procs(procs, timeout=3) - procs = alive - - for proc in procs: - proc.kill() + proc.kill() def is_proc_running(process_name): diff --git a/python/pacemaker/_cts/remote.py b/python/pacemaker/_cts/remote.py index 9d05f7819a5..aeff5069ed6 100644 --- a/python/pacemaker/_cts/remote.py +++ b/python/pacemaker/_cts/remote.py @@ -16,23 +16,19 @@ class AsyncCmd(Thread): """A class for doing the hard work of running a command on another machine.""" - def __init__(self, node, command, proc=None, delegate=None): + def __init__(self, node, command, delegate=None): """ Create a new AsyncCmd instance. Arguments: node -- The remote machine to run on command -- The ssh command string to use for remote execution - proc -- If not None, a process object previously created with Popen. - Instead of spawning a new process, we will then wait on - this process to finish and handle its output. delegate -- When the command completes, call the async_complete method on this object """ self._command = command self._delegate = delegate self._node = node - self._proc = proc Thread.__init__(self) @@ -41,33 +37,32 @@ def run(self): out = None err = None - if not self._proc: - # pylint: disable=consider-using-with - self._proc = subprocess.Popen(self._command, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, close_fds=True, - shell=True, universal_newlines=True) + # pylint: disable=consider-using-with + proc = subprocess.Popen(self._command, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, close_fds=True, + shell=True, universal_newlines=True) - logging.debug(f"cmd: async: target={self._node}, pid={self._proc.pid}: {self._command}") - self._proc.wait() + logging.debug(f"cmd: async: target={self._node}, pid={proc.pid}: {self._command}") + proc.wait() if self._delegate: - logging.debug(f"cmd: pid {self._proc.pid} returned {self._proc.returncode} to {self._delegate!r}") + logging.debug(f"cmd: pid {proc.pid} returned {proc.returncode} to {self._delegate!r}") else: - logging.debug(f"cmd: pid {self._proc.pid} returned {self._proc.returncode}") + logging.debug(f"cmd: pid {proc.pid} returned {proc.returncode}") - if self._proc.stderr: - err = self._proc.stderr.readlines() - self._proc.stderr.close() + if proc.stderr: + err = proc.stderr.readlines() + proc.stderr.close() for line in err: - logging.debug(f"cmd: stderr[{self._proc.pid}]: {line}") + logging.debug(f"cmd: stderr[{proc.pid}]: {line}") - if self._proc.stdout: - out = self._proc.stdout.readlines() - self._proc.stdout.close() + if proc.stdout: + out = proc.stdout.readlines() + proc.stdout.close() if self._delegate: - self._delegate.async_complete(self._proc.pid, self._proc.returncode, out, err) + self._delegate.async_complete(proc.pid, proc.returncode, out, err) class RemoteExec: diff --git a/python/pacemaker/_cts/scenarios.py b/python/pacemaker/_cts/scenarios.py index 13b00b060ec..7a246553543 100644 --- a/python/pacemaker/_cts/scenarios.py +++ b/python/pacemaker/_cts/scenarios.py @@ -174,10 +174,7 @@ def teardown(self, n_components=None): def incr(self, name): """Increment the given stats key.""" - if name not in self.stats: - self.stats[name] = 0 - - self.stats[name] += 1 + self.stats[name] = self.stats.get(name, 0) + 1 def run(self, iterations): """Run all the tests the given number of times.""" @@ -246,27 +243,23 @@ def run_test(self, test, testcount): def summarize(self): """Output scenario results.""" + # This dict removes duplicates in self.tests while preserving order + tests = {test.name: test for test in self.tests} + + summary_keys = ["calls", "failure", "skipped", "auditfail"] + logging.log("****************") logging.log("Overall Results:%r" % self.stats) logging.log("****************") - stat_summary = {} - summary_keys = ["calls", "failure", "skipped", "auditfail"] - logging.log("Test Summary") - for test in self.tests: - if test.name not in stat_summary: - stat_summary[test.name] = {key: 0 for key in summary_keys} - - for key in summary_keys: - stat_summary[test.name][key] += test.stats[key] - - for (name, summary) in stat_summary.items(): + for (name, test) in tests.items(): + summary = {key: test.stats[key] for key in summary_keys} logging.log(f"{f'Test {name}':<25} {summary!r}") logging.debug("Detailed Results") - for test in self.tests: - logging.debug(f"{f'Test {test.name}: ':<25} {test.stats!r}") + for (name, test) in tests.items(): + logging.debug(f"{f'Test {name}: ':<25} {test.stats!r}") logging.log("<<<<<<<<<<<<<<<< TESTS COMPLETED") @@ -389,16 +382,16 @@ def setup(self): self._cm.prepare() # Clear out the cobwebs ;-) - self._cm.stopall(verbose=True, force=True) + self._cm.stopall(force=True) # Now start the Cluster Manager on all the nodes. logging.log("Starting Cluster Manager on all nodes.") - return self._cm.startall(verbose=True, quick=True) + return self._cm.startall() def teardown(self): """Tear down the component.""" logging.log("Stopping Cluster Manager on all nodes") - self._cm.stopall(verbose=True, force=False) + self._cm.stopall(force=False) class LeaveBooted(BootCluster): diff --git a/python/pacemaker/_cts/tests/ctstest.py b/python/pacemaker/_cts/tests/ctstest.py index e2d0bbd7731..9e1f64c2ca0 100644 --- a/python/pacemaker/_cts/tests/ctstest.py +++ b/python/pacemaker/_cts/tests/ctstest.py @@ -83,10 +83,7 @@ def log_timer(self, key="test"): def incr(self, name): """Increment the given stats key.""" - if name not in self.stats: - self.stats[name] = 0 - - self.stats[name] += 1 + self.stats[name] = self.stats.get(name, 0) + 1 # Reset the test passed boolean if name == "calls":