diff --git a/changelog-entries/738.md b/changelog-entries/738.md new file mode 100644 index 000000000..5a13ff808 --- /dev/null +++ b/changelog-entries/738.md @@ -0,0 +1 @@ +- Add per-test `max_time` override in `tests.yaml` to cap preCICE simulation time without editing `precice-config.xml` manually. Applies consistently to both test runs and reference result generation. Targets only `` tags; handles multiple tags with a warning. Validates that `max_time` is a positive number. diff --git a/tools/tests/generate_reference_results.py b/tools/tests/generate_reference_results.py index 055e7b31c..d0e9a5995 100644 --- a/tools/tests/generate_reference_results.py +++ b/tools/tests/generate_reference_results.py @@ -15,6 +15,7 @@ from paths import PRECICE_TUTORIAL_DIR, PRECICE_TESTS_RUN_DIR, PRECICE_TESTS_DIR, PRECICE_REL_OUTPUT_DIR import time +import json def create_tar_gz(source_folder: Path, output_filename: Path): @@ -108,10 +109,13 @@ def main(): for test_suite in test_suites: tutorials = test_suite.cases_of_tutorial.keys() for tutorial in tutorials: - for case, reference_result in zip( - test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial]): + cases = test_suite.cases_of_tutorial[tutorial] + reference_results = test_suite.reference_results[tutorial] + max_times = test_suite.max_times.get(tutorial, [None] * len(cases)) + for case, reference_result, max_time in zip( + cases, reference_results, max_times): systemtests_to_run.add( - Systemtest(tutorial, build_args, case, reference_result)) + Systemtest(tutorial, build_args, case, reference_result, max_time=max_time)) reference_result_per_tutorial = {} current_time_string = datetime.now().strftime('%Y-%m-%d %H:%M:%S') @@ -139,6 +143,16 @@ def main(): raise RuntimeError( f"Error executing: \n {systemtest} \n Could not find result folder {reference_result_folder}\n Probably the tutorial did not run through properly. Please check corresponding logs") + # Write iterations.log hashes sidecar for implicit-coupling regression checks (issue #440) + collected = systemtest._collect_iterations_logs(systemtest.get_system_test_dir()) + if collected: + hashes = { + rel: Systemtest._sha256_file(p) for rel, p in collected + } + sidecar = systemtest.reference_result.path.with_suffix(".iterations-hashes.json") + sidecar.write_text(json.dumps(hashes, sort_keys=True, indent=2)) + logging.info(f"Wrote iterations hashes for {systemtest.reference_result.path.name}") + # write readme for tutorial in reference_result_per_tutorial.keys(): reference_results_dir = tutorial.path / "reference-results" diff --git a/tools/tests/systemtests.py b/tools/tests/systemtests.py index 8a37670eb..ce59a5d81 100644 --- a/tools/tests/systemtests.py +++ b/tools/tests/systemtests.py @@ -58,10 +58,14 @@ def main(): for test_suite in test_suites_to_execute: tutorials = test_suite.cases_of_tutorial.keys() for tutorial in tutorials: - for case, reference_result in zip( - test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial]): + cases = test_suite.cases_of_tutorial[tutorial] + reference_results = test_suite.reference_results[tutorial] + max_times = test_suite.max_times.get( + tutorial, [None] * len(cases)) + for case, reference_result, max_time in zip( + cases, reference_results, max_times): systemtests_to_run.append( - Systemtest(tutorial, build_args, case, reference_result)) + Systemtest(tutorial, build_args, case, reference_result, max_time=max_time)) if not systemtests_to_run: raise RuntimeError("Did not find any Systemtests to execute.") diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index bfb1151cf..93cd93709 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -1,11 +1,15 @@ +import hashlib +import json import subprocess -from typing import List, Dict, Optional +from typing import List, Dict, Optional, Tuple from jinja2 import Environment, FileSystemLoader from dataclasses import dataclass, field import shutil from pathlib import Path from paths import PRECICE_REL_OUTPUT_DIR, PRECICE_TOOLS_DIR, PRECICE_REL_REFERENCE_DIR, PRECICE_TESTS_DIR, PRECICE_TUTORIAL_DIR +ITERATIONS_LOGS_DIR = "iterations-logs" + from metadata_parser.metdata import Tutorial, CaseCombination, Case, ReferenceResult from .SystemtestArguments import SystemtestArguments @@ -19,7 +23,7 @@ import os -GLOBAL_TIMEOUT = 600 +GLOBAL_TIMEOUT = 900 SHORT_TIMEOUT = 10 @@ -134,6 +138,7 @@ class Systemtest: arguments: SystemtestArguments case_combination: CaseCombination reference_result: ReferenceResult + max_time: Optional[float] = None params_to_use: Dict[str, str] = field(init=False) env: Dict[str, str] = field(init=False) @@ -413,6 +418,88 @@ def _run_field_compare(self): elapsed_time = time.perf_counter() - time_start return FieldCompareResult(1, stdout_data, stderr_data, self, elapsed_time) + @staticmethod + def _sha256_file(path: Path) -> str: + """Compute SHA-256 hex digest of a file.""" + h = hashlib.sha256() + mv = memoryview(bytearray(128 * 1024)) + with open(path, 'rb', buffering=0) as f: + while n := f.readinto(mv): + h.update(mv[:n]) + return h.hexdigest() + + def _collect_iterations_logs( + self, system_test_dir: Path + ) -> List[Tuple[str, Path]]: + """ + Collect precice-*-iterations.log files from case dirs. + Returns list of (relative_path, absolute_path) e.g. ("solid-fenics/precice-Solid-iterations.log", path). + """ + collected = [] + for case in self.case_combination.cases: + case_dir = system_test_dir / Path(case.path).name + if not case_dir.exists(): + continue + for log_file in case_dir.glob("precice-*-iterations.log"): + if log_file.is_file(): + rel = f"{Path(case.path).name}/{log_file.name}" + collected.append((rel, log_file)) + return collected + + def __archive_iterations_logs(self): + """ + Copy precice-*-iterations.log from case dirs into iterations-logs/ + so they are available in CI artifacts (issue #440). + """ + collected = self._collect_iterations_logs(self.system_test_dir) + if not collected: + return + dest_dir = self.system_test_dir / ITERATIONS_LOGS_DIR + dest_dir.mkdir(exist_ok=True) + for rel, src in collected: + dest_name = Path(rel).name + if len(collected) > 1: + prefix = Path(rel).parent.name + "_" + dest_name = prefix + dest_name + shutil.copy2(src, dest_dir / dest_name) + logging.debug(f"Archived {len(collected)} iterations log(s) to {dest_dir} for {self}") + + def __compare_iterations_hashes(self) -> bool: + """ + Compare current iterations.log hashes against reference sidecar. + Returns True if comparison passes (or is skipped). Returns False if hashes differ. + """ + sidecar = self.reference_result.path.with_suffix(".iterations-hashes.json") + if not sidecar.exists(): + return True + try: + ref_hashes = json.loads(sidecar.read_text()) + except (json.JSONDecodeError, OSError) as e: + logging.warning(f"Could not read iterations hashes from {sidecar}: {e}") + return True + if not ref_hashes: + return True + collected = self._collect_iterations_logs(self.system_test_dir) + current = {rel: self._sha256_file(p) for rel, p in collected} + for rel, expected in ref_hashes.items(): + if rel not in current: + logging.critical( + f"Missing iterations log {rel} (expected from reference); {self} fails" + ) + return False + if current[rel] != expected: + logging.critical( + f"Hash mismatch for {rel} (iterations.log regression); {self} fails" + ) + return False + if len(current) != len(ref_hashes): + extra = set(current) - set(ref_hashes) + logging.critical( + f"Unexpected iterations log(s) {extra}; {self} fails" + ) + return False + return True + def _build_docker(self): """ Builds the docker image @@ -513,11 +600,56 @@ def __write_logs(self, stdout_data: List[str], stderr_data: List[str]): with open(self.system_test_dir / "stderr.log", 'w') as stderr_file: stderr_file.write("\n".join(stderr_data)) + def __apply_precice_max_time_override(self): + """ + If max_time is set, override in precice-config.xml + of the copied tutorial directory. Applies to both test runs and reference generation. + Uses a precise pattern to target only tags. + """ + if self.max_time is None: + return + if not (isinstance(self.max_time, (int, float)) and self.max_time > 0): + logging.warning( + f"Invalid max_time {self.max_time} for {self}; must be a positive number. Skipping override.") + return + config_path = self.system_test_dir / "precice-config.xml" + if not config_path.exists(): + logging.warning( + f"Requested max_time override for {self}, but no precice-config.xml " + f"found in {self.system_test_dir}" + ) + return + try: + text = config_path.read_text() + except Exception as e: + logging.warning(f"Could not read {config_path} to apply max_time override: {e}") + return + # Target only to avoid modifying time-window-size etc. + pattern = r'( tag " + f"found in {config_path}" + ) + return + if len(matches) > 1: + logging.warning( + f"Multiple tags found in {config_path}; overriding all to {self.max_time}" + ) + new_text = re.sub(pattern, rf"\g<1>{self.max_time}\g<3>", text) + try: + config_path.write_text(new_text) + logging.info(f"Overwrote max-time in {config_path} to {self.max_time} for {self}") + except Exception as e: + logging.warning(f"Failed to write updated {config_path}: {e}") + def __prepare_for_run(self, run_directory: Path): """ Prepares the run_directory with folders and datastructures needed for every systemtest execution """ self.__copy_tutorial_into_directory(run_directory) + self.__apply_precice_max_time_override() self.__copy_tools(run_directory) self.__put_gitignore(run_directory) host_uid, host_gid = self.__get_uid_gid() @@ -562,6 +694,21 @@ def run(self, run_directory: Path): solver_time=docker_run_result.runtime, fieldcompare_time=0) + self.__archive_iterations_logs() + if not self.__compare_iterations_hashes(): + self.__write_logs(std_out, std_err) + logging.critical( + f"Iterations.log hash comparison failed (regression), {self} failed" + ) + return SystemtestResult( + False, + std_out, + std_err, + self, + build_time=docker_build_result.runtime, + solver_time=docker_run_result.runtime, + fieldcompare_time=0) + fieldcompare_result = self._run_field_compare() std_out.extend(fieldcompare_result.stdout_data) std_err.extend(fieldcompare_result.stderr_data) diff --git a/tools/tests/systemtests/TestSuite.py b/tools/tests/systemtests/TestSuite.py index 9d8c2ac72..c360bf7de 100644 --- a/tools/tests/systemtests/TestSuite.py +++ b/tools/tests/systemtests/TestSuite.py @@ -10,6 +10,7 @@ class TestSuite: name: str cases_of_tutorial: Dict[Tutorial, List[CaseCombination]] reference_results: Dict[Tutorial, List[ReferenceResult]] + max_times: Dict[Tutorial, List[Optional[float]]] = field(default_factory=dict) def __repr__(self) -> str: return_string = f"Test suite: {self.name} contains:" @@ -48,6 +49,7 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): for test_suite_name in test_suites_raw: case_combinations_of_tutorial = {} reference_results_of_tutorial = {} + max_times_of_tutorial: Dict[Tutorial, List[Optional[float]]] = {} # iterate over tutorials: for tutorial_case in test_suites_raw[test_suite_name]['tutorials']: tutorial = parsed_tutorials.get_by_path(tutorial_case['path']) @@ -57,6 +59,7 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): if tutorial not in case_combinations_of_tutorial: case_combinations_of_tutorial[tutorial] = [] reference_results_of_tutorial[tutorial] = [] + max_times_of_tutorial[tutorial] = [] all_case_combinations = tutorial.case_combinations case_combination_requested = CaseCombination.from_string_list( @@ -65,12 +68,17 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): case_combinations_of_tutorial[tutorial].append(case_combination_requested) reference_results_of_tutorial[tutorial].append(ReferenceResult( tutorial_case['reference_result'], case_combination_requested)) + max_times_of_tutorial[tutorial].append(tutorial_case.get('max_time')) else: raise Exception( f"Could not find the following cases {tutorial_case['case-combination']} in the current metadata of tutorial {tutorial.name}") - testsuites.append(TestSuite(test_suite_name, case_combinations_of_tutorial, - reference_results_of_tutorial)) + testsuites.append(TestSuite( + test_suite_name, + case_combinations_of_tutorial, + reference_results_of_tutorial, + max_times_of_tutorial, + )) return cls(testsuites)