From 1ffd4f09600972067e23f1214b60164f4bd39920 Mon Sep 17 00:00:00 2001 From: AdityaGupta716 Date: Mon, 2 Mar 2026 15:18:42 +0530 Subject: [PATCH 1/6] systemtests: add configurable timeout per test case (closes #371) --- changelog-entries/371.md | 1 + tools/tests/README.md | 3 +++ tools/tests/metadata_parser/metdata.py | 1 + tools/tests/systemtests/Systemtest.py | 8 ++++---- tools/tests/systemtests/TestSuite.py | 9 ++++++++- 5 files changed, 17 insertions(+), 5 deletions(-) create mode 100644 changelog-entries/371.md diff --git a/changelog-entries/371.md b/changelog-entries/371.md new file mode 100644 index 000000000..4d9eb0694 --- /dev/null +++ b/changelog-entries/371.md @@ -0,0 +1 @@ +- Add optional `timeout` field to `tests.yaml` entries, allowing per-test timeout configuration. Defaults to 600s for backward compatibility. Applies to the solver run and fieldcompare phases. diff --git a/tools/tests/README.md b/tools/tests/README.md index 5675c47b6..39b890603 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -331,8 +331,11 @@ test_suites: - fluid-openfoam - solid-fenics reference_result: ./flow-over-heated-plate/reference-results/fluid-openfoam_solid-fenics.tar.gz + timeout: 3600 ``` +The optional `timeout` field (in seconds, default: 600) sets the maximum time allowed for the solver run and fieldcompare phases. Use this for long-running tutorials such as `heat-exchanger` or `turek-hron-fsi3`. + This defines two test suites, namely `openfoam_adapter_pr` and `openfoam_adapter_release`. Each of them defines which case combinations of which tutorials to run. ### Generate Reference Results diff --git a/tools/tests/metadata_parser/metdata.py b/tools/tests/metadata_parser/metdata.py index 75b5bb425..55f8c373e 100644 --- a/tools/tests/metadata_parser/metdata.py +++ b/tools/tests/metadata_parser/metdata.py @@ -279,6 +279,7 @@ def from_cases_tuple(cls, cases: Tuple[Case], tutorial: Tutorial): class ReferenceResult: path: Path case_combination: CaseCombination + timeout: int = 600 def __repr__(self) -> str: return f"{self.path.as_posix()}" diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index 6abc5a029..f69794ad0 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -19,7 +19,7 @@ import os -GLOBAL_TIMEOUT = 900 +BUILD_TIMEOUT = 900 SHORT_TIMEOUT = 10 @@ -394,7 +394,7 @@ def _run_field_compare(self): cwd=self.system_test_dir) try: - stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT) + stdout, stderr = process.communicate(timeout=self.reference_result.timeout) except KeyboardInterrupt as k: process.kill() raise KeyboardInterrupt from k @@ -439,7 +439,7 @@ def _build_docker(self): cwd=self.system_test_dir) try: - stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT) + stdout, stderr = process.communicate(timeout=BUILD_TIMEOUT) except KeyboardInterrupt as k: process.kill() # process.send_signal(9) @@ -483,7 +483,7 @@ def _run_tutorial(self): cwd=self.system_test_dir) try: - stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT) + stdout, stderr = process.communicate(timeout=self.reference_result.timeout) except KeyboardInterrupt as k: process.kill() # process.send_signal(9) diff --git a/tools/tests/systemtests/TestSuite.py b/tools/tests/systemtests/TestSuite.py index 9d8c2ac72..f7a0164d8 100644 --- a/tools/tests/systemtests/TestSuite.py +++ b/tools/tests/systemtests/TestSuite.py @@ -63,8 +63,15 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): tutorial_case['case_combination'], tutorial) if case_combination_requested in all_case_combinations: case_combinations_of_tutorial[tutorial].append(case_combination_requested) + timeout_raw = tutorial_case.get('timeout', 600) + if isinstance(timeout_raw, str): + timeout = int(timeout_raw.rstrip('s')) + else: + timeout = int(timeout_raw) + if timeout <= 0: + raise ValueError(f"timeout must be a positive integer, got {timeout}") reference_results_of_tutorial[tutorial].append(ReferenceResult( - tutorial_case['reference_result'], case_combination_requested)) + tutorial_case['reference_result'], case_combination_requested, timeout)) else: raise Exception( f"Could not find the following cases {tutorial_case['case-combination']} in the current metadata of tutorial {tutorial.name}") From 2ce02febbd6f4c1a5d01cc9739bfe4ef8d883e3a Mon Sep 17 00:00:00 2001 From: AdityaGupta716 Date: Fri, 6 Mar 2026 19:03:46 +0530 Subject: [PATCH 2/6] fix: rename changelog entry from issue number to PR number --- changelog-entries/{371.md => 735.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changelog-entries/{371.md => 735.md} (100%) diff --git a/changelog-entries/371.md b/changelog-entries/735.md similarity index 100% rename from changelog-entries/371.md rename to changelog-entries/735.md From 91845d59c8ac826a85fcfbccf8283a16ba73ab38 Mon Sep 17 00:00:00 2001 From: AdityaGupta716 Date: Sun, 8 Mar 2026 22:08:46 +0530 Subject: [PATCH 3/6] Refactor: move timeout from ReferenceResult to Systemtest dataclass --- tools/tests/generate_reference_results.py | 10 ++++++---- tools/tests/metadata_parser/metdata.py | 2 -- tools/tests/systemtests.py | 10 ++++++---- tools/tests/systemtests/Systemtest.py | 5 +++-- tools/tests/systemtests/TestSuite.py | 8 ++++++-- 5 files changed, 21 insertions(+), 14 deletions(-) diff --git a/tools/tests/generate_reference_results.py b/tools/tests/generate_reference_results.py index 055e7b31c..f77ae4206 100644 --- a/tools/tests/generate_reference_results.py +++ b/tools/tests/generate_reference_results.py @@ -108,10 +108,12 @@ def main(): for test_suite in test_suites: tutorials = test_suite.cases_of_tutorial.keys() for tutorial in tutorials: - for case, reference_result in zip( - test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial]): - systemtests_to_run.add( - Systemtest(tutorial, build_args, case, reference_result)) + timeouts = test_suite.timeouts.get(tutorial, [600] * len(test_suite.cases_of_tutorial[tutorial])) + for case, reference_result, timeout in zip( + test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial], timeouts): + systemtest = Systemtest(tutorial, build_args, case, reference_result) + systemtest.timeout = timeout + systemtests_to_run.add(systemtest) reference_result_per_tutorial = {} current_time_string = datetime.now().strftime('%Y-%m-%d %H:%M:%S') diff --git a/tools/tests/metadata_parser/metdata.py b/tools/tests/metadata_parser/metdata.py index 55f8c373e..ebcc2f438 100644 --- a/tools/tests/metadata_parser/metdata.py +++ b/tools/tests/metadata_parser/metdata.py @@ -279,8 +279,6 @@ def from_cases_tuple(cls, cases: Tuple[Case], tutorial: Tutorial): class ReferenceResult: path: Path case_combination: CaseCombination - timeout: int = 600 - def __repr__(self) -> str: return f"{self.path.as_posix()}" diff --git a/tools/tests/systemtests.py b/tools/tests/systemtests.py index 8a37670eb..4cc88be6b 100644 --- a/tools/tests/systemtests.py +++ b/tools/tests/systemtests.py @@ -58,10 +58,12 @@ def main(): for test_suite in test_suites_to_execute: tutorials = test_suite.cases_of_tutorial.keys() for tutorial in tutorials: - for case, reference_result in zip( - test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial]): - systemtests_to_run.append( - Systemtest(tutorial, build_args, case, reference_result)) + timeouts = test_suite.timeouts.get(tutorial, [600] * len(test_suite.cases_of_tutorial[tutorial])) + for case, reference_result, timeout in zip( + test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial], timeouts): + systemtest = Systemtest(tutorial, build_args, case, reference_result) + systemtest.timeout = timeout + systemtests_to_run.append(systemtest) if not systemtests_to_run: raise RuntimeError("Did not find any Systemtests to execute.") diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index f69794ad0..2b17c4496 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -134,6 +134,7 @@ class Systemtest: arguments: SystemtestArguments case_combination: CaseCombination reference_result: ReferenceResult + timeout: int = 600 params_to_use: Dict[str, str] = field(init=False) env: Dict[str, str] = field(init=False) @@ -394,7 +395,7 @@ def _run_field_compare(self): cwd=self.system_test_dir) try: - stdout, stderr = process.communicate(timeout=self.reference_result.timeout) + stdout, stderr = process.communicate(timeout=self.timeout) except KeyboardInterrupt as k: process.kill() raise KeyboardInterrupt from k @@ -483,7 +484,7 @@ def _run_tutorial(self): cwd=self.system_test_dir) try: - stdout, stderr = process.communicate(timeout=self.reference_result.timeout) + stdout, stderr = process.communicate(timeout=self.timeout) except KeyboardInterrupt as k: process.kill() # process.send_signal(9) diff --git a/tools/tests/systemtests/TestSuite.py b/tools/tests/systemtests/TestSuite.py index f7a0164d8..8cf070879 100644 --- a/tools/tests/systemtests/TestSuite.py +++ b/tools/tests/systemtests/TestSuite.py @@ -10,6 +10,7 @@ class TestSuite: name: str cases_of_tutorial: Dict[Tutorial, List[CaseCombination]] reference_results: Dict[Tutorial, List[ReferenceResult]] + timeouts: Dict[Tutorial, List[int]] = field(default_factory=dict) def __repr__(self) -> str: return_string = f"Test suite: {self.name} contains:" @@ -48,6 +49,7 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): for test_suite_name in test_suites_raw: case_combinations_of_tutorial = {} reference_results_of_tutorial = {} + timeouts_of_tutorial = {} # iterate over tutorials: for tutorial_case in test_suites_raw[test_suite_name]['tutorials']: tutorial = parsed_tutorials.get_by_path(tutorial_case['path']) @@ -57,6 +59,7 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): if tutorial not in case_combinations_of_tutorial: case_combinations_of_tutorial[tutorial] = [] reference_results_of_tutorial[tutorial] = [] + timeouts_of_tutorial[tutorial] = [] all_case_combinations = tutorial.case_combinations case_combination_requested = CaseCombination.from_string_list( @@ -71,13 +74,14 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): if timeout <= 0: raise ValueError(f"timeout must be a positive integer, got {timeout}") reference_results_of_tutorial[tutorial].append(ReferenceResult( - tutorial_case['reference_result'], case_combination_requested, timeout)) + tutorial_case['reference_result'], case_combination_requested)) + timeouts_of_tutorial[tutorial].append(timeout) else: raise Exception( f"Could not find the following cases {tutorial_case['case-combination']} in the current metadata of tutorial {tutorial.name}") testsuites.append(TestSuite(test_suite_name, case_combinations_of_tutorial, - reference_results_of_tutorial)) + reference_results_of_tutorial, timeouts_of_tutorial)) return cls(testsuites) From 32ec25d45f30d27f60b46c225f88a7d8a0a6ef61 Mon Sep 17 00:00:00 2001 From: AdityaGupta716 Date: Sun, 8 Mar 2026 22:10:38 +0530 Subject: [PATCH 4/6] Refactor: pass timeout as constructor argument instead of post-construction mutation --- tools/tests/generate_reference_results.py | 5 ++--- tools/tests/systemtests.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/tools/tests/generate_reference_results.py b/tools/tests/generate_reference_results.py index f77ae4206..d4670a1f5 100644 --- a/tools/tests/generate_reference_results.py +++ b/tools/tests/generate_reference_results.py @@ -111,9 +111,8 @@ def main(): timeouts = test_suite.timeouts.get(tutorial, [600] * len(test_suite.cases_of_tutorial[tutorial])) for case, reference_result, timeout in zip( test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial], timeouts): - systemtest = Systemtest(tutorial, build_args, case, reference_result) - systemtest.timeout = timeout - systemtests_to_run.add(systemtest) + systemtests_to_run.add( + Systemtest(tutorial, build_args, case, reference_result, timeout=timeout)) reference_result_per_tutorial = {} current_time_string = datetime.now().strftime('%Y-%m-%d %H:%M:%S') diff --git a/tools/tests/systemtests.py b/tools/tests/systemtests.py index 4cc88be6b..eabae150e 100644 --- a/tools/tests/systemtests.py +++ b/tools/tests/systemtests.py @@ -61,9 +61,8 @@ def main(): timeouts = test_suite.timeouts.get(tutorial, [600] * len(test_suite.cases_of_tutorial[tutorial])) for case, reference_result, timeout in zip( test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial], timeouts): - systemtest = Systemtest(tutorial, build_args, case, reference_result) - systemtest.timeout = timeout - systemtests_to_run.append(systemtest) + systemtests_to_run.append( + Systemtest(tutorial, build_args, case, reference_result, timeout=timeout)) if not systemtests_to_run: raise RuntimeError("Did not find any Systemtests to execute.") From b5af34e15f2f218ca991b60ff2249bc69214f16e Mon Sep 17 00:00:00 2001 From: AdityaGupta716 Date: Mon, 9 Mar 2026 19:26:53 +0530 Subject: [PATCH 5/6] Simplify timeout parsing: accept integers only, remove string rstrip hack --- tools/tests/systemtests/TestSuite.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tools/tests/systemtests/TestSuite.py b/tools/tests/systemtests/TestSuite.py index 8cf070879..6e85c0af8 100644 --- a/tools/tests/systemtests/TestSuite.py +++ b/tools/tests/systemtests/TestSuite.py @@ -66,11 +66,7 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): tutorial_case['case_combination'], tutorial) if case_combination_requested in all_case_combinations: case_combinations_of_tutorial[tutorial].append(case_combination_requested) - timeout_raw = tutorial_case.get('timeout', 600) - if isinstance(timeout_raw, str): - timeout = int(timeout_raw.rstrip('s')) - else: - timeout = int(timeout_raw) + timeout = int(tutorial_case.get('timeout', 600)) if timeout <= 0: raise ValueError(f"timeout must be a positive integer, got {timeout}") reference_results_of_tutorial[tutorial].append(ReferenceResult( From e4b349c3fba7e8ee3b503c7d13a382e99fafd5c9 Mon Sep 17 00:00:00 2001 From: AdityaGupta716 Date: Sun, 15 Mar 2026 16:04:21 +0530 Subject: [PATCH 6/6] fix: correct semnantic typo to description in components.yaml --- tools/tests/components.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/tests/components.yaml b/tools/tests/components.yaml index bd29ba88f..d2c02bdb5 100644 --- a/tools/tests/components.yaml +++ b/tools/tests/components.yaml @@ -32,7 +32,7 @@ python-bindings: description: Tutorial git reference to use default: "master" PYTHON_BINDINGS_REF: - semnantic: Git ref of the Python bindings to use + description: Git ref of the Python bindings to use default: "master" openfoam-adapter: @@ -75,10 +75,10 @@ fenics-adapter: description: Tutorial git reference to use default: "master" PYTHON_BINDINGS_REF: - semnantic: Git ref of the Python bindings to use + description: Git ref of the Python bindings to use default: "master" FENICS_ADAPTER_REF: - semnantic: Git ref of the fenics adapter to use + description: Git ref of the fenics adapter to use default: "master" nutils-adapter: @@ -98,7 +98,7 @@ nutils-adapter: description: Tutorial git reference to use default: "master" PYTHON_BINDINGS_REF: - semnantic: Git ref of the Python bindings to use + description: Git ref of the Python bindings to use default: "master" calculix-adapter: @@ -190,7 +190,7 @@ dumux-adapter: description: Version of DuMux to use default: "3.7" DUMUX_ADAPTER_REF: - semnantic: Git ref of the dumux adapter to use + description: Git ref of the dumux adapter to use default: "main" micro-manager: