diff --git a/changelog-entries/735.md b/changelog-entries/735.md new file mode 100644 index 000000000..4d9eb0694 --- /dev/null +++ b/changelog-entries/735.md @@ -0,0 +1 @@ +- Add optional `timeout` field to `tests.yaml` entries, allowing per-test timeout configuration. Defaults to 600s for backward compatibility. Applies to the solver run and fieldcompare phases. diff --git a/tools/tests/README.md b/tools/tests/README.md index 5675c47b6..39b890603 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -331,8 +331,11 @@ test_suites: - fluid-openfoam - solid-fenics reference_result: ./flow-over-heated-plate/reference-results/fluid-openfoam_solid-fenics.tar.gz + timeout: 3600 ``` +The optional `timeout` field (in seconds, default: 600) sets the maximum time allowed for the solver run and fieldcompare phases. Use this for long-running tutorials such as `heat-exchanger` or `turek-hron-fsi3`. + This defines two test suites, namely `openfoam_adapter_pr` and `openfoam_adapter_release`. Each of them defines which case combinations of which tutorials to run. ### Generate Reference Results diff --git a/tools/tests/components.yaml b/tools/tests/components.yaml index bd29ba88f..d2c02bdb5 100644 --- a/tools/tests/components.yaml +++ b/tools/tests/components.yaml @@ -32,7 +32,7 @@ python-bindings: description: Tutorial git reference to use default: "master" PYTHON_BINDINGS_REF: - semnantic: Git ref of the Python bindings to use + description: Git ref of the Python bindings to use default: "master" openfoam-adapter: @@ -75,10 +75,10 @@ fenics-adapter: description: Tutorial git reference to use default: "master" PYTHON_BINDINGS_REF: - semnantic: Git ref of the Python bindings to use + description: Git ref of the Python bindings to use default: "master" FENICS_ADAPTER_REF: - semnantic: Git ref of the fenics adapter to use + description: Git ref of the fenics adapter to use default: "master" nutils-adapter: @@ -98,7 +98,7 @@ nutils-adapter: description: Tutorial git reference to use default: "master" PYTHON_BINDINGS_REF: - semnantic: Git ref of the Python bindings to use + description: Git ref of the Python bindings to use default: "master" calculix-adapter: @@ -190,7 +190,7 @@ dumux-adapter: description: Version of DuMux to use default: "3.7" DUMUX_ADAPTER_REF: - semnantic: Git ref of the dumux adapter to use + description: Git ref of the dumux adapter to use default: "main" micro-manager: diff --git a/tools/tests/generate_reference_results.py b/tools/tests/generate_reference_results.py index 055e7b31c..d4670a1f5 100644 --- a/tools/tests/generate_reference_results.py +++ b/tools/tests/generate_reference_results.py @@ -108,10 +108,11 @@ def main(): for test_suite in test_suites: tutorials = test_suite.cases_of_tutorial.keys() for tutorial in tutorials: - for case, reference_result in zip( - test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial]): + timeouts = test_suite.timeouts.get(tutorial, [600] * len(test_suite.cases_of_tutorial[tutorial])) + for case, reference_result, timeout in zip( + test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial], timeouts): systemtests_to_run.add( - Systemtest(tutorial, build_args, case, reference_result)) + Systemtest(tutorial, build_args, case, reference_result, timeout=timeout)) reference_result_per_tutorial = {} current_time_string = datetime.now().strftime('%Y-%m-%d %H:%M:%S') diff --git a/tools/tests/metadata_parser/metdata.py b/tools/tests/metadata_parser/metdata.py index 75b5bb425..ebcc2f438 100644 --- a/tools/tests/metadata_parser/metdata.py +++ b/tools/tests/metadata_parser/metdata.py @@ -279,7 +279,6 @@ def from_cases_tuple(cls, cases: Tuple[Case], tutorial: Tutorial): class ReferenceResult: path: Path case_combination: CaseCombination - def __repr__(self) -> str: return f"{self.path.as_posix()}" diff --git a/tools/tests/systemtests.py b/tools/tests/systemtests.py index 8a37670eb..eabae150e 100644 --- a/tools/tests/systemtests.py +++ b/tools/tests/systemtests.py @@ -58,10 +58,11 @@ def main(): for test_suite in test_suites_to_execute: tutorials = test_suite.cases_of_tutorial.keys() for tutorial in tutorials: - for case, reference_result in zip( - test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial]): + timeouts = test_suite.timeouts.get(tutorial, [600] * len(test_suite.cases_of_tutorial[tutorial])) + for case, reference_result, timeout in zip( + test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial], timeouts): systemtests_to_run.append( - Systemtest(tutorial, build_args, case, reference_result)) + Systemtest(tutorial, build_args, case, reference_result, timeout=timeout)) if not systemtests_to_run: raise RuntimeError("Did not find any Systemtests to execute.") diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index 6abc5a029..2b17c4496 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -19,7 +19,7 @@ import os -GLOBAL_TIMEOUT = 900 +BUILD_TIMEOUT = 900 SHORT_TIMEOUT = 10 @@ -134,6 +134,7 @@ class Systemtest: arguments: SystemtestArguments case_combination: CaseCombination reference_result: ReferenceResult + timeout: int = 600 params_to_use: Dict[str, str] = field(init=False) env: Dict[str, str] = field(init=False) @@ -394,7 +395,7 @@ def _run_field_compare(self): cwd=self.system_test_dir) try: - stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT) + stdout, stderr = process.communicate(timeout=self.timeout) except KeyboardInterrupt as k: process.kill() raise KeyboardInterrupt from k @@ -439,7 +440,7 @@ def _build_docker(self): cwd=self.system_test_dir) try: - stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT) + stdout, stderr = process.communicate(timeout=BUILD_TIMEOUT) except KeyboardInterrupt as k: process.kill() # process.send_signal(9) @@ -483,7 +484,7 @@ def _run_tutorial(self): cwd=self.system_test_dir) try: - stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT) + stdout, stderr = process.communicate(timeout=self.timeout) except KeyboardInterrupt as k: process.kill() # process.send_signal(9) diff --git a/tools/tests/systemtests/TestSuite.py b/tools/tests/systemtests/TestSuite.py index 9d8c2ac72..6e85c0af8 100644 --- a/tools/tests/systemtests/TestSuite.py +++ b/tools/tests/systemtests/TestSuite.py @@ -10,6 +10,7 @@ class TestSuite: name: str cases_of_tutorial: Dict[Tutorial, List[CaseCombination]] reference_results: Dict[Tutorial, List[ReferenceResult]] + timeouts: Dict[Tutorial, List[int]] = field(default_factory=dict) def __repr__(self) -> str: return_string = f"Test suite: {self.name} contains:" @@ -48,6 +49,7 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): for test_suite_name in test_suites_raw: case_combinations_of_tutorial = {} reference_results_of_tutorial = {} + timeouts_of_tutorial = {} # iterate over tutorials: for tutorial_case in test_suites_raw[test_suite_name]['tutorials']: tutorial = parsed_tutorials.get_by_path(tutorial_case['path']) @@ -57,20 +59,25 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): if tutorial not in case_combinations_of_tutorial: case_combinations_of_tutorial[tutorial] = [] reference_results_of_tutorial[tutorial] = [] + timeouts_of_tutorial[tutorial] = [] all_case_combinations = tutorial.case_combinations case_combination_requested = CaseCombination.from_string_list( tutorial_case['case_combination'], tutorial) if case_combination_requested in all_case_combinations: case_combinations_of_tutorial[tutorial].append(case_combination_requested) + timeout = int(tutorial_case.get('timeout', 600)) + if timeout <= 0: + raise ValueError(f"timeout must be a positive integer, got {timeout}") reference_results_of_tutorial[tutorial].append(ReferenceResult( tutorial_case['reference_result'], case_combination_requested)) + timeouts_of_tutorial[tutorial].append(timeout) else: raise Exception( f"Could not find the following cases {tutorial_case['case-combination']} in the current metadata of tutorial {tutorial.name}") testsuites.append(TestSuite(test_suite_name, case_combinations_of_tutorial, - reference_results_of_tutorial)) + reference_results_of_tutorial, timeouts_of_tutorial)) return cls(testsuites)