systemtests: add configurable timeout per test case

changelog-entries/735.md

@@ -0,0 +1 @@
+- Add optional `timeout` field to `tests.yaml` entries, allowing per-test timeout configuration. Defaults to 600s for backward compatibility. Applies to the solver run and fieldcompare phases.

tools/tests/README.md

@@ -331,8 +331,11 @@ test_suites:
           - fluid-openfoam
           - solid-fenics
         reference_result: ./flow-over-heated-plate/reference-results/fluid-openfoam_solid-fenics.tar.gz
+        timeout: 3600
 ```
 
+The optional `timeout` field (in seconds, default: 600) sets the maximum time allowed for the solver run and fieldcompare phases. Use this for long-running tutorials such as `heat-exchanger` or `turek-hron-fsi3`.
+
 This defines two test suites, namely `openfoam_adapter_pr` and `openfoam_adapter_release`. Each of them defines which case combinations of which tutorials to run.
 
 ### Generate Reference Results

tools/tests/components.yaml

@@ -32,7 +32,7 @@ python-bindings:
       description: Tutorial git reference to use
       default: "master"
     PYTHON_BINDINGS_REF:
-      semnantic: Git ref of the Python bindings to use
+      description: Git ref of the Python bindings to use
       default: "master"
 
 openfoam-adapter:
@@ -75,10 +75,10 @@ fenics-adapter:
       description: Tutorial git reference to use
       default: "master"
     PYTHON_BINDINGS_REF:
-      semnantic: Git ref of the Python bindings to use
+      description: Git ref of the Python bindings to use
       default: "master"
     FENICS_ADAPTER_REF:
-      semnantic: Git ref of the fenics adapter to use
+      description: Git ref of the fenics adapter to use
       default: "master"
 
 nutils-adapter:
@@ -98,7 +98,7 @@ nutils-adapter:
       description: Tutorial git reference to use
       default: "master"
     PYTHON_BINDINGS_REF:
-      semnantic: Git ref of the Python bindings to use
+      description: Git ref of the Python bindings to use
       default: "master"
 
 calculix-adapter:
@@ -190,7 +190,7 @@ dumux-adapter:
       description: Version of DuMux to use
       default: "3.7"
     DUMUX_ADAPTER_REF:
-      semnantic: Git ref of the dumux adapter to use
+      description: Git ref of the dumux adapter to use
       default: "main"
 
 micro-manager:

tools/tests/generate_reference_results.py

@@ -108,10 +108,11 @@ def main():
     for test_suite in test_suites:
         tutorials = test_suite.cases_of_tutorial.keys()
         for tutorial in tutorials:
-            for case, reference_result in zip(
-                    test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial]):
+            timeouts = test_suite.timeouts.get(tutorial, [600] * len(test_suite.cases_of_tutorial[tutorial]))
+            for case, reference_result, timeout in zip(
+                    test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial], timeouts):
                 systemtests_to_run.add(
-                    Systemtest(tutorial, build_args, case, reference_result))
+                    Systemtest(tutorial, build_args, case, reference_result, timeout=timeout))
 
     reference_result_per_tutorial = {}
     current_time_string = datetime.now().strftime('%Y-%m-%d %H:%M:%S')

tools/tests/metadata_parser/metdata.py

@@ -279,7 +279,6 @@ def from_cases_tuple(cls, cases: Tuple[Case], tutorial: Tutorial):
 class ReferenceResult:
     path: Path
     case_combination: CaseCombination
-
     def __repr__(self) -> str:
         return f"{self.path.as_posix()}"
 

tools/tests/systemtests.py

@@ -58,10 +58,11 @@ def main():
         for test_suite in test_suites_to_execute:
             tutorials = test_suite.cases_of_tutorial.keys()
             for tutorial in tutorials:
-                for case, reference_result in zip(
-                        test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial]):
+                timeouts = test_suite.timeouts.get(tutorial, [600] * len(test_suite.cases_of_tutorial[tutorial]))
+                for case, reference_result, timeout in zip(
+                        test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial], timeouts):
                     systemtests_to_run.append(
-                        Systemtest(tutorial, build_args, case, reference_result))
+                        Systemtest(tutorial, build_args, case, reference_result, timeout=timeout))
 
     if not systemtests_to_run:
         raise RuntimeError("Did not find any Systemtests to execute.")

tools/tests/systemtests/Systemtest.py

@@ -19,7 +19,7 @@
 import os
 
 
-GLOBAL_TIMEOUT = 900
+BUILD_TIMEOUT = 900
 SHORT_TIMEOUT = 10
 
 
@@ -134,6 +134,7 @@ class Systemtest:
     arguments: SystemtestArguments
     case_combination: CaseCombination
     reference_result: ReferenceResult
+    timeout: int = 600
     params_to_use: Dict[str, str] = field(init=False)
     env: Dict[str, str] = field(init=False)
 
@@ -394,7 +395,7 @@ def _run_field_compare(self):
                                        cwd=self.system_test_dir)
 
             try:
-                stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT)
+                stdout, stderr = process.communicate(timeout=self.timeout)
             except KeyboardInterrupt as k:
                 process.kill()
                 raise KeyboardInterrupt from k
@@ -439,7 +440,7 @@ def _build_docker(self):
                                        cwd=self.system_test_dir)
 
             try:
-                stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT)
+                stdout, stderr = process.communicate(timeout=BUILD_TIMEOUT)
             except KeyboardInterrupt as k:
                 process.kill()
                 # process.send_signal(9)
@@ -483,7 +484,7 @@ def _run_tutorial(self):
                                        cwd=self.system_test_dir)
 
             try:
-                stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT)
+                stdout, stderr = process.communicate(timeout=self.timeout)
             except KeyboardInterrupt as k:
                 process.kill()
                 # process.send_signal(9)

tools/tests/systemtests/TestSuite.py

@@ -10,6 +10,7 @@ class TestSuite:
     name: str
     cases_of_tutorial: Dict[Tutorial, List[CaseCombination]]
     reference_results: Dict[Tutorial, List[ReferenceResult]]
+    timeouts: Dict[Tutorial, List[int]] = field(default_factory=dict)
 
     def __repr__(self) -> str:
         return_string = f"Test suite: {self.name} contains:"
@@ -48,6 +49,7 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials):
             for test_suite_name in test_suites_raw:
                 case_combinations_of_tutorial = {}
                 reference_results_of_tutorial = {}
+                timeouts_of_tutorial = {}
                 # iterate over tutorials:
                 for tutorial_case in test_suites_raw[test_suite_name]['tutorials']:
                     tutorial = parsed_tutorials.get_by_path(tutorial_case['path'])
@@ -57,20 +59,25 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials):
                     if tutorial not in case_combinations_of_tutorial:
                         case_combinations_of_tutorial[tutorial] = []
                         reference_results_of_tutorial[tutorial] = []
+                        timeouts_of_tutorial[tutorial] = []
 
                     all_case_combinations = tutorial.case_combinations
                     case_combination_requested = CaseCombination.from_string_list(
                         tutorial_case['case_combination'], tutorial)
                     if case_combination_requested in all_case_combinations:
                         case_combinations_of_tutorial[tutorial].append(case_combination_requested)
+                        timeout = int(tutorial_case.get('timeout', 600))
+                        if timeout <= 0:
+                            raise ValueError(f"timeout must be a positive integer, got {timeout}")
                         reference_results_of_tutorial[tutorial].append(ReferenceResult(
                             tutorial_case['reference_result'], case_combination_requested))
+                        timeouts_of_tutorial[tutorial].append(timeout)
                     else:
                         raise Exception(
                             f"Could not find the following cases {tutorial_case['case-combination']} in the current metadata of tutorial {tutorial.name}")
 
                 testsuites.append(TestSuite(test_suite_name, case_combinations_of_tutorial,
-                                            reference_results_of_tutorial))
+                                            reference_results_of_tutorial, timeouts_of_tutorial))
 
         return cls(testsuites)