From 35bceeacc2962a79c932f1ed26c14652286f0b12 Mon Sep 17 00:00:00 2001 From: AdityaGupta716 Date: Sat, 7 Mar 2026 09:38:52 +0530 Subject: [PATCH 1/4] systemtests: archive fieldcompare diff files on failure --- changelog-entries/742.md | 1 + tools/tests/README.md | 2 +- tools/tests/systemtests/Systemtest.py | 31 ++++++++++++++++++++++++++- 3 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 changelog-entries/742.md diff --git a/changelog-entries/742.md b/changelog-entries/742.md new file mode 100644 index 000000000..72f0991e0 --- /dev/null +++ b/changelog-entries/742.md @@ -0,0 +1 @@ +- Archive fieldcompare diff files to `diff-results/` on failure so they are included in the CI artifact and can be downloaded for inspection. diff --git a/tools/tests/README.md b/tools/tests/README.md index 5675c47b6..cd887e54f 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -105,7 +105,7 @@ In this case, building and running seems to work out, but the tests fail because The easiest way to debug a systemtest run is first to have a look at the output written into the action on GitHub. If this does not provide enough hints, the next step is to download the generated `system_tests_run__` artifact. Note that by default this will only be generated if the systemtests fail. -Inside the archive, a test-specific subfolder like `flow-over-heated-plate_fluid-openfoam-solid-fenics_2023-11-19-211723` contains two log files: a `stderr.log` and `stdout.log`. This can be a starting point for a further investigation. +Inside the archive, a test-specific subfolder like `flow-over-heated-plate_fluid-openfoam-solid-fenics_2023-11-19-211723` contains two log files: a `stderr.log` and `stdout.log`. This can be a starting point for a further investigation. If fieldcompare detected differences, any diff VTK files (e.g. `diff_*.vtu`) are copied into a `diff-results/` subfolder in the same directory — open these in ParaView to see exactly where results diverge from the reference. ## Adding new tests diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index bfb1151cf..07a247de8 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -19,7 +19,8 @@ import os -GLOBAL_TIMEOUT = 600 +GLOBAL_TIMEOUT = 900 +DIFF_RESULTS_DIR = "diff-results" SHORT_TIMEOUT = 10 @@ -513,6 +514,33 @@ def __write_logs(self, stdout_data: List[str], stderr_data: List[str]): with open(self.system_test_dir / "stderr.log", 'w') as stderr_file: stderr_file.write("\n".join(stderr_data)) + def __archive_diff_files(self): + """ + Copies any diff VTK files produced by fieldcompare into a dedicated + diff-results/ folder inside the system test directory so they are + included in the CI artifact and can be downloaded for inspection. + """ + precice_exports = self.system_test_dir / PRECICE_REL_OUTPUT_DIR + diff_dest = self.system_test_dir / DIFF_RESULTS_DIR + if not precice_exports.exists(): + logging.debug("No precice-exports directory found, skipping diff file archiving") + return + diff_files = (list(precice_exports.rglob("diff_*.vtu")) + + list(precice_exports.rglob("diff_*.vtk")) + + list(precice_exports.rglob("diff_*.vtp"))) + if not diff_files: + logging.debug("No diff files found in precice-exports, skipping archiving") + return + diff_dest.mkdir(exist_ok=True) + for diff_file in diff_files: + rel_path = diff_file.relative_to(precice_exports) + dest_file = diff_dest / rel_path + dest_file.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(diff_file, dest_file) + logging.debug(f"Archived diff file: {rel_path} -> {diff_dest}") + logging.info(f"Archived {len(diff_files)} diff file(s) to {diff_dest}") + + def __prepare_for_run(self, run_directory: Path): """ Prepares the run_directory with folders and datastructures needed for every systemtest execution @@ -566,6 +594,7 @@ def run(self, run_directory: Path): std_out.extend(fieldcompare_result.stdout_data) std_err.extend(fieldcompare_result.stderr_data) if fieldcompare_result.exit_code != 0: + self.__archive_diff_files() self.__write_logs(std_out, std_err) logging.critical(f"Fieldcompare returned non zero exit code, therefore {self} failed") return SystemtestResult( From aa4d631db9142580985a22dc451632df2e38ffe2 Mon Sep 17 00:00:00 2001 From: AdityaGupta716 Date: Sun, 8 Mar 2026 07:07:30 +0530 Subject: [PATCH 2/4] fix: rename changelog entry to reference issue #441 --- changelog-entries/{742.md => 441.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changelog-entries/{742.md => 441.md} (100%) diff --git a/changelog-entries/742.md b/changelog-entries/441.md similarity index 100% rename from changelog-entries/742.md rename to changelog-entries/441.md From f9e96b037b924c70b2fca2f2bf0064cbbbb021b5 Mon Sep 17 00:00:00 2001 From: AdityaGupta716 Date: Sun, 8 Mar 2026 12:57:44 +0530 Subject: [PATCH 3/4] fix: improve __archive_diff_files robustness --- tools/tests/systemtests/Systemtest.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index 07a247de8..8972c3d04 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -516,20 +516,33 @@ def __write_logs(self, stdout_data: List[str], stderr_data: List[str]): def __archive_diff_files(self): """ - Copies any diff VTK files produced by fieldcompare into a dedicated + Copies any diff files produced by fieldcompare into a dedicated diff-results/ folder inside the system test directory so they are included in the CI artifact and can be downloaded for inspection. + + Collects all files matching diff_* (any format fieldcompare may produce, + including .vtu, .vtk, .vtp, .hdf, .h5, .csv) to avoid silently missing + non-VTK diff outputs. """ precice_exports = self.system_test_dir / PRECICE_REL_OUTPUT_DIR diff_dest = self.system_test_dir / DIFF_RESULTS_DIR + try: + diff_dest.relative_to(precice_exports) + logging.warning( + f"diff-results dir {diff_dest} is inside precice-exports {precice_exports}; " + "skipping archiving to avoid self-copy loop") + return + except ValueError: + pass # Expected: diff_dest is not under precice_exports, safe to proceed + if not precice_exports.exists(): logging.debug("No precice-exports directory found, skipping diff file archiving") return - diff_files = (list(precice_exports.rglob("diff_*.vtu")) + - list(precice_exports.rglob("diff_*.vtk")) + - list(precice_exports.rglob("diff_*.vtp"))) + diff_files = [f for f in precice_exports.rglob("diff_*") if f.is_file()] if not diff_files: - logging.debug("No diff files found in precice-exports, skipping archiving") + logging.warning( + f"Fieldcompare failed but no diff_* files were found in {precice_exports}; " + "results may have diverged without producing diff output (check tolerances or output format)") return diff_dest.mkdir(exist_ok=True) for diff_file in diff_files: @@ -537,7 +550,7 @@ def __archive_diff_files(self): dest_file = diff_dest / rel_path dest_file.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(diff_file, dest_file) - logging.debug(f"Archived diff file: {rel_path} -> {diff_dest}") + logging.debug(f"Archived diff file: {rel_path} -> {dest_file}") logging.info(f"Archived {len(diff_files)} diff file(s) to {diff_dest}") From b36fef4e034808dc38fd6b4bf3043b8fae4769b3 Mon Sep 17 00:00:00 2001 From: AdityaGupta716 Date: Sun, 15 Mar 2026 16:07:47 +0530 Subject: [PATCH 4/4] fix: align timeout constants with add-timeout-support; fix semnantic typo; fix double blank line --- tools/tests/components.yaml | 10 +++++----- tools/tests/systemtests/Systemtest.py | 9 ++++----- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/tools/tests/components.yaml b/tools/tests/components.yaml index bd29ba88f..d2c02bdb5 100644 --- a/tools/tests/components.yaml +++ b/tools/tests/components.yaml @@ -32,7 +32,7 @@ python-bindings: description: Tutorial git reference to use default: "master" PYTHON_BINDINGS_REF: - semnantic: Git ref of the Python bindings to use + description: Git ref of the Python bindings to use default: "master" openfoam-adapter: @@ -75,10 +75,10 @@ fenics-adapter: description: Tutorial git reference to use default: "master" PYTHON_BINDINGS_REF: - semnantic: Git ref of the Python bindings to use + description: Git ref of the Python bindings to use default: "master" FENICS_ADAPTER_REF: - semnantic: Git ref of the fenics adapter to use + description: Git ref of the fenics adapter to use default: "master" nutils-adapter: @@ -98,7 +98,7 @@ nutils-adapter: description: Tutorial git reference to use default: "master" PYTHON_BINDINGS_REF: - semnantic: Git ref of the Python bindings to use + description: Git ref of the Python bindings to use default: "master" calculix-adapter: @@ -190,7 +190,7 @@ dumux-adapter: description: Version of DuMux to use default: "3.7" DUMUX_ADAPTER_REF: - semnantic: Git ref of the dumux adapter to use + description: Git ref of the dumux adapter to use default: "main" micro-manager: diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index 8972c3d04..22e6e4265 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -19,7 +19,7 @@ import os -GLOBAL_TIMEOUT = 900 +BUILD_TIMEOUT = 900 DIFF_RESULTS_DIR = "diff-results" SHORT_TIMEOUT = 10 @@ -395,7 +395,7 @@ def _run_field_compare(self): cwd=self.system_test_dir) try: - stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT) + stdout, stderr = process.communicate(timeout=self.timeout) except KeyboardInterrupt as k: process.kill() raise KeyboardInterrupt from k @@ -440,7 +440,7 @@ def _build_docker(self): cwd=self.system_test_dir) try: - stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT) + stdout, stderr = process.communicate(timeout=BUILD_TIMEOUT) except KeyboardInterrupt as k: process.kill() # process.send_signal(9) @@ -484,7 +484,7 @@ def _run_tutorial(self): cwd=self.system_test_dir) try: - stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT) + stdout, stderr = process.communicate(timeout=self.timeout) except KeyboardInterrupt as k: process.kill() # process.send_signal(9) @@ -553,7 +553,6 @@ def __archive_diff_files(self): logging.debug(f"Archived diff file: {rel_path} -> {dest_file}") logging.info(f"Archived {len(diff_files)} diff file(s) to {diff_dest}") - def __prepare_for_run(self, run_directory: Path): """ Prepares the run_directory with folders and datastructures needed for every systemtest execution