systemtests: archive fieldcompare diff files on failure

changelog-entries/441.md

@@ -0,0 +1 @@
+- Archive fieldcompare diff files to `diff-results/` on failure so they are included in the CI artifact and can be downloaded for inspection.

tools/tests/README.md

@@ -105,7 +105,7 @@ In this case, building and running seems to work out, but the tests fail because
 
 The easiest way to debug a systemtest run is first to have a look at the output written into the action on GitHub.
 If this does not provide enough hints, the next step is to download the generated `system_tests_run_<run_id>_<run_attempt>` artifact. Note that by default this will only be generated if the systemtests fail.
-Inside the archive, a test-specific subfolder like `flow-over-heated-plate_fluid-openfoam-solid-fenics_2023-11-19-211723` contains two log files: a `stderr.log` and `stdout.log`. This can be a starting point for a further investigation.
+Inside the archive, a test-specific subfolder like `flow-over-heated-plate_fluid-openfoam-solid-fenics_2023-11-19-211723` contains two log files: a `stderr.log` and `stdout.log`. This can be a starting point for a further investigation. If fieldcompare detected differences, any diff VTK files (e.g. `diff_*.vtu`) are copied into a `diff-results/` subfolder in the same directory — open these in ParaView to see exactly where results diverge from the reference.
 
 ## Adding new tests
 

tools/tests/components.yaml

@@ -32,7 +32,7 @@ python-bindings:
       description: Tutorial git reference to use
       default: "master"
     PYTHON_BINDINGS_REF:
-      semnantic: Git ref of the Python bindings to use
+      description: Git ref of the Python bindings to use
       default: "master"
 
 openfoam-adapter:
@@ -75,10 +75,10 @@ fenics-adapter:
       description: Tutorial git reference to use
       default: "master"
     PYTHON_BINDINGS_REF:
-      semnantic: Git ref of the Python bindings to use
+      description: Git ref of the Python bindings to use
       default: "master"
     FENICS_ADAPTER_REF:
-      semnantic: Git ref of the fenics adapter to use
+      description: Git ref of the fenics adapter to use
       default: "master"
 
 nutils-adapter:
@@ -98,7 +98,7 @@ nutils-adapter:
       description: Tutorial git reference to use
       default: "master"
     PYTHON_BINDINGS_REF:
-      semnantic: Git ref of the Python bindings to use
+      description: Git ref of the Python bindings to use
       default: "master"
 
 calculix-adapter:
@@ -190,7 +190,7 @@ dumux-adapter:
       description: Version of DuMux to use
       default: "3.7"
     DUMUX_ADAPTER_REF:
-      semnantic: Git ref of the dumux adapter to use
+      description: Git ref of the dumux adapter to use
       default: "main"
 
 micro-manager:

tools/tests/systemtests/Systemtest.py

@@ -19,7 +19,8 @@
 import os
 
 
-GLOBAL_TIMEOUT = 900
+BUILD_TIMEOUT = 900
+DIFF_RESULTS_DIR = "diff-results"
 SHORT_TIMEOUT = 10
 
 
@@ -394,7 +395,7 @@ def _run_field_compare(self):
                                        cwd=self.system_test_dir)
 
             try:
-                stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT)
+                stdout, stderr = process.communicate(timeout=self.timeout)
             except KeyboardInterrupt as k:
                 process.kill()
                 raise KeyboardInterrupt from k
@@ -439,7 +440,7 @@ def _build_docker(self):
                                        cwd=self.system_test_dir)
 
             try:
-                stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT)
+                stdout, stderr = process.communicate(timeout=BUILD_TIMEOUT)
             except KeyboardInterrupt as k:
                 process.kill()
                 # process.send_signal(9)
@@ -483,7 +484,7 @@ def _run_tutorial(self):
                                        cwd=self.system_test_dir)
 
             try:
-                stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT)
+                stdout, stderr = process.communicate(timeout=self.timeout)
             except KeyboardInterrupt as k:
                 process.kill()
                 # process.send_signal(9)
@@ -513,6 +514,45 @@ def __write_logs(self, stdout_data: List[str], stderr_data: List[str]):
         with open(self.system_test_dir / "stderr.log", 'w') as stderr_file:
             stderr_file.write("\n".join(stderr_data))
 
+    def __archive_diff_files(self):
+        """
+        Copies any diff files produced by fieldcompare into a dedicated
+        diff-results/ folder inside the system test directory so they are
+        included in the CI artifact and can be downloaded for inspection.
+
+        Collects all files matching diff_* (any format fieldcompare may produce,
+        including .vtu, .vtk, .vtp, .hdf, .h5, .csv) to avoid silently missing
+        non-VTK diff outputs.
+        """
+        precice_exports = self.system_test_dir / PRECICE_REL_OUTPUT_DIR
+        diff_dest = self.system_test_dir / DIFF_RESULTS_DIR
+        try:
+            diff_dest.relative_to(precice_exports)
+            logging.warning(
+                f"diff-results dir {diff_dest} is inside precice-exports {precice_exports}; "
+                "skipping archiving to avoid self-copy loop")
+            return
+        except ValueError:
+            pass  # Expected: diff_dest is not under precice_exports, safe to proceed
+
+        if not precice_exports.exists():
+            logging.debug("No precice-exports directory found, skipping diff file archiving")
+            return
+        diff_files = [f for f in precice_exports.rglob("diff_*") if f.is_file()]
+        if not diff_files:
+            logging.warning(
+                f"Fieldcompare failed but no diff_* files were found in {precice_exports}; "
+                "results may have diverged without producing diff output (check tolerances or output format)")
+            return
+        diff_dest.mkdir(exist_ok=True)
+        for diff_file in diff_files:
+            rel_path = diff_file.relative_to(precice_exports)
+            dest_file = diff_dest / rel_path
+            dest_file.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy2(diff_file, dest_file)
+            logging.debug(f"Archived diff file: {rel_path} -> {dest_file}")
+        logging.info(f"Archived {len(diff_files)} diff file(s) to {diff_dest}")
+
     def __prepare_for_run(self, run_directory: Path):
         """
         Prepares the run_directory with folders and datastructures needed for every systemtest execution
@@ -566,6 +606,7 @@ def run(self, run_directory: Path):
         std_out.extend(fieldcompare_result.stdout_data)
         std_err.extend(fieldcompare_result.stderr_data)
         if fieldcompare_result.exit_code != 0:
+            self.__archive_diff_files()
             self.__write_logs(std_out, std_err)
             logging.critical(f"Fieldcompare returned non zero exit code, therefore {self} failed")
             return SystemtestResult(