systemtests: fix Python bindings version ignored due to tutorial venv (fixes #584)

changelog-entries/584.md

@@ -0,0 +1 @@
+- Fix Python bindings version being ignored in system tests: permanently activate `/home/precice/venv` in the `python_bindings`, `fenics_adapter`, and `nutils_adapter` Docker stages via `VIRTUAL_ENV` and `PATH` env vars; set `PRECICE_TUTORIALS_NO_VENV=1` in `python-bindings`, `fenics-adapter`, `nutils-adapter`, and `su2-adapter` component templates so tutorial run scripts skip creating their own venv; add a sanity check that fails fast if precice is not importable when the escape hatch is set (fixes #584).

flow-over-heated-plate/fluid-su2/run.sh

@@ -6,7 +6,7 @@ exec > >(tee --append "$LOGFILE") 2>&1
 
 if [ ! -v PRECICE_TUTORIALS_NO_VENV ]
 then
-    python3 -m venv --system-site-package .venv
+    python3 -m venv --system-site-packages .venv
     . .venv/bin/activate
     pip install -r requirements.txt && pip freeze > pip-installed-packages.log
 fi

tools/tests/component-templates/fenics-adapter.yaml

@@ -5,12 +5,15 @@ build:
       - {{key}}={{value}}
     {% endfor %}
   target: fenics_adapter
+environment:
+  - PRECICE_TUTORIALS_NO_VENV=1
 depends_on:    
   prepare:
     condition: service_completed_successfully
 volumes:
   - {{ run_directory }}:/runs
 command: >
-  /bin/bash -c "id && 
+  /bin/bash -c "id &&
+  [ -n \"$$PRECICE_TUTORIALS_NO_VENV\" ] && (python3 -c 'import precice' || { echo 'ERROR: PRECICE_TUTORIALS_NO_VENV set but precice not importable. Check Docker image.'; exit 1; }) ;
   cd '/runs/{{ tutorial_folder }}/{{ case_folder }}' &&
   {{ run }} | tee system-tests_{{ case_folder }}.log 2>&1"

tools/tests/component-templates/nutils-adapter.yaml

@@ -5,12 +5,15 @@ build:
       - {{key}}={{value}}
     {% endfor %}
   target: nutils_adapter
+environment:
+  - PRECICE_TUTORIALS_NO_VENV=1
 depends_on:    
   prepare:
     condition: service_completed_successfully
 volumes:
   - {{ run_directory }}:/runs
 command: >
-  /bin/bash -c "id && 
+  /bin/bash -c "id &&
+  [ -n \"$$PRECICE_TUTORIALS_NO_VENV\" ] && (python3 -c 'import precice' || { echo 'ERROR: PRECICE_TUTORIALS_NO_VENV set but precice not importable. Check Docker image.'; exit 1; }) ;
   cd '/runs/{{ tutorial_folder }}/{{ case_folder }}' &&
   {{ run }} | tee system-tests_{{ case_folder }}.log 2>&1"

tools/tests/component-templates/python-bindings.yaml

@@ -5,12 +5,15 @@ build:
       - {{key}}={{value}}
     {% endfor %}
   target: python_bindings
+environment:
+  - PRECICE_TUTORIALS_NO_VENV=1
 depends_on:    
   prepare:
     condition: service_completed_successfully
 volumes:
   - {{ run_directory }}:/runs
 command: >
-  /bin/bash -c "id && 
+  /bin/bash -c "id &&
+  [ -n \"$$PRECICE_TUTORIALS_NO_VENV\" ] && (python3 -c 'import precice' || { echo 'ERROR: PRECICE_TUTORIALS_NO_VENV set but precice not importable. Check Docker image.'; exit 1; }) ;
   cd '/runs/{{ tutorial_folder }}/{{ case_folder }}' &&
   {{ run }} | tee system-tests_{{ case_folder }}.log 2>&1"

tools/tests/component-templates/su2-adapter.yaml

@@ -5,12 +5,15 @@ build:
       - {{key}}={{value}}
     {% endfor %}
   target: su2_adapter
+environment:
+  - PRECICE_TUTORIALS_NO_VENV=1
 depends_on:    
   prepare:
     condition: service_completed_successfully
 volumes:
   - {{ run_directory }}:/runs
 command: >
-  /bin/bash -c "id && 
+  /bin/bash -c "id &&
+  [ -n \"$$PRECICE_TUTORIALS_NO_VENV\" ] && (python3 -c 'import precice' || { echo 'ERROR: PRECICE_TUTORIALS_NO_VENV set but precice not importable. Check Docker image.'; exit 1; }) ;
   cd '/runs/{{ tutorial_folder }}/{{ case_folder }}' &&
   SU2_RUN="/home/precice/SU2_RUN/bin" PYTHONPATH="/home/precice/SU2_RUN/bin:$PYTHONPATH" {{ run }} | tee system-tests_{{ case_folder }}.log 2>&1"

tools/tests/components.yaml

@@ -32,7 +32,7 @@ python-bindings:
       description: Tutorial git reference to use
       default: "master"
     PYTHON_BINDINGS_REF:
-      semnantic: Git ref of the Python bindings to use
+      description: Git ref of the Python bindings to use
       default: "master"
 
 openfoam-adapter:
@@ -75,10 +75,10 @@ fenics-adapter:
       description: Tutorial git reference to use
       default: "master"
     PYTHON_BINDINGS_REF:
-      semnantic: Git ref of the Python bindings to use
+      description: Git ref of the Python bindings to use
       default: "master"
     FENICS_ADAPTER_REF:
-      semnantic: Git ref of the fenics adapter to use
+      description: Git ref of the fenics adapter to use
       default: "master"
 
 nutils-adapter:
@@ -98,7 +98,7 @@ nutils-adapter:
       description: Tutorial git reference to use
       default: "master"
     PYTHON_BINDINGS_REF:
-      semnantic: Git ref of the Python bindings to use
+      description: Git ref of the Python bindings to use
       default: "master"
 
 calculix-adapter:
@@ -190,7 +190,7 @@ dumux-adapter:
       description: Version of DuMux to use
       default: "3.7"
     DUMUX_ADAPTER_REF:
-      semnantic: Git ref of the dumux adapter to use
+      description: Git ref of the dumux adapter to use
       default: "main"
 
 micro-manager:

tools/tests/dockerfiles/ubuntu_2404/Dockerfile

@@ -91,7 +91,9 @@ WORKDIR /home/precice
 RUN python3 -m venv /home/precice/venv && \
     . /home/precice/venv/bin/activate && \
     pip3 install git+https://github.com/precice/python-bindings.git@${PYTHON_BINDINGS_REF} && \
-    pip3 install matplotlib 
+    pip3 install matplotlib
+ENV VIRTUAL_ENV="/home/precice/venv"
+ENV PATH="/home/precice/venv/bin:$PATH"
 
 FROM precice_dependecies AS fenics_adapter
 COPY --from=python_bindings /home/precice/.local /home/precice/.local
@@ -105,6 +107,8 @@ ARG FENICS_ADAPTER_REF
 RUN python3 -m venv --system-site-packages /home/precice/venv && \
     . /home/precice/venv/bin/activate && \
     pip3 install git+https://github.com/precice/fenics-adapter.git@${FENICS_ADAPTER_REF}
+ENV VIRTUAL_ENV="/home/precice/venv"
+ENV PATH="/home/precice/venv/bin:$PATH"
 
 
 FROM precice_dependecies AS nutils_adapter
@@ -114,6 +118,8 @@ USER precice
 RUN python3 -m venv /home/precice/venv && \
     . /home/precice/venv/bin/activate && \
     pip3 install nutils
+ENV VIRTUAL_ENV="/home/precice/venv"
+ENV PATH="/home/precice/venv/bin:$PATH"
 
 
 FROM precice_dependecies AS calculix_adapter

tools/tests/systemtests/Systemtest.py

@@ -19,7 +19,7 @@
 import os
 
 
-GLOBAL_TIMEOUT = 900
+BUILD_TIMEOUT = 900
 SHORT_TIMEOUT = 10
 
 
@@ -394,7 +394,7 @@ def _run_field_compare(self):
                                        cwd=self.system_test_dir)
 
             try:
-                stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT)
+                stdout, stderr = process.communicate(timeout=self.timeout)
             except KeyboardInterrupt as k:
                 process.kill()
                 raise KeyboardInterrupt from k
@@ -439,7 +439,7 @@ def _build_docker(self):
                                        cwd=self.system_test_dir)
 
             try:
-                stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT)
+                stdout, stderr = process.communicate(timeout=BUILD_TIMEOUT)
             except KeyboardInterrupt as k:
                 process.kill()
                 # process.send_signal(9)
@@ -483,7 +483,7 @@ def _run_tutorial(self):
                                        cwd=self.system_test_dir)
 
             try:
-                stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT)
+                stdout, stderr = process.communicate(timeout=self.timeout)
             except KeyboardInterrupt as k:
                 process.kill()
                 # process.send_signal(9)