From 3a1251fad2ef4b3eda90e897a35279fc82db2e79 Mon Sep 17 00:00:00 2001 From: Eugene Vinitsky Date: Fri, 10 Jul 2026 10:58:50 -0400 Subject: [PATCH 1/2] Pin smoke Dockerfile to amd64; guard missing overlay template The smoke image is x86-64-only (bit-exact AVX2 golden, qemu-x86_64-static, amd64-only raylib archive in setup.py) but had no platform pin, so builds on Apple Silicon produced arm64 images that fail at link time. Pin FROM to linux/amd64 and document the constraint in the header comment. setup_container.sh create-overlay blindly cp'd the NYU-Greene-only OVERLAY_TEMPLATE default under set -e, giving a bare "No such file or directory" elsewhere. Add a fail-fast guard that names the path and points at OVERLAY_TEMPLATE / apptainer overlay create. Document the NYU-only defaults in README.md and docs/cluster_training.md. Co-Authored-By: Claude Fable 5 --- AGENTIC_PR | 11 +++++++++++ README.md | 2 +- docs/cluster_training.md | 2 ++ scripts/setup_container.sh | 9 +++++++++ tests/smoke_tests/Dockerfile | 8 +++++++- 5 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 AGENTIC_PR diff --git a/AGENTIC_PR b/AGENTIC_PR new file mode 100644 index 0000000000..21089c2a2e --- /dev/null +++ b/AGENTIC_PR @@ -0,0 +1,11 @@ +Le conteneur et l'ancre + +Sur puce de silicium, la pomme s'égare, +le golden exige un coeur x86 ; +on épingle la plateforme, phare +dans la brume des architectures perplexes. + +Le gabarit d'ext3, trésor de Greene, +n'existe point ailleurs — qu'on le dise enfin : +un garde-fou s'éveille, la faute est moins mesquine, +et l'overlay se crée d'un geste apptainer, serein. diff --git a/README.md b/README.md index 5875800aab..547db0d789 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ The venv lives outside the overlay because fuse2fs is single-threaded (~10 MB/s) | `VENV_PATH` | `/scratch/$USER/venvs/pufferdrive` | | `CONTAINER_PYTHON` | `/ext3/miniforge3/bin/python3` | -The defaults match NYU Greene's filesystem layout. Override the env vars before invoking `setup_container.sh` if your cluster differs. +The defaults match NYU Greene's filesystem layout. Override the env vars before invoking `setup_container.sh` if your cluster differs. In particular, the `/share/apps` defaults (`IMAGE_PATH`, `OVERLAY_TEMPLATE`) exist only on NYU Greene. `OVERLAY_TEMPLATE` is just a gzipped empty ext3 image: on other clusters point it at your site's template, or skip `create-overlay` and create the overlay directly with `apptainer overlay create --size 15360 --create-dir /ext3 `. Set `IMAGE_PATH` to any CUDA-capable `.sif` (e.g. `apptainer pull docker://nvcr.io/nvidia/cuda:12.8.1-cudnn-devel-ubuntu24.04`). **One-time setup:** diff --git a/docs/cluster_training.md b/docs/cluster_training.md index 552f2df279..5672234797 100644 --- a/docs/cluster_training.md +++ b/docs/cluster_training.md @@ -8,6 +8,8 @@ How to run PufferDrive training on a SLURM cluster. This is written with the NYU # One-time per cluster: create the singularity overlay and install deps # into the venv (this also installs submitit and the other submission # deps as part of the project's pyproject.toml). +# Non-NYU clusters: export OVERLAY_TEMPLATE/IMAGE_PATH first, or create the +# overlay with `apptainer overlay create` (see the README defaults table). ./scripts/setup_container.sh create-overlay sbatch --account= --gres=gpu:1 --cpus-per-task=8 --mem=32gb --time=60 \ --wrap "./scripts/setup_container.sh install" diff --git a/scripts/setup_container.sh b/scripts/setup_container.sh index 7623039f2d..15b721e43d 100755 --- a/scripts/setup_container.sh +++ b/scripts/setup_container.sh @@ -52,6 +52,15 @@ create_overlay() { exit 1 fi + if [ ! -f "$OVERLAY_TEMPLATE" ]; then + echo "ERROR: overlay template not found at $OVERLAY_TEMPLATE" + echo "The default template path exists only on NYU Greene. Either:" + echo " - export OVERLAY_TEMPLATE to your cluster's gzipped blank-ext3 template, or" + echo " - create the overlay directly (singularity >= 3.8: 'singularity overlay create'):" + echo " apptainer overlay create --size 15360 --create-dir /ext3 \"$OVERLAY_PATH\"" + exit 1 + fi + echo "Copying and extracting overlay (this may take a few minutes)..." cp "$OVERLAY_TEMPLATE" "$CONTAINER_DIR/" TEMPLATE_NAME=$(basename "$OVERLAY_TEMPLATE") diff --git a/tests/smoke_tests/Dockerfile b/tests/smoke_tests/Dockerfile index 82278d3a97..1e84116f9a 100644 --- a/tests/smoke_tests/Dockerfile +++ b/tests/smoke_tests/Dockerfile @@ -14,7 +14,13 @@ # -v "$PWD/tests/smoke_tests/data:/app/tests/smoke_tests/data" \ # pufferdrive-smoke # then commit tests/smoke_tests/data/drive_smoke_golden.json. -FROM python:3.11-slim-bookworm +# +# This image is x86-64-only by definition: the golden is bit-exact AVX2 math +# run under qemu-x86_64-static, and setup.py ships only the amd64 raylib +# archive. On ARM hosts (e.g. Apple Silicon) Docker runs it under emulation — +# pass --platform linux/amd64 to docker build/run if the FROM pin below is +# not honored. +FROM --platform=linux/amd64 python:3.11-slim-bookworm ENV DEBIAN_FRONTEND=noninteractive \ PUFFER_CPU=1 \ From 691fdf330ce3b823a52f404ff89435212e9d0d1f Mon Sep 17 00:00:00 2001 From: Eugene Vinitsky Date: Fri, 10 Jul 2026 13:08:55 -0400 Subject: [PATCH 2/2] Remove AGENTIC_PR file Co-Authored-By: Claude Fable 5 --- AGENTIC_PR | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 AGENTIC_PR diff --git a/AGENTIC_PR b/AGENTIC_PR deleted file mode 100644 index 21089c2a2e..0000000000 --- a/AGENTIC_PR +++ /dev/null @@ -1,11 +0,0 @@ -Le conteneur et l'ancre - -Sur puce de silicium, la pomme s'égare, -le golden exige un coeur x86 ; -on épingle la plateforme, phare -dans la brume des architectures perplexes. - -Le gabarit d'ext3, trésor de Greene, -n'existe point ailleurs — qu'on le dise enfin : -un garde-fou s'éveille, la faute est moins mesquine, -et l'overlay se crée d'un geste apptainer, serein.