Merge main updates to monopod_wilks

.github/workflows/publish.yml

@@ -14,7 +14,7 @@ on:
 
 jobs:
   image-publish:
-    uses: WIPACrepo/wipac-dev-workflows/.github/workflows/image-publish.yml@v1.13
+    uses: WIPACrepo/wipac-dev-workflows/.github/workflows/image-publish.yml@v1.20
     permissions: # for GITHUB_TOKEN
       packages: write
     with:

Dockerfile

@@ -7,8 +7,15 @@ ARG ICETRAY_VERSION=v1.15.3-ubuntu22.04
 FROM icecube/icetray:icetray-devel-v1.15.3_pr4012-ubuntu22.04 AS prod
 
 # gcd files
-RUN mkdir -p /opt/i3-data/baseline_gcds && \
-    wget -nv -N -t 5 -P /opt/i3-data/baseline_gcds -r -l 1 -A *.i3* -nd http://prod-exe.icecube.wisc.edu/baseline_gcds/ && \
+# -- toggle GCD download at build time (1=download, 0=skip)
+ARG INCLUDE_GCD=1
+RUN set -eux; \
+    mkdir -p /opt/i3-data/baseline_gcds; \
+    if [ "${INCLUDE_GCD}" = "1" ]; then \
+        wget -nv -N -t 5 -P /opt/i3-data/baseline_gcds -r -l 1 -A '*.i3*' -nd http://prod-exe.icecube.wisc.edu/baseline_gcds/; \
+    else \
+        echo "Skipping baseline_gcds download (INCLUDE_GCD=${INCLUDE_GCD})"; \
+    fi; \
     chmod -R u+rwX,go+rX,go-w /opt/i3-data/baseline_gcds
 
 #

pyproject.toml

@@ -51,12 +51,18 @@ mypy = [
     'oms-mqclient[rabbitmq]',
 ] # do not edit — autogenerated by wipac-dev-py-setup-action
 
+[tool.setuptools]
+packages = [
+    'skymap_scanner',
+    'skymap_scanner.client',
+    'skymap_scanner.recos',
+    'skymap_scanner.recos.common',
+    'skymap_scanner.server',
+    'skymap_scanner.utils',
+] # do not edit — autogenerated by wipac-dev-py-setup-action
+
 [tool.setuptools.package-data]
 "*" = ["py.typed"] # do not edit — autogenerated by wipac-dev-py-setup-action
 
-[tool.setuptools.packages.find] # do not edit — autogenerated by wipac-dev-py-setup-action
-namespaces = false
-exclude = ["test", "tests", "doc", "docs", "resource", "resources", "example", "examples"]
-
 [tool.setuptools_scm] # do not edit — autogenerated by wipac-dev-py-setup-action
-fallback_version = "UNTAGGED" # only used when SCM metadata is completely unavailable (e.g., no '.git/'). NOTE: in shallow clones without git tags, setuptools-scm will still infer a commit-based version (like '0.1.dev1+g79c9b4b'); this value will NOT be used in that case.
+fallback_version = "CANNOT_BUILD_WITHOUT_GIT_DIR" # used when SCM metadata is missing (no '.git/') — if '.git/' exists but does not contain tags, a commit-based version (like '0.1.dev1+g79c9b4b') will be used instead.

resources/build_apptainer_image.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+set -euo pipefail
+echo "now: $(date -u +"%Y-%m-%dT%H:%M:%S.%3N")"
+
+########################################################################
+# Require environment variables
+########################################################################
+if [[ -z "${_SCANNER_IMAGE_DOCKER:-}" ]]; then
+    echo "::error:: _SCANNER_IMAGE_DOCKER must be set"
+    exit 1
+fi
+if [[ -z "${APPTAINER_CACHEDIR:-}" ]]; then
+    echo "::error:: APPTAINER_CACHEDIR must be set"
+    exit 1
+fi
+
+APPTAINER_SANDBOX_DIR="skymap_scanner.sandbox"
+
+########################################################################
+# Build sandbox image
+########################################################################
+echo "building apptainer sandbox at: ${APPTAINER_SANDBOX_DIR}"
+# ensure a clean target (apptainer refuses to overwrite without --force)
+rm -rf "${APPTAINER_SANDBOX_DIR}" || true
+apptainer build --sandbox --force "${APPTAINER_SANDBOX_DIR}" docker-daemon://"$_SCANNER_IMAGE_DOCKER"
+du -sh "${APPTAINER_SANDBOX_DIR}" || ls -lh "${APPTAINER_SANDBOX_DIR}"
+
+########################################################################
+# Drop apptainer caches
+########################################################################
+echo "clearing apptainer caches..."
+du -sh "$APPTAINER_CACHEDIR" || true
+rm -rf "$APPTAINER_CACHEDIR" || true
+
+########################################################################
+# Free docker stuff now that sandbox is built
+########################################################################
+echo "clearing docker things..."
+BEFORE="$(df -B1 --output=avail / | tail -1)"
+
+# -- docker layers
+docker ps -a --filter "ancestor=$_SCANNER_IMAGE_DOCKER" -q | xargs -r docker rm -f
+docker rmi -f "$_SCANNER_IMAGE_DOCKER" || true
+
+# -- prune buildkit + volume
+docker ps -aq --filter "label=name=buildx_buildkit" | xargs -r docker rm -f || true
+docker ps -aq --filter "ancestor=moby/buildkit:buildx-stable-1" | xargs -r docker rm -f || true
+docker buildx ls | awk 'NR>1{gsub(/\*$/,"",$1); if($1!="default" && $1!="") print $1}' | xargs -r -n1 docker buildx rm -f || true
+docker builder prune -af || true
+docker system prune -af --volumes || true
+docker volume ls -q --filter 'name=buildx_buildkit_.*_state' | xargs -r docker volume rm -f || true
+
+# -- report
+AFTER="$(df -B1 --output=avail / | tail -1)"
+DELTA="$((AFTER - BEFORE))"
+GIB="$(awk -v b="$DELTA" 'BEGIN{printf "%.2f", b/1024/1024/1024}')"
+MIB="$(awk -v b="$DELTA" 'BEGIN{printf "%.0f", b/1024/1024}')"
+echo "Freed: ${GIB} GiB (${MIB} MiB)"

resources/launch_scripts/launch_worker.sh

@@ -1,47 +1,46 @@
 #!/bin/bash
 set -euo pipefail
-set -ex
 
 ########################################################################
 #
 # Launch a Skymap Scanner worker
+#   Uses '_CI_SCANNER_CONTAINER_PLATFORM' to control the mode:
 #
-# Run worker on ewms pilot
+# - Docker mode:
+#       runs the pilot image (docker) that runs Docker scanner clients
+# - Apptainer mode:
+#       runs the pilot image (docker) that runs Apptainer scanner clients
 #
 ########################################################################
 
+########################################################################
+# Common (platform-agnostic) setup
+########################################################################
+
 # establish pilot's root path
 tmp_rootdir="$(pwd)/pilot-$(uuidgen)"
-mkdir $tmp_rootdir
-cd $tmp_rootdir
+mkdir "$tmp_rootdir"
+cd "$tmp_rootdir"
 export EWMS_PILOT_DATA_DIR_PARENT_PATH_ON_HOST="$tmp_rootdir"
 
 # mark startup.json's dir to be bind-mounted into the task container (by the pilot)
 # -> check that the dir only has one file, otherwise we may end up binding extra dirs
 python -c 'import os; assert os.listdir(os.path.dirname(os.environ["CI_SKYSCAN_STARTUP_JSON"])) == ["startup.json"]'
 export EWMS_PILOT_EXTERNAL_DIRECTORIES="$(dirname "$CI_SKYSCAN_STARTUP_JSON")"
 
-# task image, args, env
-if [ -n "${_RUN_THIS_SIF_IMAGE:-}" ]; then
-    # place a duplicate of the file b/c the pilot transforms this into another format, so there are issues w/ parallelizing
-    export EWMS_PILOT_TASK_IMAGE="$tmp_rootdir/$(basename "$_RUN_THIS_SIF_IMAGE")"
-    cp "$_RUN_THIS_SIF_IMAGE" "$EWMS_PILOT_TASK_IMAGE"
-    export _EWMS_PILOT_CONTAINER_PLATFORM="apptainer"
-else
-    export EWMS_PILOT_TASK_IMAGE="$CI_DOCKER_IMAGE_TAG"
-    export _EWMS_PILOT_CONTAINER_PLATFORM="docker" # NOTE: technically not needed b/c this is the default value
-    export _EWMS_PILOT_DOCKER_SHM_SIZE="6gb"       # this only needed in ci--the infra would set this in prod
-fi
+# args
 export EWMS_PILOT_TASK_ARGS="python -m skymap_scanner.client --infile {{INFILE}} --outfile {{OUTFILE}} --client-startup-json $CI_SKYSCAN_STARTUP_JSON"
-json_var=$(env | grep -E '^(SKYSCAN_|_SKYSCAN_)' | awk -F= '{printf "\"%s\":\"%s\",", $1, $2}' | sed 's/,$//') # must remove last comma
+
+# marshal SKYSCAN/_SKYSCAN_ env into JSON
+json_var=$(env | grep -E '^(SKYSCAN_|_SKYSCAN_)' | awk -F= '{printf "\"%s\":\"%s\",", $1, $2}' | sed 's/,$//')
 json_var="{$json_var}"
 export EWMS_PILOT_TASK_ENV_JSON="$json_var"
 
-# file types -- controls intermittent serialization
+# file types
 export EWMS_PILOT_INFILE_EXT="JSON"
 export EWMS_PILOT_OUTFILE_EXT="JSON"
 
-# Load JSON values
+# queues/broker config
 export EWMS_PILOT_QUEUE_INCOMING=$(jq -r '.toclient.name' "$_EWMS_JSON_ON_HOST")
 export EWMS_PILOT_QUEUE_INCOMING_AUTH_TOKEN=$(jq -r '.toclient.auth_token' "$_EWMS_JSON_ON_HOST")
 export EWMS_PILOT_QUEUE_INCOMING_BROKER_TYPE=$(jq -r '.toclient.broker_type' "$_EWMS_JSON_ON_HOST")
@@ -53,11 +52,62 @@ export EWMS_PILOT_QUEUE_OUTGOING_BROKER_TYPE=$(jq -r '.fromclient.broker_type' "
 export EWMS_PILOT_QUEUE_OUTGOING_BROKER_ADDRESS=$(jq -r '.fromclient.broker_address' "$_EWMS_JSON_ON_HOST")
 
 
-# run!
-ENV="$(dirname $tmp_rootdir)/pyenv-$(basename $tmp_rootdir)"
-pip install virtualenv
-virtualenv --python python3 "$ENV"
-. "$ENV"/bin/activate
-pip install --upgrade pip
-pip install ewms-pilot[rabbitmq]
-python -m ewms_pilot
+########################################################################
+# Run!
+########################################################################
+
+# ─────────────── Case: Docker ───────────────
+if [[ "${_CI_SCANNER_CONTAINER_PLATFORM}" == "docker" ]]; then
+
+    # docker-specific pilot env vars
+    export EWMS_PILOT_TASK_IMAGE="$_SCANNER_IMAGE_DOCKER"
+    export _EWMS_PILOT_DOCKER_SHM_SIZE="6gb"  # CI-specific; prod infra should set this
+
+    # Required env for the helper
+    export DOOD_OUTER_IMAGE="$_PILOT_IMAGE_FOR_DOCKER_SCANNER_CLIENT"
+    # Network for the outer container
+    if [[ -z "${DOOD_NETWORK:-}" ]]; then
+        echo "::error::DOOD_NETWORK must be set — this should've been set in '.github/workflows/tests.yml'. Did it not get forwarded to this script?"
+        exit 1
+    fi
+    # Bind dirs: the pilot needs these paths visible at the same locations
+    # - tmp_rootdir (RW)
+    # - startup.json's parent (RO)
+    export DOOD_BIND_RW_DIRS="$tmp_rootdir"
+    export DOOD_BIND_RO_DIRS="$(dirname "$CI_SKYSCAN_STARTUP_JSON")"
+    # Forward envs by prefix and explicit list
+    export DOOD_FORWARD_ENV_PREFIXES="EWMS_ _EWMS_ SKYSCAN_ _SKYSCAN_"
+    export DOOD_FORWARD_ENV_VARS="CI_SKYSCAN_STARTUP_JSON"
+
+    # run (curl script first)
+    tmp_for_dood_sh=$(mktemp -d)
+    curl -fsSL "$CI_SCRIPT_URL_DOOD_RUN" -o "$tmp_for_dood_sh/run-dood.sh"
+    chmod +x "$tmp_for_dood_sh/run-dood.sh"
+    "$tmp_for_dood_sh/run-dood.sh"
+
+# ─────────────── Case: Apptainer ───────────────
+elif [[ "${_CI_SCANNER_CONTAINER_PLATFORM}" == "apptainer" ]]; then
+
+    # apptainer-specific pilot env vars
+    export EWMS_PILOT_TASK_IMAGE="$_SCANNER_IMAGE_APPTAINER"
+    export _EWMS_PILOT_APPTAINER_IMAGE_DIRECTORY_MUST_BE_PRESENT=True
+
+    # run
+    docker run --rm \
+        --privileged \
+        --network=host \
+        \
+        -v "$tmp_rootdir:$tmp_rootdir" \
+        -v "$(dirname "$CI_SKYSCAN_STARTUP_JSON"):$(dirname "$CI_SKYSCAN_STARTUP_JSON")":ro \
+        -v "$_SCANNER_IMAGE_APPTAINER:$_SCANNER_IMAGE_APPTAINER":ro \
+        \
+        --env CI_SKYSCAN_STARTUP_JSON="$CI_SKYSCAN_STARTUP_JSON" \
+        $(env | grep -E '^(EWMS_|_EWMS_)' | cut -d'=' -f1 | sed 's/^/--env /') \
+        \
+        "$_PILOT_IMAGE_FOR_APPTAINER_SCANNER_CLIENT"
+
+# ─────────────── Case: Unknown??? ───────────────
+else
+    echo "::error::cannot launch worker — unknown '_CI_SCANNER_CONTAINER_PLATFORM=$_CI_SCANNER_CONTAINER_PLATFORM'"
+    exit 2
+fi