feat(ci): run the GARM end-to-end test on ProdStack (ISD-5876)

.github/workflows/charms_integration.yaml

@@ -16,6 +16,10 @@ jobs:
       actions: read
     with:
       working-directory: .
+      # Restrict the matrix to the integration backend. The filter defaults to empty,
+      # which would include every backend in spread.yaml — and so run the ProdStack
+      # e2e suite on every pull request.
+      spread-jobs-include: 'integration-test-ci:*'
       test-secret-1-name: TEST_GITHUB_APP_ID
       test-secret-2-name: TEST_GITHUB_APP_INSTALLATION_ID
       test-secret-3-name: TEST_GITHUB_APP_PRIVATE_KEY

.github/workflows/garm_e2e.yaml

@@ -10,15 +10,57 @@ name: GARM E2E
 
 on:
   workflow_dispatch:
+  # Temporary: runs this workflow from its own development branch, so it can be
+  # exercised before it exists on the default branch and can be dispatched.
+  # Removed again once it has served its purpose. A label gate would be tidier but
+  # the repository's labels are managed, and `run-e2e` was deleted from the repo.
+  pull_request:
+    types: [opened, synchronize]
 
 permissions:
   contents: read
 
+concurrency:
+  group: garm-e2e-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
+  # Builds the rocks and charms the suite deploys, and publishes artifacts.build.yaml
+  # for opcli to fetch during the spread prepare.
+  build:
+    if: >-
+      github.event_name == 'workflow_dispatch'
+      || github.head_ref == 'feat/garm-e2e-implementation-ISD-5876'
+    uses: canonical/charm-ci/.github/workflows/build-artifacts.yml@v0.0.1-alpha.10
+    permissions:
+      contents: read
+      packages: write
+      actions: read
+    with:
+      working-directory: .
+
   e2e:
     name: GARM E2E test
+    # Waits for the build rather than provisioning alongside it: the private-endpoint
+    # runner is scarce, and starting early would only hold it idle until the artifacts
+    # the spread prepare fetches actually exist.
+    needs: [build]
+    # `pull_request`, never `pull_request_target`: a fork's code must not run with
+    # access to these secrets. A fork PR gets none, so it can only fail fast.
+    if: >-
+      github.event_name == 'workflow_dispatch'
+      || github.head_ref == 'feat/garm-e2e-implementation-ISD-5876'
+    permissions:
+      contents: read
+      packages: read
+      # opcli fetches the build job's artifacts from this workflow run.
+      actions: read
     runs-on: self-hosted-linux-amd64-noble-private-endpoint-medium
-    timeout-minutes: 15
+    # Longer than the suite's own kill-timeout in spread.yaml, so spread stops the run
+    # first and says which task hung, rather than the job being cut from underneath it.
+    # Generous on purpose: it is not worth tuning before a real run says how long a
+    # deploy, a spawn and a dispatch actually take here.
+    timeout-minutes: 130
     steps:
       - uses: actions/checkout@v7.0.1
 
@@ -52,13 +94,16 @@ jobs:
           E2E_GITHUB_APP_ID: ${{ secrets.E2E_GITHUB_APP_ID }}
           E2E_GITHUB_APP_INSTALLATION_ID: ${{ secrets.E2E_GITHUB_APP_INSTALLATION_ID }}
           E2E_GITHUB_APP_PRIVATE_KEY: ${{ secrets.E2E_GITHUB_APP_PRIVATE_KEY }}
+          E2E_RUNNER_IMAGE_NAME: ${{ secrets.E2E_RUNNER_IMAGE_NAME }}
+          E2E_OPENSTACK_FLAVOR: ${{ secrets.E2E_OPENSTACK_FLAVOR }}
+          E2E_RUNNER_HTTP_PROXY: ${{ secrets.E2E_RUNNER_HTTP_PROXY }}
         run: |
           set -euo pipefail
 
           for KEY in OS_AUTH_URL OS_PROJECT_NAME OS_USER_DOMAIN_NAME \
                      OS_PROJECT_DOMAIN_NAME OS_REGION_NAME OS_NETWORK \
                      E2E_GITHUB_APP_ID E2E_GITHUB_APP_INSTALLATION_ID \
-                     E2E_GITHUB_APP_PRIVATE_KEY; do
+                     E2E_GITHUB_APP_PRIVATE_KEY E2E_RUNNER_IMAGE_NAME; do
             if [ -z "${!KEY:-}" ]; then
               echo "::error::Missing repository secret: $KEY (Settings > Secrets > Actions)."
               exit 1
@@ -68,10 +113,20 @@ jobs:
           # Everything but the key is single-line already.
           for KEY in OS_AUTH_URL OS_PROJECT_NAME OS_USER_DOMAIN_NAME \
                      OS_PROJECT_DOMAIN_NAME OS_REGION_NAME OS_NETWORK \
-                     E2E_GITHUB_APP_ID E2E_GITHUB_APP_INSTALLATION_ID; do
+                     E2E_GITHUB_APP_ID E2E_GITHUB_APP_INSTALLATION_ID \
+                     E2E_RUNNER_IMAGE_NAME; do
             echo "${KEY}=${!KEY}" >> "$GITHUB_ENV"
           done
 
+          # Written only when set: the fixtures fall back to their own defaults for
+          # these two, and an exported empty string is not absent -- it would win
+          # over the default rather than yield to it.
+          for KEY in E2E_OPENSTACK_FLAVOR E2E_RUNNER_HTTP_PROXY; do
+            if [ -n "${!KEY:-}" ]; then
+              echo "${KEY}=${!KEY}" >> "$GITHUB_ENV"
+            fi
+          done
+
           # The key travels through two channels that take one KEY=value per line —
           # this file, and later opcli's pytest-environment-template, which rejects a
           # line without an '='. Normalising here rather than requiring a pre-encoded
@@ -202,12 +257,72 @@ jobs:
           set -euo pipefail
 
           uv tool install python-openstackclient
+          # uv installs into ~/.local/bin, which later steps do not inherit; the
+          # cleanup step needs the client too.
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
 
           TOKEN_EXPIRES=$(openstack token issue -f value -c expires)
           echo "OpenStack token issued (expires: ${TOKEN_EXPIRES}). Tenant reachable."
 
+      - name: Install opcli and spread
+        run: |
+          set -euo pipefail
+          uv tool install "opcli[cli] @ git+https://github.com/canonical/charm-ci.git@main"
+          sudo snap install go --classic
+          go install github.com/canonical/spread/cmd/spread@latest
+          sudo ln -sf ~/go/bin/spread /usr/local/bin/spread
+
+      - name: Point concierge at a MetalLB range on this host
+        run: |
+          set -euo pipefail
+
+          # traefik-k8s asks for a LoadBalancer service, which microk8s cannot satisfy
+          # unaided. Pinning the pool to this host's own address is what makes GARM's
+          # callback and metadata URLs resolve to something a runner VM on the tenant
+          # can actually reach -- an in-cluster service address cannot serve them.
+          HOST_IP=$(ip route get 1.1.1.1 | grep -oP 'src \K\S+')
+          echo "MetalLB pool pinned to ${HOST_IP}"
+          sed "s/@HOST_IP@/${HOST_IP}/g" concierge-e2e.yaml.tmpl > concierge-e2e.yaml
+          echo "CONCIERGE=concierge-e2e.yaml" >> "$GITHUB_ENV"
+
       - name: Run the E2E suite
+        env:
+          CI: "true"
+          # Spread reads this from the host environment and the prepare script uses it
+          # to download the build job's artifacts. Actions does not export it on its own.
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           set -euo pipefail
-          uv tool install tox --with tox-uv
-          tox -e garm-e2e
+
+          # Ask opcli for the selector rather than writing it out here, so a rename in
+          # spread.yaml cannot leave this step silently selecting nothing.
+          SELECTOR=$(opcli spread jobs --include 'e2e-test-ci:*' \
+            | python3 -c 'import json,sys; print(json.load(sys.stdin)["include"][0]["selector"])')
+          echo "Running ${SELECTOR}"
+          opcli spread run -- -vv "${SELECTOR}"
+
+      # Deliberately no juju debug-log or status dump here: both replay charm output
+      # that carries the rendered GARM config, and so the tenant password. The suite
+      # collects its own diagnostics through the sentinel redactor instead, and a
+      # re-run with debug logging drops into the tmate session above.
+      - name: Delete runners left behind on the tenant
+        if: always()
+        run: |
+          set -euo pipefail
+
+          # GARM stamps every server it creates with garm-controller-id. Matching that
+          # alongside the runner image scopes the sweep to VMs this suite's GARM
+          # created, so a failure that skips the suite's own teardown cannot leave them
+          # running and billing.
+          ORPHANS=$(openstack server list --long --image "$E2E_RUNNER_IMAGE_NAME" -f json \
+            | python3 -c "import json,sys; print('\n'.join(s['ID'] for s in json.load(sys.stdin) if 'garm-controller-id' in (s.get('Properties') or {})))")
+
+          if [ -z "$ORPHANS" ]; then
+            echo "No runners left on the tenant."
+            exit 0
+          fi
+
+          echo "$ORPHANS" | while read -r ID; do
+            echo "Deleting orphaned runner ${ID}"
+            openstack server delete --wait "$ID" || echo "::warning::Could not delete ${ID}"
+          done