From 73dae0eac71716fffb5bd3a50fb9400e55ae0b21 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Mon, 17 Aug 2026 13:57:23 +0000 Subject: [PATCH 01/21] feat(ci): add the GARM E2E workflow and credential path (ISD-5876) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Land the two workflow files first, on their own, because workflow_dispatch does not register until a workflow exists on the default branch — so neither the E2E driver nor the runner-side job it dispatches can be exercised from a feature branch before they are on main. garm_e2e.yaml resolves everything the end-to-end test needs to authenticate: the OpenStack username and password from Vault via AppRole, the rest of the tenant settings and a dedicated GitHub App from repository secrets. It runs on the private-endpoint runner, is manually triggered, and is not a merge gate. The E2E's GitHub App is deliberately a different one from the integration suite's: it registers and tears down a runner scale set, so it needs organization-level runner permissions the integration App has no reason to hold. garm_e2e_test_run.yaml lands with its final workflow_dispatch inputs, since that contract is the one part not testable from a branch afterwards. The test module itself is a stub asserting only that the credentials reach pytest without any of them appearing in the output. The deployment and the end-to-end assertions follow. --- .github/workflows/garm_e2e.yaml | 163 +++++++++++++++++++++++ .github/workflows/garm_e2e_test_run.yaml | 37 +++++ AGENTS.md | 5 +- CONTRIBUTING.md | 72 ++++++++++ charms/tests/e2e/test_garm_e2e.py | 56 ++++++++ tox.ini | 23 ++++ 6 files changed, 354 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/garm_e2e.yaml create mode 100644 .github/workflows/garm_e2e_test_run.yaml create mode 100644 charms/tests/e2e/test_garm_e2e.py diff --git a/.github/workflows/garm_e2e.yaml b/.github/workflows/garm_e2e.yaml new file mode 100644 index 00000000..13726afb --- /dev/null +++ b/.github/workflows/garm_e2e.yaml @@ -0,0 +1,163 @@ +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. +# +# GARM end-to-end test workflow. +# +# Manual trigger only, and deliberately not a merge gate. Dispatch it with +# --ref to run a feature branch's version. + +name: GARM E2E + +on: + workflow_dispatch: + +jobs: + e2e: + name: GARM E2E test + runs-on: self-hosted-linux-amd64-noble-private-endpoint-medium + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7.0.1 + + - name: Set up tmate session (self-hosted) + if: runner.debug == 1 && runner.environment == 'self-hosted' + uses: canonical/action-tmate@main + with: + detached: true + timeout-minutes: 60 + + # These are all secrets, so the runner masks them in the log without an + # explicit ::add-mask::. The E2E_GITHUB_APP_* trio is a GitHub App of its + # own, separate from the TEST_GITHUB_APP_* one the integration suite uses: + # this one has to manage runner scale sets, which that one must not. + - name: Load tenant and GitHub App configuration + env: + OS_AUTH_URL: ${{ secrets.OS_AUTH_URL }} + OS_PROJECT_NAME: ${{ secrets.OS_PROJECT_NAME }} + OS_USER_DOMAIN_NAME: ${{ secrets.OS_USER_DOMAIN_NAME }} + OS_PROJECT_DOMAIN_NAME: ${{ secrets.OS_PROJECT_DOMAIN_NAME }} + OS_REGION_NAME: ${{ secrets.OS_REGION_NAME }} + OS_NETWORK: ${{ secrets.OS_NETWORK }} + E2E_GITHUB_APP_ID: ${{ secrets.E2E_GITHUB_APP_ID }} + E2E_GITHUB_APP_INSTALLATION_ID: ${{ secrets.E2E_GITHUB_APP_INSTALLATION_ID }} + E2E_GITHUB_APP_PRIVATE_KEY: ${{ secrets.E2E_GITHUB_APP_PRIVATE_KEY }} + run: | + set -euo pipefail + + for KEY in OS_AUTH_URL OS_PROJECT_NAME OS_USER_DOMAIN_NAME \ + OS_PROJECT_DOMAIN_NAME OS_REGION_NAME OS_NETWORK \ + E2E_GITHUB_APP_ID E2E_GITHUB_APP_INSTALLATION_ID \ + E2E_GITHUB_APP_PRIVATE_KEY; do + if [ -z "${!KEY:-}" ]; then + echo "::error::Missing repository secret: $KEY (Settings > Secrets > Actions)." + exit 1 + fi + done + + # Everything but the key is single-line already. + for KEY in OS_AUTH_URL OS_PROJECT_NAME OS_USER_DOMAIN_NAME \ + OS_PROJECT_DOMAIN_NAME OS_REGION_NAME OS_NETWORK \ + E2E_GITHUB_APP_ID E2E_GITHUB_APP_INSTALLATION_ID; do + echo "${KEY}=${!KEY}" >> "$GITHUB_ENV" + done + + # The key travels through two channels that take one KEY=value per line — + # this file, and later opcli's pytest-environment-template, which rejects a + # line without an '='. Normalising here rather than requiring a pre-encoded + # secret keeps the encoding out of everyone's setup instructions. An + # already-encoded value passes through, so both forms work. + case "$E2E_GITHUB_APP_PRIVATE_KEY" in + *-----BEGIN*) + KEY_B64=$(printf '%s' "$E2E_GITHUB_APP_PRIVATE_KEY" | base64 -w0) + ;; + *) + KEY_B64=$E2E_GITHUB_APP_PRIVATE_KEY + ;; + esac + # The encoded form is a different string from the secret, so the runner does + # not mask it on its own. + echo "::add-mask::$KEY_B64" + echo "E2E_GITHUB_APP_PRIVATE_KEY=$KEY_B64" >> "$GITHUB_ENV" + + # openstackclient needs this stated explicitly for a v3 auth URL. + echo "OS_IDENTITY_API_VERSION=3" >> "$GITHUB_ENV" + + - name: Fetch tenant credentials from Vault + env: + VAULT_ADDR: ${{ secrets.VAULT_ADDR }} + VAULT_APPROLE_ROLE_ID: ${{ secrets.VAULT_APPROLE_ROLE_ID }} + VAULT_APPROLE_SECRET_ID: ${{ secrets.VAULT_APPROLE_SECRET_ID }} + VAULT_KV_PATH: ${{ secrets.E2E_VAULT_KV_PATH || 'kv/data/garm-e2e/prodstack' }} + run: | + set -euo pipefail + + for KEY in VAULT_ADDR VAULT_APPROLE_ROLE_ID VAULT_APPROLE_SECRET_ID; do + if [ -z "${!KEY:-}" ]; then + echo "::error::Missing repository secret: $KEY (Settings > Secrets > Actions)." + exit 1 + fi + done + + # The response body holds the tenant credentials, so remove it however this + # step exits — set -e would otherwise skip the cleanup on every failure path, + # on a runner whose disk outlives the job. + RESPONSE=$(mktemp) + trap 'rm -f "$RESPONSE"' EXIT + + # curl --fail suppresses the response body, which would reduce a 403 to a + # bare exit code on the step whose job is to say what is misconfigured. + vault_call() { + curl -sS -o "$RESPONSE" -w '%{http_code}' "$@" + } + + STATUS=$(vault_call -X POST -H "Content-Type: application/json" \ + -d "$(jq -n --arg r "$VAULT_APPROLE_ROLE_ID" --arg s "$VAULT_APPROLE_SECRET_ID" \ + '{role_id: $r, secret_id: $s}')" \ + "${VAULT_ADDR}/v1/auth/approle/login") + if [ "$STATUS" != "200" ]; then + echo "::error::Vault AppRole login failed with HTTP $STATUS: $(jq -rc '.errors // empty' "$RESPONSE")" + exit 1 + fi + + VAULT_TOKEN=$(jq -r '.auth.client_token' "$RESPONSE") + echo "::add-mask::$VAULT_TOKEN" + if [ -z "$VAULT_TOKEN" ] || [ "$VAULT_TOKEN" = "null" ]; then + echo "::error::Vault AppRole login returned no client token." + exit 1 + fi + + STATUS=$(vault_call -X GET -H "X-Vault-Token: $VAULT_TOKEN" \ + "${VAULT_ADDR}/v1/${VAULT_KV_PATH}") + if [ "$STATUS" != "200" ]; then + echo "::error::Vault read of ${VAULT_KV_PATH} failed with HTTP $STATUS: $(jq -rc '.errors // empty' "$RESPONSE")" + exit 1 + fi + + for KEY in OS_USERNAME OS_PASSWORD; do + VALUE=$(jq -r --arg k "$KEY" '.data.data[$k] // empty' "$RESPONSE") + echo "::add-mask::$VALUE" + if [ -z "$VALUE" ]; then + echo "::error::Field '$KEY' missing from the Vault secret at ${VAULT_KV_PATH}." + exit 1 + fi + echo "${KEY}=${VALUE}" >> "$GITHUB_ENV" + done + + echo "Tenant credentials fetched and masked." + + - name: Assert the credentials authenticate against the tenant + run: | + set -euo pipefail + + # Ubuntu 24.04 enforces PEP 668, so a bare pip install into the system + # interpreter fails with externally-managed-environment. + pipx install python-openstackclient + + TOKEN_EXPIRES=$(openstack token issue -f value -c expires) + echo "OpenStack token issued (expires: ${TOKEN_EXPIRES}). Tenant reachable." + + - name: Run the E2E suite + run: | + set -euo pipefail + pipx install tox + tox -e garm-e2e diff --git a/.github/workflows/garm_e2e_test_run.yaml b/.github/workflows/garm_e2e_test_run.yaml new file mode 100644 index 00000000..936b2940 --- /dev/null +++ b/.github/workflows/garm_e2e_test_run.yaml @@ -0,0 +1,37 @@ +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. +# +# Trivial workflow that a spawned runner executes to prove the full chain: +# GARM → OpenStack → VM → runner register → job dispatch → pick up → exit. +# Modelled on github-runner-operator/.github/workflows/e2e_test_run.yaml. + +name: GARM E2E test run + +on: + workflow_dispatch: + inputs: + runner-label: + description: "Self-hosted runner label to target" + required: true + type: string + +jobs: + e2e-test: + name: GARM E2E test + # A single combined label, matching this repo's runner naming + # (e.g. self-hosted-linux-amd64-noble-medium) and how GitHub routes a job to a + # GARM scale set — by its name as one label. + runs-on: "${{ inputs.runner-label }}" + timeout-minutes: 10 + steps: + - name: Assert basic runner liveness + run: | + echo "=== GARM E2E test run ===" + echo "Runner: $(hostname)" + echo "Kernel: $(uname -a)" + - name: Assert egress through aproxy + run: | + echo "Checking egress connectivity..." + curl -sf --max-time 10 https://github.com > /dev/null 2>&1 && \ + echo "Egress OK (via aproxy)" || \ + { echo "Egress FAILED"; exit 1; } diff --git a/AGENTS.md b/AGENTS.md index 7a505949..b191d527 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,7 +8,7 @@ guidance in `.github/instructions/` and the human-facing `CONTRIBUTING.md`. | Path | Contents | | --- | --- | -| `charms/` | Four Juju charms (see below) plus shared integration tests in `charms/tests/integration/`. | +| `charms/` | Four Juju charms (see below) plus shared integration tests in `charms/tests/integration/` and the GARM end-to-end test in `charms/tests/e2e/`. | | `cmd/` | Go application entry points: `planner`, `webhook-gateway`. | | `internal/` | Shared Go packages (`database`, `github`, `planner`, `queue`, `server`, `telemetry`, `webhook`, …) — the application logic the paas charms package and deploy. | | `*-rockcraft.yaml`, `build-*-rock.sh` (repo root) | Rock/image build definitions and their build scripts. | @@ -33,6 +33,7 @@ charms — there are four charms, not two). - **Per-charm Python checks** — from the charm directory, `tox -c tox.toml` (envs `fmt`, `lint`, `complexity`, `static`, `unit`, `coverage-report`; ruff, codespell, pyright, pytest+coverage). CI runs these per charm via `tox -c tox.toml`. - **Integration tests** (root `tox.ini`) — `tox -e -integration` (`garm`, `webhook-gateway`, `planner`, `garm-configurator`) or `tox -e charms-integration` for all. Requires a live Juju model (jubilant + pytest-operator). +- **GARM end-to-end test** (root `tox.ini`) — `tox -e garm-e2e`. Runs against a real OpenStack tenant from `garm_e2e.yaml`, is manually dispatched, and is **not** a merge gate. See `CONTRIBUTING.md` §"GARM E2E". - **`actions/` Python** — `tox -e actions-lint`, `tox -e actions-static`, `tox -e actions-unit`. - **Go** — `go test ./...`. - `charmcraft pack` — build a charm (run from the charm dir; not wired into tox). @@ -88,7 +89,7 @@ For **`garm-configurator`** (plain `ops`): - **DO** fix a missing AAA docstring on any test you move or edit. It's followed unevenly (`planner-operator` and `garm` yes; `garm-configurator` and `webhook-gateway-operator` not yet), so imitating the nearest neighbour is not a reliable guide. -- Integration tests live in the shared `charms/tests/integration/`. +- Integration tests live in the shared `charms/tests/integration/`; the GARM end-to-end test lives in `charms/tests/e2e/`, kept separate so it stays out of the PR test matrix. ## 12-factor divergences from the canonical charm-engineer guidance diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9dc34f6b..6f0d78bb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -231,3 +231,75 @@ rockcraft.skopeo copy \ oci-archive:webhook-gateway_0.1_amd64.rock \ docker://localhost:32000/webhook-gateway:0.1 ``` + +### GARM E2E + +The GARM end-to-end test (`charms/tests/e2e/`) exercises the full chain on ProdStack: +the charm starts, the configurator delivers config, the GARM API becomes reachable, the +provider authenticates to OpenStack, a VM is created from the runner image, the runner +registers with GitHub, and a dispatched job runs and exits clean. + +> **TODO:** only the workflows and the credential path exist so far. The test currently +> asserts nothing beyond the credentials reaching pytest; the deployment and the +> end-to-end assertions land in a follow-up. + +It is triggered manually and is **not** a merge gate: + +```shell +gh workflow run garm_e2e.yaml --ref +``` + +`--ref` selects which branch's version of both the workflow and the test code runs, so +changes to the end-to-end test can be exercised without merging them first. + +#### Required secrets and variables + +Infrastructure details are secrets, not variables — endpoints, project and network names +included. The runner masks secret values in the log automatically, so the workflow does +not register masks for these itself. + +| Name | Description | +| --- | --- | +| `VAULT_ADDR` | Vault server address | +| `VAULT_APPROLE_ROLE_ID` | Vault AppRole role ID | +| `VAULT_APPROLE_SECRET_ID` | Vault AppRole secret ID | +| `OS_AUTH_URL` | Keystone endpoint, e.g. `https://keystone.example.com:5000/v3` | +| `OS_PROJECT_NAME` | OpenStack project/tenant name | +| `OS_USER_DOMAIN_NAME` | OpenStack user domain name | +| `OS_PROJECT_DOMAIN_NAME` | OpenStack project domain name | +| `OS_REGION_NAME` | OpenStack region name | +| `OS_NETWORK` | OpenStack network for runner VMs | +| `E2E_GITHUB_APP_ID` | GitHub App ID | +| `E2E_GITHUB_APP_INSTALLATION_ID` | Installation ID of that App on this repository | +| `E2E_GITHUB_APP_PRIVATE_KEY` | That App's private key (PEM), base64-encoded | +| `E2E_VAULT_KV_PATH` | Optional. Defaults to `kv/data/garm-e2e/prodstack` | + +The OpenStack username and password are **not** repository secrets. They are read at run +time from the Vault KV v2 secret above, which must hold `OS_USERNAME` and `OS_PASSWORD`. + +The `E2E_GITHUB_APP_*` trio is a GitHub App of its own, distinct from the +`TEST_GITHUB_APP_*` one the integration suite uses. The end-to-end test registers and +tears down a runner scale set on this repository, so its App needs `Administration: +read & write` here, which the integration App has no reason to hold. + +Paste the private key into `E2E_GITHUB_APP_PRIVATE_KEY` exactly as GitHub issues it — +the PEM, newlines and all. The workflow reduces it to a single line before it enters the +two `KEY=value` channels that carry it to pytest, neither of which can hold a multi-line +value. A key that is already base64-encoded is accepted unchanged, so +`TEST_GITHUB_APP_PRIVATE_KEY`'s existing encoded form stays valid. + +#### Credential hygiene + +The test authenticates against a production tenant, so the run logs are part of the +contract: + +- Values read from Vault are `::add-mask::`ed in the same step that reads them, before any + later step runs. Masking registered with the runner scrubs every subsequent log line, + including output from code we do not control; a secret read inside the test process gets + none of that. Repository secrets are masked by the runner already. +- They reach later steps through `$GITHUB_ENV` and pytest through tox's `pass_env` — never + through command line arguments, which would expose them in `ps` and in pytest's header. +- Steps handling credentials use `set -euo pipefail` and never `set -x`. +- Assertions report the *name* of a missing setting, never its value, and use + `pytest.fail` rather than `assert` so that assertion rewriting cannot introspect + `os.environ` into the failure output. diff --git a/charms/tests/e2e/test_garm_e2e.py b/charms/tests/e2e/test_garm_e2e.py new file mode 100644 index 00000000..0cd545df --- /dev/null +++ b/charms/tests/e2e/test_garm_e2e.py @@ -0,0 +1,56 @@ +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. + +"""GARM end-to-end test. + +TODO: This module is deliberately incomplete. It exists so that the workflows have +something to run; the end-to-end implementation lands in a follow-up. What is here +asserts only that the credentials reach pytest. + +Still to come: deploy postgresql, GARM, a traefik ingress and garm-configurator; assert +the GARM API is reachable and the provider authenticates to OpenStack; assert a VM is +created from the runner image and the runner registers with GitHub; then dispatch +``garm_e2e_test_run.yaml`` at the scale set's label and assert the job is picked up and +exits clean. +""" + +import os + +import pytest + +# Settings the deployment fixtures will need. The OpenStack username and password come +# from Vault, the rest from repository secrets; both routes converge on the job +# environment, which tox forwards to pytest. +REQUIRED_SETTINGS = ( + "OS_AUTH_URL", + "OS_USERNAME", + "OS_PASSWORD", + "OS_PROJECT_NAME", + "OS_USER_DOMAIN_NAME", + "OS_PROJECT_DOMAIN_NAME", + "OS_REGION_NAME", + "OS_NETWORK", + "E2E_GITHUB_APP_ID", + "E2E_GITHUB_APP_INSTALLATION_ID", + "E2E_GITHUB_APP_PRIVATE_KEY", +) + + +@pytest.mark.parametrize("setting", REQUIRED_SETTINGS) +def test_setting_reaches_pytest(setting: str): + """ + arrange: The workflow has resolved the settings, taking the OpenStack username and + password from Vault and the rest from repository secrets, and exported them. + act: Read the setting pytest inherited through tox. + assert: It is present, so the deployment fixtures can authenticate to the tenant and + to GitHub. + Only the name is reported on failure — printing the value would defeat the + masking the workflow applied. + """ + # pytest.fail rather than assert: assertion rewriting would introspect the + # expression and dump the whole of os.environ into the failure output. + if not os.environ.get(setting): + pytest.fail( + f"{setting} did not reach pytest. Check that the workflow exports it and " + f"that tox passes it through in the garm-e2e environment." + ) diff --git a/tox.ini b/tox.ini index 3b320dc5..e6a08579 100644 --- a/tox.ini +++ b/tox.ini @@ -112,6 +112,29 @@ commands = --log-cli-level=INFO \ {posargs:{[vars]tests_path}/integration} +[testenv:garm-e2e] +pass_env = + PYTEST_ADDOPTS + OPCLI_ARTIFACTS_BUILD_YAML + SPREAD_JOB + JUJU_* + KUBECONFIG + OS_* + E2E_* + GITHUB_RUN_ID + GITHUB_REF_NAME +description = Run the GARM end-to-end test against ProdStack +set_env = + PYTHONPATH = {tox_root}/charms +deps = + -r {[vars]tests_path}/integration/requirements.txt +commands = + pytest -v \ + -s \ + --tb native \ + --log-cli-level=INFO \ + {posargs:{[vars]tests_path}/e2e} + [testenv:actions-lint] description = Run formatting and lint checks for Python code under actions/ deps = From 09389cad29cef5262e8bb3cebc16b9e6d37ee7e3 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Thu, 20 Aug 2026 11:45:42 +0000 Subject: [PATCH 02/21] ci(garm-e2e): drop the pipx and jq runner assumptions Copilot review: the workflow assumed both tools were present on the private-endpoint runner image, and pipx was the only occurrence in the repo. Install tox and the OpenStack client the way every other workflow here does, via setup-uv and uv tool install. Replace curl and jq with a stdlib Python script, which removes a dependency that would otherwise have to be installed over the private endpoint's restricted egress, and keeps the Vault response in memory so the credentials never reach the runner's disk at all. --- .github/workflows/garm_e2e.yaml | 147 +++++++++++++++++++------------- 1 file changed, 88 insertions(+), 59 deletions(-) diff --git a/.github/workflows/garm_e2e.yaml b/.github/workflows/garm_e2e.yaml index 13726afb..13f4ba37 100644 --- a/.github/workflows/garm_e2e.yaml +++ b/.github/workflows/garm_e2e.yaml @@ -19,6 +19,14 @@ jobs: steps: - uses: actions/checkout@v7.0.1 + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - name: Set up uv + uses: astral-sh/setup-uv@v10.0.1 + - name: Set up tmate session (self-hosted) if: runner.debug == 1 && runner.environment == 'self-hosted' uses: canonical/action-tmate@main @@ -88,70 +96,91 @@ jobs: VAULT_APPROLE_ROLE_ID: ${{ secrets.VAULT_APPROLE_ROLE_ID }} VAULT_APPROLE_SECRET_ID: ${{ secrets.VAULT_APPROLE_SECRET_ID }} VAULT_KV_PATH: ${{ secrets.E2E_VAULT_KV_PATH || 'kv/data/garm-e2e/prodstack' }} + # Python rather than curl and jq: it keeps the response in memory, so the + # credentials never reach the runner's disk, and it drops a dependency that + # would otherwise have to be installed over the private endpoint's egress. run: | - set -euo pipefail - - for KEY in VAULT_ADDR VAULT_APPROLE_ROLE_ID VAULT_APPROLE_SECRET_ID; do - if [ -z "${!KEY:-}" ]; then - echo "::error::Missing repository secret: $KEY (Settings > Secrets > Actions)." - exit 1 - fi - done - - # The response body holds the tenant credentials, so remove it however this - # step exits — set -e would otherwise skip the cleanup on every failure path, - # on a runner whose disk outlives the job. - RESPONSE=$(mktemp) - trap 'rm -f "$RESPONSE"' EXIT - - # curl --fail suppresses the response body, which would reduce a 403 to a - # bare exit code on the step whose job is to say what is misconfigured. - vault_call() { - curl -sS -o "$RESPONSE" -w '%{http_code}' "$@" - } - - STATUS=$(vault_call -X POST -H "Content-Type: application/json" \ - -d "$(jq -n --arg r "$VAULT_APPROLE_ROLE_ID" --arg s "$VAULT_APPROLE_SECRET_ID" \ - '{role_id: $r, secret_id: $s}')" \ - "${VAULT_ADDR}/v1/auth/approle/login") - if [ "$STATUS" != "200" ]; then - echo "::error::Vault AppRole login failed with HTTP $STATUS: $(jq -rc '.errors // empty' "$RESPONSE")" - exit 1 - fi - - VAULT_TOKEN=$(jq -r '.auth.client_token' "$RESPONSE") - echo "::add-mask::$VAULT_TOKEN" - if [ -z "$VAULT_TOKEN" ] || [ "$VAULT_TOKEN" = "null" ]; then - echo "::error::Vault AppRole login returned no client token." - exit 1 - fi - - STATUS=$(vault_call -X GET -H "X-Vault-Token: $VAULT_TOKEN" \ - "${VAULT_ADDR}/v1/${VAULT_KV_PATH}") - if [ "$STATUS" != "200" ]; then - echo "::error::Vault read of ${VAULT_KV_PATH} failed with HTTP $STATUS: $(jq -rc '.errors // empty' "$RESPONSE")" - exit 1 - fi - - for KEY in OS_USERNAME OS_PASSWORD; do - VALUE=$(jq -r --arg k "$KEY" '.data.data[$k] // empty' "$RESPONSE") - echo "::add-mask::$VALUE" - if [ -z "$VALUE" ]; then - echo "::error::Field '$KEY' missing from the Vault secret at ${VAULT_KV_PATH}." - exit 1 - fi - echo "${KEY}=${VALUE}" >> "$GITHUB_ENV" - done - - echo "Tenant credentials fetched and masked." + python3 - <<'SCRIPT' + import json + import os + import sys + import urllib.error + import urllib.request + + + def fail(message): + print(f"::error::{message}") + sys.exit(1) + + + def require(name): + value = os.environ.get(name) + if not value: + fail(f"Missing repository secret: {name} (Settings > Secrets > Actions).") + return value + + + def call(url, token=None, payload=None): + request = urllib.request.Request( + url, data=json.dumps(payload).encode() if payload else None + ) + request.add_header("Content-Type", "application/json") + if token: + request.add_header("X-Vault-Token", token) + try: + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response) + except urllib.error.HTTPError as exc: + # Vault reports what is misconfigured in the body; a bare status code + # would leave this step unable to say why it failed. Only the errors + # key is surfaced, never the whole body, which on a success would be + # the credentials themselves. + try: + errors = json.load(exc).get("errors", []) + except (ValueError, AttributeError): + errors = [] + fail(f"Vault request failed with HTTP {exc.code}: {errors}") + except urllib.error.URLError as exc: + fail(f"Vault at {addr} is unreachable: {exc.reason}") + + + addr = require("VAULT_ADDR").rstrip("/") + kv_path = require("VAULT_KV_PATH").strip("/") + + login = call( + f"{addr}/v1/auth/approle/login", + payload={ + "role_id": require("VAULT_APPROLE_ROLE_ID"), + "secret_id": require("VAULT_APPROLE_SECRET_ID"), + }, + ) + token = login.get("auth", {}).get("client_token") + if not token: + fail("Vault AppRole login returned no client token.") + print(f"::add-mask::{token}") + + secret = call(f"{addr}/v1/{kv_path}", token=token).get("data", {}).get("data", {}) + + with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as env_file: + for key in ("OS_USERNAME", "OS_PASSWORD"): + value = secret.get(key) + if not value: + fail( + f"Field {key!r} missing from the Vault secret at the configured " + f"KV path (E2E_VAULT_KV_PATH)." + ) + # Mask before writing: everything downstream may echo the env file. + print(f"::add-mask::{value}") + env_file.write(f"{key}={value}\n") + + print("Tenant credentials fetched and masked.") + SCRIPT - name: Assert the credentials authenticate against the tenant run: | set -euo pipefail - # Ubuntu 24.04 enforces PEP 668, so a bare pip install into the system - # interpreter fails with externally-managed-environment. - pipx install python-openstackclient + uv tool install python-openstackclient TOKEN_EXPIRES=$(openstack token issue -f value -c expires) echo "OpenStack token issued (expires: ${TOKEN_EXPIRES}). Tenant reachable." @@ -159,5 +188,5 @@ jobs: - name: Run the E2E suite run: | set -euo pipefail - pipx install tox + uv tool install tox --with tox-uv tox -e garm-e2e From 33e2f6f87eeeb7ab7020d0fd855eabe75331a395 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Thu, 20 Aug 2026 11:52:25 +0000 Subject: [PATCH 03/21] docs(garm-e2e): correct the disk claim and the private-key description MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copilot review: the step comment claimed the credentials never reach disk, but $GITHUB_ENV is a file — what the rewrite actually removes is the response body becoming a temporary file. Say that instead. The secrets table still described the key as base64-encoded after the workflow learned to accept a PEM, and the paragraph explaining it ended by naming the integration suite's secret, which this workflow does not use. --- .github/workflows/garm_e2e.yaml | 7 ++++--- CONTRIBUTING.md | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/garm_e2e.yaml b/.github/workflows/garm_e2e.yaml index 13f4ba37..7b3052b6 100644 --- a/.github/workflows/garm_e2e.yaml +++ b/.github/workflows/garm_e2e.yaml @@ -96,9 +96,10 @@ jobs: VAULT_APPROLE_ROLE_ID: ${{ secrets.VAULT_APPROLE_ROLE_ID }} VAULT_APPROLE_SECRET_ID: ${{ secrets.VAULT_APPROLE_SECRET_ID }} VAULT_KV_PATH: ${{ secrets.E2E_VAULT_KV_PATH || 'kv/data/garm-e2e/prodstack' }} - # Python rather than curl and jq: it keeps the response in memory, so the - # credentials never reach the runner's disk, and it drops a dependency that - # would otherwise have to be installed over the private endpoint's egress. + # Python rather than curl and jq: the Vault response body never becomes a + # temporary file, so only the two values it yields are handed on — through + # $GITHUB_ENV, like every other secret in this job. It also drops a dependency + # that would otherwise be installed over the private endpoint's egress. run: | python3 - <<'SCRIPT' import json diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6f0d78bb..ea9dc8a1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -271,7 +271,7 @@ not register masks for these itself. | `OS_NETWORK` | OpenStack network for runner VMs | | `E2E_GITHUB_APP_ID` | GitHub App ID | | `E2E_GITHUB_APP_INSTALLATION_ID` | Installation ID of that App on this repository | -| `E2E_GITHUB_APP_PRIVATE_KEY` | That App's private key (PEM), base64-encoded | +| `E2E_GITHUB_APP_PRIVATE_KEY` | That App's private key. Paste the PEM as issued; base64 is also accepted | | `E2E_VAULT_KV_PATH` | Optional. Defaults to `kv/data/garm-e2e/prodstack` | The OpenStack username and password are **not** repository secrets. They are read at run @@ -285,8 +285,8 @@ read & write` here, which the integration App has no reason to hold. Paste the private key into `E2E_GITHUB_APP_PRIVATE_KEY` exactly as GitHub issues it — the PEM, newlines and all. The workflow reduces it to a single line before it enters the two `KEY=value` channels that carry it to pytest, neither of which can hold a multi-line -value. A key that is already base64-encoded is accepted unchanged, so -`TEST_GITHUB_APP_PRIVATE_KEY`'s existing encoded form stays valid. +value. A value that is already base64-encoded is accepted unchanged, so an existing +encoded key does not need re-entering. #### Credential hygiene From ea48a71b674627d244f346f9a819c42fc69a9a31 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Thu, 20 Aug 2026 12:01:13 +0000 Subject: [PATCH 04/21] ci(garm-e2e): make the egress check a real if-then-else actionlint/shellcheck SC2015: in 'A && B || C', C also runs when A succeeds and B fails, so the failure branch was reachable from a successful curl. --- .github/workflows/garm_e2e_test_run.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/garm_e2e_test_run.yaml b/.github/workflows/garm_e2e_test_run.yaml index 936b2940..8f05fced 100644 --- a/.github/workflows/garm_e2e_test_run.yaml +++ b/.github/workflows/garm_e2e_test_run.yaml @@ -32,6 +32,9 @@ jobs: - name: Assert egress through aproxy run: | echo "Checking egress connectivity..." - curl -sf --max-time 10 https://github.com > /dev/null 2>&1 && \ - echo "Egress OK (via aproxy)" || \ - { echo "Egress FAILED"; exit 1; } + if curl -sf --max-time 10 https://github.com > /dev/null 2>&1; then + echo "Egress OK (via aproxy)" + else + echo "Egress FAILED" + exit 1 + fi From 6732ad37c5daa82b29ae7b3122eca47a0bc2f993 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Thu, 20 Aug 2026 12:03:54 +0000 Subject: [PATCH 05/21] ci(garm-e2e): allow label-gated runs on a pull request workflow_dispatch reads the workflow file from the default branch, so a change to garm_e2e.yaml itself cannot be exercised before it lands. Labelling a pull request run-e2e runs the branch's version. Gated on the label rather than firing on every pull request, so an unrelated change does not take the private-endpoint runner, and so it does not have to be removed again before merging. --- .github/workflows/garm_e2e.yaml | 10 ++++++++++ CONTRIBUTING.md | 5 +++++ 2 files changed, 15 insertions(+) diff --git a/.github/workflows/garm_e2e.yaml b/.github/workflows/garm_e2e.yaml index 7b3052b6..042f341c 100644 --- a/.github/workflows/garm_e2e.yaml +++ b/.github/workflows/garm_e2e.yaml @@ -10,10 +10,20 @@ name: GARM E2E on: workflow_dispatch: + # Opt-in pre-merge runs: label a pull request `run-e2e` to exercise this workflow + # from its branch. A label rather than every push, because the job occupies the + # private-endpoint runner, which is shared and scarce. + pull_request: + types: [opened, synchronize, labeled] jobs: e2e: name: GARM E2E test + # `pull_request`, never `pull_request_target`: a fork's code must not run with + # access to these secrets. A fork PR gets none, so it can only fail fast. + if: >- + github.event_name == 'workflow_dispatch' + || contains(github.event.pull_request.labels.*.name, 'run-e2e') runs-on: self-hosted-linux-amd64-noble-private-endpoint-medium timeout-minutes: 15 steps: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ea9dc8a1..f44d2c9a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -252,6 +252,11 @@ gh workflow run garm_e2e.yaml --ref `--ref` selects which branch's version of both the workflow and the test code runs, so changes to the end-to-end test can be exercised without merging them first. +A pull request labelled `run-e2e` also runs it, on every push while the label is +applied. That is the way to exercise a change to `garm_e2e.yaml` itself before it +lands, since `workflow_dispatch` only reads the workflow file from the default branch. +Remove the label when finished — the job occupies the private-endpoint runner. + #### Required secrets and variables Infrastructure details are secrets, not variables — endpoints, project and network names From ec904f90604e0a32c3c56eb88a115e428f19d969 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Thu, 20 Aug 2026 12:16:46 +0000 Subject: [PATCH 06/21] ci(garm-e2e): name the fields the Vault secret does hold The first real run failed on a missing OS_USERNAME, which leaves the reader guessing what the secret is keyed on. Report the keys that are present; a mismatch here is almost always a naming difference, and field names are not themselves secret. --- .github/workflows/garm_e2e.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/garm_e2e.yaml b/.github/workflows/garm_e2e.yaml index 042f341c..d92e9697 100644 --- a/.github/workflows/garm_e2e.yaml +++ b/.github/workflows/garm_e2e.yaml @@ -176,9 +176,11 @@ jobs: for key in ("OS_USERNAME", "OS_PASSWORD"): value = secret.get(key) if not value: + # Name what the secret does hold: the mismatch is almost always a + # field naming difference, and the keys are not themselves secret. fail( f"Field {key!r} missing from the Vault secret at the configured " - f"KV path (E2E_VAULT_KV_PATH)." + f"KV path (E2E_VAULT_KV_PATH). Fields present: {sorted(secret)}." ) # Mask before writing: everything downstream may echo the env file. print(f"::add-mask::{value}") From fdfb51d77dc02bc2e9e559250488700a752f6028 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Thu, 20 Aug 2026 12:20:58 +0000 Subject: [PATCH 07/21] ci(garm-e2e): read the Vault secret's own field names The secret is keyed on username and password; map them onto OS_USERNAME and OS_PASSWORD, which is what openstackclient and the tests read. Reverts printing the available field names, which was there to identify this mismatch and has served its purpose. --- .github/workflows/garm_e2e.yaml | 14 +++++++------- CONTRIBUTING.md | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/garm_e2e.yaml b/.github/workflows/garm_e2e.yaml index d92e9697..504d689a 100644 --- a/.github/workflows/garm_e2e.yaml +++ b/.github/workflows/garm_e2e.yaml @@ -172,19 +172,19 @@ jobs: secret = call(f"{addr}/v1/{kv_path}", token=token).get("data", {}).get("data", {}) + # The Vault secret is keyed on the plain names; openstackclient and the tests + # read them as OS_*. with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as env_file: - for key in ("OS_USERNAME", "OS_PASSWORD"): - value = secret.get(key) + for field, variable in (("username", "OS_USERNAME"), ("password", "OS_PASSWORD")): + value = secret.get(field) if not value: - # Name what the secret does hold: the mismatch is almost always a - # field naming difference, and the keys are not themselves secret. fail( - f"Field {key!r} missing from the Vault secret at the configured " - f"KV path (E2E_VAULT_KV_PATH). Fields present: {sorted(secret)}." + f"Field {field!r} missing from the Vault secret at the " + f"configured KV path (E2E_VAULT_KV_PATH)." ) # Mask before writing: everything downstream may echo the env file. print(f"::add-mask::{value}") - env_file.write(f"{key}={value}\n") + env_file.write(f"{variable}={value}\n") print("Tenant credentials fetched and masked.") SCRIPT diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f44d2c9a..9dcd0ec4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -280,7 +280,7 @@ not register masks for these itself. | `E2E_VAULT_KV_PATH` | Optional. Defaults to `kv/data/garm-e2e/prodstack` | The OpenStack username and password are **not** repository secrets. They are read at run -time from the Vault KV v2 secret above, which must hold `OS_USERNAME` and `OS_PASSWORD`. +time from the Vault KV v2 secret above, which must hold `username` and `password`. The `E2E_GITHUB_APP_*` trio is a GitHub App of its own, distinct from the `TEST_GITHUB_APP_*` one the integration suite uses. The end-to-end test registers and From 42804a6b97efd428ba0356b49b603ee3e733b00c Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Thu, 20 Aug 2026 12:25:09 +0000 Subject: [PATCH 08/21] ci(garm-e2e): accept both Vault KV layouts The read returned HTTP 200 but extracted nothing, because the fields were not where a KV v2 response puts them. Take data.data when it exists and data otherwise, so the mount version does not have to be known up front, and say so in the error when neither yields the field. --- .github/workflows/garm_e2e.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/garm_e2e.yaml b/.github/workflows/garm_e2e.yaml index 504d689a..125043ad 100644 --- a/.github/workflows/garm_e2e.yaml +++ b/.github/workflows/garm_e2e.yaml @@ -170,7 +170,10 @@ jobs: fail("Vault AppRole login returned no client token.") print(f"::add-mask::{token}") - secret = call(f"{addr}/v1/{kv_path}", token=token).get("data", {}).get("data", {}) + # KV v2 nests the fields under data.data; KV v1 puts them directly in data. + # Accept either, so the mount version does not have to be known up front. + body = call(f"{addr}/v1/{kv_path}", token=token).get("data", {}) + secret = body.get("data", body) # The Vault secret is keyed on the plain names; openstackclient and the tests # read them as OS_*. @@ -180,7 +183,10 @@ jobs: if not value: fail( f"Field {field!r} missing from the Vault secret at the " - f"configured KV path (E2E_VAULT_KV_PATH)." + f"configured KV path (E2E_VAULT_KV_PATH). If the read " + f"succeeds but no field is found, the path is addressing the " + f"wrong thing: a KV v2 mount needs a literal 'data/' after " + f"the mount name, a KV v1 mount must not have it." ) # Mask before writing: everything downstream may echo the env file. print(f"::add-mask::{value}") From 9567582e41f374e0cf44f087bd6d9b9b9b0d7a98 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Thu, 20 Aug 2026 12:33:11 +0000 Subject: [PATCH 09/21] ci(garm-e2e): report a missing credential without describing the secret What a Vault secret contains, down to the names of its fields, is infrastructure detail that does not belong in a CI log. Report only that the expected fields were not found and that the secret needs checking by hand. Validate before writing, so a secret holding only one of the two cannot leave a half-populated environment for a later step. --- .github/workflows/garm_e2e.yaml | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/garm_e2e.yaml b/.github/workflows/garm_e2e.yaml index 125043ad..8bdbacc7 100644 --- a/.github/workflows/garm_e2e.yaml +++ b/.github/workflows/garm_e2e.yaml @@ -175,22 +175,25 @@ jobs: body = call(f"{addr}/v1/{kv_path}", token=token).get("data", {}) secret = body.get("data", body) - # The Vault secret is keyed on the plain names; openstackclient and the tests - # read them as OS_*. + # The Vault secret is keyed on the plain names; openstackclient and the + # tests read them as OS_*. + fields = (("username", "OS_USERNAME"), ("password", "OS_PASSWORD")) + + # Checked before anything is written, so a half-populated environment cannot + # reach a later step. Nothing about the secret is reported: what it contains, + # down to the names of its fields, is not for a CI log. + if not all(secret.get(field) for field, _ in fields): + fail( + "The Vault secret does not provide both a 'username' and a " + "'password' field. Check it manually at the path configured in " + "E2E_VAULT_KV_PATH." + ) + with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as env_file: - for field, variable in (("username", "OS_USERNAME"), ("password", "OS_PASSWORD")): - value = secret.get(field) - if not value: - fail( - f"Field {field!r} missing from the Vault secret at the " - f"configured KV path (E2E_VAULT_KV_PATH). If the read " - f"succeeds but no field is found, the path is addressing the " - f"wrong thing: a KV v2 mount needs a literal 'data/' after " - f"the mount name, a KV v1 mount must not have it." - ) + for field, variable in fields: # Mask before writing: everything downstream may echo the env file. - print(f"::add-mask::{value}") - env_file.write(f"{variable}={value}\n") + print(f"::add-mask::{secret[field]}") + env_file.write(f"{variable}={secret[field]}\n") print("Tenant credentials fetched and masked.") SCRIPT From c7a9b98e80a7ebbceeafdba4ab27cf6115ab21f4 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Thu, 20 Aug 2026 12:40:01 +0000 Subject: [PATCH 10/21] ci(garm-e2e): drop the temporary pull_request trigger It existed to prove the workflow runs on the private-endpoint runner before merging, which run 32369451367 did. Back to workflow_dispatch only. --- .github/workflows/garm_e2e.yaml | 10 ---------- CONTRIBUTING.md | 5 ----- 2 files changed, 15 deletions(-) diff --git a/.github/workflows/garm_e2e.yaml b/.github/workflows/garm_e2e.yaml index 8bdbacc7..d0d254b2 100644 --- a/.github/workflows/garm_e2e.yaml +++ b/.github/workflows/garm_e2e.yaml @@ -10,20 +10,10 @@ name: GARM E2E on: workflow_dispatch: - # Opt-in pre-merge runs: label a pull request `run-e2e` to exercise this workflow - # from its branch. A label rather than every push, because the job occupies the - # private-endpoint runner, which is shared and scarce. - pull_request: - types: [opened, synchronize, labeled] jobs: e2e: name: GARM E2E test - # `pull_request`, never `pull_request_target`: a fork's code must not run with - # access to these secrets. A fork PR gets none, so it can only fail fast. - if: >- - github.event_name == 'workflow_dispatch' - || contains(github.event.pull_request.labels.*.name, 'run-e2e') runs-on: self-hosted-linux-amd64-noble-private-endpoint-medium timeout-minutes: 15 steps: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9dcd0ec4..159956e5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -252,11 +252,6 @@ gh workflow run garm_e2e.yaml --ref `--ref` selects which branch's version of both the workflow and the test code runs, so changes to the end-to-end test can be exercised without merging them first. -A pull request labelled `run-e2e` also runs it, on every push while the label is -applied. That is the way to exercise a change to `garm_e2e.yaml` itself before it -lands, since `workflow_dispatch` only reads the workflow file from the default branch. -Remove the label when finished — the job occupies the private-endpoint runner. - #### Required secrets and variables Infrastructure details are secrets, not variables — endpoints, project and network names From 47427ac7198b01e0fb9b9c8780a053055264cb30 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Thu, 20 Aug 2026 12:46:58 +0000 Subject: [PATCH 11/21] ci(garm-e2e): mask the normalised Vault address, accept a wrapped key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two consequences of masking being literal-substring matching, both found in review: rstrip('/') makes the address a different string from the secret whenever VAULT_ADDR carries a trailing slash, so the runner would not have masked the form that reaches the unreachable-host error. Mask it at the point it is derived. The already-encoded branch passed the secret through untouched, but base64 wraps at 76 columns unless told otherwise, and a wrapped blob would break the env file — contradicting the documented promise that an encoded key is accepted. Strip whitespace there too. --- .github/workflows/garm_e2e.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/garm_e2e.yaml b/.github/workflows/garm_e2e.yaml index d0d254b2..061cc1cc 100644 --- a/.github/workflows/garm_e2e.yaml +++ b/.github/workflows/garm_e2e.yaml @@ -79,7 +79,9 @@ jobs: KEY_B64=$(printf '%s' "$E2E_GITHUB_APP_PRIVATE_KEY" | base64 -w0) ;; *) - KEY_B64=$E2E_GITHUB_APP_PRIVATE_KEY + # Already encoded: strip whitespace, since `base64` wraps at 76 columns + # unless told otherwise and a wrapped blob would break the env file. + KEY_B64=$(printf '%s' "$E2E_GITHUB_APP_PRIVATE_KEY" | tr -d '[:space:]') ;; esac # The encoded form is a different string from the secret, so the runner does @@ -146,6 +148,10 @@ jobs: addr = require("VAULT_ADDR").rstrip("/") + # Normalising makes this a different string from the secret whenever + # VAULT_ADDR carries a trailing slash, and masking matches literal + # substrings, so the runner would not mask the derived form. + print(f"::add-mask::{addr}") kv_path = require("VAULT_KV_PATH").strip("/") login = call( From 1e8110212c7dcbf417783bff1a594ea6a4c66b62 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Thu, 20 Aug 2026 12:56:59 +0000 Subject: [PATCH 12/21] ci(garm-e2e): declare least-privilege permissions Both workflows inherited the repository default GITHUB_TOKEN scope. Neither needs more than read access to the repository, and one of them handles production tenant credentials, so state the scope rather than inheriting it. Also drop 'and variables' from the secrets heading, left over from before every infrastructure setting became a secret. --- .github/workflows/garm_e2e.yaml | 3 +++ .github/workflows/garm_e2e_test_run.yaml | 3 +++ CONTRIBUTING.md | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/garm_e2e.yaml b/.github/workflows/garm_e2e.yaml index 061cc1cc..f25f1200 100644 --- a/.github/workflows/garm_e2e.yaml +++ b/.github/workflows/garm_e2e.yaml @@ -11,6 +11,9 @@ name: GARM E2E on: workflow_dispatch: +permissions: + contents: read + jobs: e2e: name: GARM E2E test diff --git a/.github/workflows/garm_e2e_test_run.yaml b/.github/workflows/garm_e2e_test_run.yaml index 8f05fced..41694cd2 100644 --- a/.github/workflows/garm_e2e_test_run.yaml +++ b/.github/workflows/garm_e2e_test_run.yaml @@ -15,6 +15,9 @@ on: required: true type: string +permissions: + contents: read + jobs: e2e-test: name: GARM E2E test diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 159956e5..05b805d7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -252,7 +252,7 @@ gh workflow run garm_e2e.yaml --ref `--ref` selects which branch's version of both the workflow and the test code runs, so changes to the end-to-end test can be exercised without merging them first. -#### Required secrets and variables +#### Required secrets Infrastructure details are secrets, not variables — endpoints, project and network names included. The runner masks secret values in the log automatically, so the workflow does From ae5fe98c6a8869fbb5a4f74ed690dd0fea0ab5f3 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Thu, 20 Aug 2026 14:17:01 +0000 Subject: [PATCH 13/21] ci(garm-e2e): re-add the label trigger to verify the review fixes The last green run predates the least-privilege permissions, the masked Vault address and the whitespace-stripped key. Narrowing the token scope in particular can only fail at runtime. Removed again once verified. --- .github/workflows/garm_e2e.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/garm_e2e.yaml b/.github/workflows/garm_e2e.yaml index f25f1200..dc30ef6e 100644 --- a/.github/workflows/garm_e2e.yaml +++ b/.github/workflows/garm_e2e.yaml @@ -10,6 +10,10 @@ name: GARM E2E on: workflow_dispatch: + # Temporary: label a pull request `run-e2e` to exercise this workflow from its + # branch. Removed again once the run has served its purpose. + pull_request: + types: [opened, synchronize, labeled] permissions: contents: read @@ -17,6 +21,11 @@ permissions: jobs: e2e: name: GARM E2E test + # `pull_request`, never `pull_request_target`: a fork's code must not run with + # access to these secrets. A fork PR gets none, so it can only fail fast. + if: >- + github.event_name == 'workflow_dispatch' + || contains(github.event.pull_request.labels.*.name, 'run-e2e') runs-on: self-hosted-linux-amd64-noble-private-endpoint-medium timeout-minutes: 15 steps: From 7c91717856e3006b56b2d9951450cdaf31e12a65 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Thu, 20 Aug 2026 14:27:39 +0000 Subject: [PATCH 14/21] ci(garm-e2e): drop the label trigger again Run 32379213143 confirmed the workflow still passes under the narrowed token scope, with the masked Vault address and the whitespace-stripped key. --- .github/workflows/garm_e2e.yaml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/garm_e2e.yaml b/.github/workflows/garm_e2e.yaml index dc30ef6e..f25f1200 100644 --- a/.github/workflows/garm_e2e.yaml +++ b/.github/workflows/garm_e2e.yaml @@ -10,10 +10,6 @@ name: GARM E2E on: workflow_dispatch: - # Temporary: label a pull request `run-e2e` to exercise this workflow from its - # branch. Removed again once the run has served its purpose. - pull_request: - types: [opened, synchronize, labeled] permissions: contents: read @@ -21,11 +17,6 @@ permissions: jobs: e2e: name: GARM E2E test - # `pull_request`, never `pull_request_target`: a fork's code must not run with - # access to these secrets. A fork PR gets none, so it can only fail fast. - if: >- - github.event_name == 'workflow_dispatch' - || contains(github.event.pull_request.labels.*.name, 'run-e2e') runs-on: self-hosted-linux-amd64-noble-private-endpoint-medium timeout-minutes: 15 steps: From 2d579ec6109f6fadbf3a0fc7c0de728ed0914a66 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Fri, 21 Aug 2026 05:18:38 +0000 Subject: [PATCH 15/21] ci(garm-e2e): make the runner probe assert what its name claims The egress step was named for aproxy but only proved that github.com was reachable. The aproxy bootstrap fails open -- every error path in it skips the nftables redirect and exits 0 -- so a runner whose proxy was never wired up passed that check wherever the tenant had a route of its own. Match aproxy's own log to tell the two apart, and probe :80 as well as :443, since aproxy reads the destination from the Host header on one and from TLS SNI on the other. Verified against aproxy 0.2.5 (the snap the charm installs) behind a real nftables redirect: it logs `host=:` per relayed connection, which is what the check matches. Match with a `case` glob rather than a pipe into `grep -q`: `grep -q` exits at its first match and SIGPIPEs its writer, so under `set -o pipefail`, which is the shell Actions runs steps with, the pipeline failed despite matching. Drop the stderr redirect that hid curl's reason for failing, and report the runner user and home, which much of the tooling in a job assumes. --- .github/workflows/garm_e2e_test_run.yaml | 63 ++++++++++++++++++++---- 1 file changed, 54 insertions(+), 9 deletions(-) diff --git a/.github/workflows/garm_e2e_test_run.yaml b/.github/workflows/garm_e2e_test_run.yaml index 41694cd2..19e9b375 100644 --- a/.github/workflows/garm_e2e_test_run.yaml +++ b/.github/workflows/garm_e2e_test_run.yaml @@ -4,6 +4,11 @@ # Trivial workflow that a spawned runner executes to prove the full chain: # GARM → OpenStack → VM → runner register → job dispatch → pick up → exit. # Modelled on github-runner-operator/.github/workflows/e2e_test_run.yaml. +# +# Reaching this workflow at all is most of the assertion: it only runs if a VM +# was booted from the published image and its runner registered and claimed the +# job. The steps below cover what that does not — that the runner is usable for +# real work. Image contents are ISD298's remit, not this workflow's; keep it short. name: GARM E2E test run @@ -27,17 +32,57 @@ jobs: runs-on: "${{ inputs.runner-label }}" timeout-minutes: 10 steps: - - name: Assert basic runner liveness + - name: Report runner identity run: | echo "=== GARM E2E test run ===" echo "Runner: $(hostname)" - echo "Kernel: $(uname -a)" - - name: Assert egress through aproxy + echo "Kernel: $(uname -sr)" + echo "User: $(whoami)" + echo "Home: $HOME" + - name: Assert egress + run: | + # The charm exports no proxy variables into the runner environment: it + # delivers the proxy transparently, as aproxy plus an nftables redirect of + # :80 and :443. A plain curl is therefore what exercises that path, and + # --noproxy keeps that true if a future image starts setting http_proxy. + # Both ports are probed because aproxy reads the destination differently + # for each — Host header on :80, TLS SNI on :443 — and jobs need both + # (apt over :80, everything else over :443). + curl -sSf --noproxy '*' --max-time 30 -o /dev/null http://github.com + curl -sSf --noproxy '*' --max-time 30 -o /dev/null https://github.com + echo "Egress reached github.com on :80 and :443" + - name: Assert egress traversed aproxy run: | - echo "Checking egress connectivity..." - if curl -sf --max-time 10 https://github.com > /dev/null 2>&1; then - echo "Egress OK (via aproxy)" - else - echo "Egress FAILED" - exit 1 + # The aproxy bootstrap fails open by design: every error path in it skips + # the nftables redirect and exits 0, so a runner whose proxy was never + # wired up still boots, still registers, and still passes the step above + # wherever the tenant happens to have a route of its own. aproxy's log is + # what tells the two apart. + if ! snap list aproxy > /dev/null 2>&1; then + echo "No aproxy on this runner, so the scale set was configured without a proxy." + exit 0 + fi + # Reading the log needs root even for a user in `adm`; treat losing that + # as unverified rather than as a failure, so this cannot turn into a red + # run over a sudo policy change. + if ! APROXY_LOG=$(sudo -n snap logs aproxy.aproxy -n=all 2>/dev/null); then + echo "::warning::aproxy's log is unreadable, leaving the redirect unverified." + exit 0 fi + # aproxy logs one `host=:` per relayed connection. The log is + # only ever matched, never printed: it sits alongside the snap + # configuration, which holds the upstream proxy and may embed credentials. + # Matched with a `case` glob rather than a pipe into `grep -q`, because + # `grep -q` exits at its first match and so SIGPIPEs the writer feeding it, + # which under `set -o pipefail` -- the shell Actions runs steps with -- fails + # the pipeline despite the match. + for HOSTPORT in github.com:80 github.com:443; do + case "$APROXY_LOG" in + *"host=$HOSTPORT"*) ;; + *) + echo "::error::aproxy is installed but relayed no connection to $HOSTPORT, so egress bypassed it. Check the 00-aproxy pre-install script in the runner's cloud-init output." + exit 1 + ;; + esac + done + echo "Egress traversed aproxy on :80 and :443" From 8599069e0169951479b1e3e6a2c82a9277bbb138 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Fri, 21 Aug 2026 05:34:31 +0000 Subject: [PATCH 16/21] ci(garm-e2e): narrow this PR to the dispatch target Only garm_e2e_test_run.yaml has an ordering constraint: nothing can dispatch it until it is registered on the default branch, so it cannot be exercised before it merges and it is the one file worth reviewing on its own. garm_e2e.yaml and the tox/docs wiring around it have a pre-merge path -- a label-gated trigger verified them twice on the private-endpoint runner -- so they lose nothing by following in a separate PR. The two workflows do not reference each other. --- .github/workflows/garm_e2e.yaml | 213 ------------------------------ AGENTS.md | 5 +- CONTRIBUTING.md | 72 ---------- charms/tests/e2e/test_garm_e2e.py | 56 -------- tox.ini | 23 ---- 5 files changed, 2 insertions(+), 367 deletions(-) delete mode 100644 .github/workflows/garm_e2e.yaml delete mode 100644 charms/tests/e2e/test_garm_e2e.py diff --git a/.github/workflows/garm_e2e.yaml b/.github/workflows/garm_e2e.yaml deleted file mode 100644 index f25f1200..00000000 --- a/.github/workflows/garm_e2e.yaml +++ /dev/null @@ -1,213 +0,0 @@ -# Copyright 2026 Canonical Ltd. -# See LICENSE file for licensing details. -# -# GARM end-to-end test workflow. -# -# Manual trigger only, and deliberately not a merge gate. Dispatch it with -# --ref to run a feature branch's version. - -name: GARM E2E - -on: - workflow_dispatch: - -permissions: - contents: read - -jobs: - e2e: - name: GARM E2E test - runs-on: self-hosted-linux-amd64-noble-private-endpoint-medium - timeout-minutes: 15 - steps: - - uses: actions/checkout@v7.0.1 - - - name: Set up Python - uses: actions/setup-python@v7 - with: - python-version: "3.12" - - - name: Set up uv - uses: astral-sh/setup-uv@v10.0.1 - - - name: Set up tmate session (self-hosted) - if: runner.debug == 1 && runner.environment == 'self-hosted' - uses: canonical/action-tmate@main - with: - detached: true - timeout-minutes: 60 - - # These are all secrets, so the runner masks them in the log without an - # explicit ::add-mask::. The E2E_GITHUB_APP_* trio is a GitHub App of its - # own, separate from the TEST_GITHUB_APP_* one the integration suite uses: - # this one has to manage runner scale sets, which that one must not. - - name: Load tenant and GitHub App configuration - env: - OS_AUTH_URL: ${{ secrets.OS_AUTH_URL }} - OS_PROJECT_NAME: ${{ secrets.OS_PROJECT_NAME }} - OS_USER_DOMAIN_NAME: ${{ secrets.OS_USER_DOMAIN_NAME }} - OS_PROJECT_DOMAIN_NAME: ${{ secrets.OS_PROJECT_DOMAIN_NAME }} - OS_REGION_NAME: ${{ secrets.OS_REGION_NAME }} - OS_NETWORK: ${{ secrets.OS_NETWORK }} - E2E_GITHUB_APP_ID: ${{ secrets.E2E_GITHUB_APP_ID }} - E2E_GITHUB_APP_INSTALLATION_ID: ${{ secrets.E2E_GITHUB_APP_INSTALLATION_ID }} - E2E_GITHUB_APP_PRIVATE_KEY: ${{ secrets.E2E_GITHUB_APP_PRIVATE_KEY }} - run: | - set -euo pipefail - - for KEY in OS_AUTH_URL OS_PROJECT_NAME OS_USER_DOMAIN_NAME \ - OS_PROJECT_DOMAIN_NAME OS_REGION_NAME OS_NETWORK \ - E2E_GITHUB_APP_ID E2E_GITHUB_APP_INSTALLATION_ID \ - E2E_GITHUB_APP_PRIVATE_KEY; do - if [ -z "${!KEY:-}" ]; then - echo "::error::Missing repository secret: $KEY (Settings > Secrets > Actions)." - exit 1 - fi - done - - # Everything but the key is single-line already. - for KEY in OS_AUTH_URL OS_PROJECT_NAME OS_USER_DOMAIN_NAME \ - OS_PROJECT_DOMAIN_NAME OS_REGION_NAME OS_NETWORK \ - E2E_GITHUB_APP_ID E2E_GITHUB_APP_INSTALLATION_ID; do - echo "${KEY}=${!KEY}" >> "$GITHUB_ENV" - done - - # The key travels through two channels that take one KEY=value per line — - # this file, and later opcli's pytest-environment-template, which rejects a - # line without an '='. Normalising here rather than requiring a pre-encoded - # secret keeps the encoding out of everyone's setup instructions. An - # already-encoded value passes through, so both forms work. - case "$E2E_GITHUB_APP_PRIVATE_KEY" in - *-----BEGIN*) - KEY_B64=$(printf '%s' "$E2E_GITHUB_APP_PRIVATE_KEY" | base64 -w0) - ;; - *) - # Already encoded: strip whitespace, since `base64` wraps at 76 columns - # unless told otherwise and a wrapped blob would break the env file. - KEY_B64=$(printf '%s' "$E2E_GITHUB_APP_PRIVATE_KEY" | tr -d '[:space:]') - ;; - esac - # The encoded form is a different string from the secret, so the runner does - # not mask it on its own. - echo "::add-mask::$KEY_B64" - echo "E2E_GITHUB_APP_PRIVATE_KEY=$KEY_B64" >> "$GITHUB_ENV" - - # openstackclient needs this stated explicitly for a v3 auth URL. - echo "OS_IDENTITY_API_VERSION=3" >> "$GITHUB_ENV" - - - name: Fetch tenant credentials from Vault - env: - VAULT_ADDR: ${{ secrets.VAULT_ADDR }} - VAULT_APPROLE_ROLE_ID: ${{ secrets.VAULT_APPROLE_ROLE_ID }} - VAULT_APPROLE_SECRET_ID: ${{ secrets.VAULT_APPROLE_SECRET_ID }} - VAULT_KV_PATH: ${{ secrets.E2E_VAULT_KV_PATH || 'kv/data/garm-e2e/prodstack' }} - # Python rather than curl and jq: the Vault response body never becomes a - # temporary file, so only the two values it yields are handed on — through - # $GITHUB_ENV, like every other secret in this job. It also drops a dependency - # that would otherwise be installed over the private endpoint's egress. - run: | - python3 - <<'SCRIPT' - import json - import os - import sys - import urllib.error - import urllib.request - - - def fail(message): - print(f"::error::{message}") - sys.exit(1) - - - def require(name): - value = os.environ.get(name) - if not value: - fail(f"Missing repository secret: {name} (Settings > Secrets > Actions).") - return value - - - def call(url, token=None, payload=None): - request = urllib.request.Request( - url, data=json.dumps(payload).encode() if payload else None - ) - request.add_header("Content-Type", "application/json") - if token: - request.add_header("X-Vault-Token", token) - try: - with urllib.request.urlopen(request, timeout=30) as response: - return json.load(response) - except urllib.error.HTTPError as exc: - # Vault reports what is misconfigured in the body; a bare status code - # would leave this step unable to say why it failed. Only the errors - # key is surfaced, never the whole body, which on a success would be - # the credentials themselves. - try: - errors = json.load(exc).get("errors", []) - except (ValueError, AttributeError): - errors = [] - fail(f"Vault request failed with HTTP {exc.code}: {errors}") - except urllib.error.URLError as exc: - fail(f"Vault at {addr} is unreachable: {exc.reason}") - - - addr = require("VAULT_ADDR").rstrip("/") - # Normalising makes this a different string from the secret whenever - # VAULT_ADDR carries a trailing slash, and masking matches literal - # substrings, so the runner would not mask the derived form. - print(f"::add-mask::{addr}") - kv_path = require("VAULT_KV_PATH").strip("/") - - login = call( - f"{addr}/v1/auth/approle/login", - payload={ - "role_id": require("VAULT_APPROLE_ROLE_ID"), - "secret_id": require("VAULT_APPROLE_SECRET_ID"), - }, - ) - token = login.get("auth", {}).get("client_token") - if not token: - fail("Vault AppRole login returned no client token.") - print(f"::add-mask::{token}") - - # KV v2 nests the fields under data.data; KV v1 puts them directly in data. - # Accept either, so the mount version does not have to be known up front. - body = call(f"{addr}/v1/{kv_path}", token=token).get("data", {}) - secret = body.get("data", body) - - # The Vault secret is keyed on the plain names; openstackclient and the - # tests read them as OS_*. - fields = (("username", "OS_USERNAME"), ("password", "OS_PASSWORD")) - - # Checked before anything is written, so a half-populated environment cannot - # reach a later step. Nothing about the secret is reported: what it contains, - # down to the names of its fields, is not for a CI log. - if not all(secret.get(field) for field, _ in fields): - fail( - "The Vault secret does not provide both a 'username' and a " - "'password' field. Check it manually at the path configured in " - "E2E_VAULT_KV_PATH." - ) - - with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as env_file: - for field, variable in fields: - # Mask before writing: everything downstream may echo the env file. - print(f"::add-mask::{secret[field]}") - env_file.write(f"{variable}={secret[field]}\n") - - print("Tenant credentials fetched and masked.") - SCRIPT - - - name: Assert the credentials authenticate against the tenant - run: | - set -euo pipefail - - uv tool install python-openstackclient - - TOKEN_EXPIRES=$(openstack token issue -f value -c expires) - echo "OpenStack token issued (expires: ${TOKEN_EXPIRES}). Tenant reachable." - - - name: Run the E2E suite - run: | - set -euo pipefail - uv tool install tox --with tox-uv - tox -e garm-e2e diff --git a/AGENTS.md b/AGENTS.md index b191d527..7a505949 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,7 +8,7 @@ guidance in `.github/instructions/` and the human-facing `CONTRIBUTING.md`. | Path | Contents | | --- | --- | -| `charms/` | Four Juju charms (see below) plus shared integration tests in `charms/tests/integration/` and the GARM end-to-end test in `charms/tests/e2e/`. | +| `charms/` | Four Juju charms (see below) plus shared integration tests in `charms/tests/integration/`. | | `cmd/` | Go application entry points: `planner`, `webhook-gateway`. | | `internal/` | Shared Go packages (`database`, `github`, `planner`, `queue`, `server`, `telemetry`, `webhook`, …) — the application logic the paas charms package and deploy. | | `*-rockcraft.yaml`, `build-*-rock.sh` (repo root) | Rock/image build definitions and their build scripts. | @@ -33,7 +33,6 @@ charms — there are four charms, not two). - **Per-charm Python checks** — from the charm directory, `tox -c tox.toml` (envs `fmt`, `lint`, `complexity`, `static`, `unit`, `coverage-report`; ruff, codespell, pyright, pytest+coverage). CI runs these per charm via `tox -c tox.toml`. - **Integration tests** (root `tox.ini`) — `tox -e -integration` (`garm`, `webhook-gateway`, `planner`, `garm-configurator`) or `tox -e charms-integration` for all. Requires a live Juju model (jubilant + pytest-operator). -- **GARM end-to-end test** (root `tox.ini`) — `tox -e garm-e2e`. Runs against a real OpenStack tenant from `garm_e2e.yaml`, is manually dispatched, and is **not** a merge gate. See `CONTRIBUTING.md` §"GARM E2E". - **`actions/` Python** — `tox -e actions-lint`, `tox -e actions-static`, `tox -e actions-unit`. - **Go** — `go test ./...`. - `charmcraft pack` — build a charm (run from the charm dir; not wired into tox). @@ -89,7 +88,7 @@ For **`garm-configurator`** (plain `ops`): - **DO** fix a missing AAA docstring on any test you move or edit. It's followed unevenly (`planner-operator` and `garm` yes; `garm-configurator` and `webhook-gateway-operator` not yet), so imitating the nearest neighbour is not a reliable guide. -- Integration tests live in the shared `charms/tests/integration/`; the GARM end-to-end test lives in `charms/tests/e2e/`, kept separate so it stays out of the PR test matrix. +- Integration tests live in the shared `charms/tests/integration/`. ## 12-factor divergences from the canonical charm-engineer guidance diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 05b805d7..9dc34f6b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -231,75 +231,3 @@ rockcraft.skopeo copy \ oci-archive:webhook-gateway_0.1_amd64.rock \ docker://localhost:32000/webhook-gateway:0.1 ``` - -### GARM E2E - -The GARM end-to-end test (`charms/tests/e2e/`) exercises the full chain on ProdStack: -the charm starts, the configurator delivers config, the GARM API becomes reachable, the -provider authenticates to OpenStack, a VM is created from the runner image, the runner -registers with GitHub, and a dispatched job runs and exits clean. - -> **TODO:** only the workflows and the credential path exist so far. The test currently -> asserts nothing beyond the credentials reaching pytest; the deployment and the -> end-to-end assertions land in a follow-up. - -It is triggered manually and is **not** a merge gate: - -```shell -gh workflow run garm_e2e.yaml --ref -``` - -`--ref` selects which branch's version of both the workflow and the test code runs, so -changes to the end-to-end test can be exercised without merging them first. - -#### Required secrets - -Infrastructure details are secrets, not variables — endpoints, project and network names -included. The runner masks secret values in the log automatically, so the workflow does -not register masks for these itself. - -| Name | Description | -| --- | --- | -| `VAULT_ADDR` | Vault server address | -| `VAULT_APPROLE_ROLE_ID` | Vault AppRole role ID | -| `VAULT_APPROLE_SECRET_ID` | Vault AppRole secret ID | -| `OS_AUTH_URL` | Keystone endpoint, e.g. `https://keystone.example.com:5000/v3` | -| `OS_PROJECT_NAME` | OpenStack project/tenant name | -| `OS_USER_DOMAIN_NAME` | OpenStack user domain name | -| `OS_PROJECT_DOMAIN_NAME` | OpenStack project domain name | -| `OS_REGION_NAME` | OpenStack region name | -| `OS_NETWORK` | OpenStack network for runner VMs | -| `E2E_GITHUB_APP_ID` | GitHub App ID | -| `E2E_GITHUB_APP_INSTALLATION_ID` | Installation ID of that App on this repository | -| `E2E_GITHUB_APP_PRIVATE_KEY` | That App's private key. Paste the PEM as issued; base64 is also accepted | -| `E2E_VAULT_KV_PATH` | Optional. Defaults to `kv/data/garm-e2e/prodstack` | - -The OpenStack username and password are **not** repository secrets. They are read at run -time from the Vault KV v2 secret above, which must hold `username` and `password`. - -The `E2E_GITHUB_APP_*` trio is a GitHub App of its own, distinct from the -`TEST_GITHUB_APP_*` one the integration suite uses. The end-to-end test registers and -tears down a runner scale set on this repository, so its App needs `Administration: -read & write` here, which the integration App has no reason to hold. - -Paste the private key into `E2E_GITHUB_APP_PRIVATE_KEY` exactly as GitHub issues it — -the PEM, newlines and all. The workflow reduces it to a single line before it enters the -two `KEY=value` channels that carry it to pytest, neither of which can hold a multi-line -value. A value that is already base64-encoded is accepted unchanged, so an existing -encoded key does not need re-entering. - -#### Credential hygiene - -The test authenticates against a production tenant, so the run logs are part of the -contract: - -- Values read from Vault are `::add-mask::`ed in the same step that reads them, before any - later step runs. Masking registered with the runner scrubs every subsequent log line, - including output from code we do not control; a secret read inside the test process gets - none of that. Repository secrets are masked by the runner already. -- They reach later steps through `$GITHUB_ENV` and pytest through tox's `pass_env` — never - through command line arguments, which would expose them in `ps` and in pytest's header. -- Steps handling credentials use `set -euo pipefail` and never `set -x`. -- Assertions report the *name* of a missing setting, never its value, and use - `pytest.fail` rather than `assert` so that assertion rewriting cannot introspect - `os.environ` into the failure output. diff --git a/charms/tests/e2e/test_garm_e2e.py b/charms/tests/e2e/test_garm_e2e.py deleted file mode 100644 index 0cd545df..00000000 --- a/charms/tests/e2e/test_garm_e2e.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2026 Canonical Ltd. -# See LICENSE file for licensing details. - -"""GARM end-to-end test. - -TODO: This module is deliberately incomplete. It exists so that the workflows have -something to run; the end-to-end implementation lands in a follow-up. What is here -asserts only that the credentials reach pytest. - -Still to come: deploy postgresql, GARM, a traefik ingress and garm-configurator; assert -the GARM API is reachable and the provider authenticates to OpenStack; assert a VM is -created from the runner image and the runner registers with GitHub; then dispatch -``garm_e2e_test_run.yaml`` at the scale set's label and assert the job is picked up and -exits clean. -""" - -import os - -import pytest - -# Settings the deployment fixtures will need. The OpenStack username and password come -# from Vault, the rest from repository secrets; both routes converge on the job -# environment, which tox forwards to pytest. -REQUIRED_SETTINGS = ( - "OS_AUTH_URL", - "OS_USERNAME", - "OS_PASSWORD", - "OS_PROJECT_NAME", - "OS_USER_DOMAIN_NAME", - "OS_PROJECT_DOMAIN_NAME", - "OS_REGION_NAME", - "OS_NETWORK", - "E2E_GITHUB_APP_ID", - "E2E_GITHUB_APP_INSTALLATION_ID", - "E2E_GITHUB_APP_PRIVATE_KEY", -) - - -@pytest.mark.parametrize("setting", REQUIRED_SETTINGS) -def test_setting_reaches_pytest(setting: str): - """ - arrange: The workflow has resolved the settings, taking the OpenStack username and - password from Vault and the rest from repository secrets, and exported them. - act: Read the setting pytest inherited through tox. - assert: It is present, so the deployment fixtures can authenticate to the tenant and - to GitHub. - Only the name is reported on failure — printing the value would defeat the - masking the workflow applied. - """ - # pytest.fail rather than assert: assertion rewriting would introspect the - # expression and dump the whole of os.environ into the failure output. - if not os.environ.get(setting): - pytest.fail( - f"{setting} did not reach pytest. Check that the workflow exports it and " - f"that tox passes it through in the garm-e2e environment." - ) diff --git a/tox.ini b/tox.ini index e6a08579..3b320dc5 100644 --- a/tox.ini +++ b/tox.ini @@ -112,29 +112,6 @@ commands = --log-cli-level=INFO \ {posargs:{[vars]tests_path}/integration} -[testenv:garm-e2e] -pass_env = - PYTEST_ADDOPTS - OPCLI_ARTIFACTS_BUILD_YAML - SPREAD_JOB - JUJU_* - KUBECONFIG - OS_* - E2E_* - GITHUB_RUN_ID - GITHUB_REF_NAME -description = Run the GARM end-to-end test against ProdStack -set_env = - PYTHONPATH = {tox_root}/charms -deps = - -r {[vars]tests_path}/integration/requirements.txt -commands = - pytest -v \ - -s \ - --tb native \ - --log-cli-level=INFO \ - {posargs:{[vars]tests_path}/e2e} - [testenv:actions-lint] description = Run formatting and lint checks for Python code under actions/ deps = From 5c7dbf1f72ff1875f9e326b3d1ff7e2a239e0f05 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Fri, 21 Aug 2026 05:48:29 +0000 Subject: [PATCH 17/21] ci(garm-e2e): keep the runner probe to what it needs to assert Drop the aproxy log check and the --noproxy flag. The flag was a no-op -- the charm exports no proxy variables into the runner environment -- and the log check was diagnosing the proxy's configuration rather than the chain this test exists to prove, at the cost of three failure modes of its own. --- .github/workflows/garm_e2e_test_run.yaml | 58 +++--------------------- 1 file changed, 6 insertions(+), 52 deletions(-) diff --git a/.github/workflows/garm_e2e_test_run.yaml b/.github/workflows/garm_e2e_test_run.yaml index 19e9b375..b559b0ee 100644 --- a/.github/workflows/garm_e2e_test_run.yaml +++ b/.github/workflows/garm_e2e_test_run.yaml @@ -1,14 +1,10 @@ # Copyright 2026 Canonical Ltd. # See LICENSE file for licensing details. # -# Trivial workflow that a spawned runner executes to prove the full chain: -# GARM → OpenStack → VM → runner register → job dispatch → pick up → exit. -# Modelled on github-runner-operator/.github/workflows/e2e_test_run.yaml. -# -# Reaching this workflow at all is most of the assertion: it only runs if a VM -# was booted from the published image and its runner registered and claimed the -# job. The steps below cover what that does not — that the runner is usable for -# real work. Image contents are ISD298's remit, not this workflow's; keep it short. +# The job a GARM-spawned runner executes. Reaching it at all is most of the +# assertion: it only runs if a VM was booted from the published image and its +# runner registered and claimed the job. The steps cover what that does not — +# that the runner is usable for real work. Image contents are ISD298's remit. name: GARM E2E test run @@ -41,48 +37,6 @@ jobs: echo "Home: $HOME" - name: Assert egress run: | - # The charm exports no proxy variables into the runner environment: it - # delivers the proxy transparently, as aproxy plus an nftables redirect of - # :80 and :443. A plain curl is therefore what exercises that path, and - # --noproxy keeps that true if a future image starts setting http_proxy. - # Both ports are probed because aproxy reads the destination differently - # for each — Host header on :80, TLS SNI on :443 — and jobs need both - # (apt over :80, everything else over :443). - curl -sSf --noproxy '*' --max-time 30 -o /dev/null http://github.com - curl -sSf --noproxy '*' --max-time 30 -o /dev/null https://github.com + curl -sSf --max-time 30 -o /dev/null http://github.com + curl -sSf --max-time 30 -o /dev/null https://github.com echo "Egress reached github.com on :80 and :443" - - name: Assert egress traversed aproxy - run: | - # The aproxy bootstrap fails open by design: every error path in it skips - # the nftables redirect and exits 0, so a runner whose proxy was never - # wired up still boots, still registers, and still passes the step above - # wherever the tenant happens to have a route of its own. aproxy's log is - # what tells the two apart. - if ! snap list aproxy > /dev/null 2>&1; then - echo "No aproxy on this runner, so the scale set was configured without a proxy." - exit 0 - fi - # Reading the log needs root even for a user in `adm`; treat losing that - # as unverified rather than as a failure, so this cannot turn into a red - # run over a sudo policy change. - if ! APROXY_LOG=$(sudo -n snap logs aproxy.aproxy -n=all 2>/dev/null); then - echo "::warning::aproxy's log is unreadable, leaving the redirect unverified." - exit 0 - fi - # aproxy logs one `host=:` per relayed connection. The log is - # only ever matched, never printed: it sits alongside the snap - # configuration, which holds the upstream proxy and may embed credentials. - # Matched with a `case` glob rather than a pipe into `grep -q`, because - # `grep -q` exits at its first match and so SIGPIPEs the writer feeding it, - # which under `set -o pipefail` -- the shell Actions runs steps with -- fails - # the pipeline despite the match. - for HOSTPORT in github.com:80 github.com:443; do - case "$APROXY_LOG" in - *"host=$HOSTPORT"*) ;; - *) - echo "::error::aproxy is installed but relayed no connection to $HOSTPORT, so egress bypassed it. Check the 00-aproxy pre-install script in the runner's cloud-init output." - exit 1 - ;; - esac - done - echo "Egress traversed aproxy on :80 and :443" From f50500b339fbd76cf2b39df6c19b37e4288fcd5c Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Fri, 21 Aug 2026 06:08:55 +0000 Subject: [PATCH 18/21] ci(garm-e2e): probe egress once The redirect the check guards against covers :80 and :443 alike, so a second request on the other port distinguishes nothing. --- .github/workflows/garm_e2e_test_run.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/garm_e2e_test_run.yaml b/.github/workflows/garm_e2e_test_run.yaml index b559b0ee..cb84c649 100644 --- a/.github/workflows/garm_e2e_test_run.yaml +++ b/.github/workflows/garm_e2e_test_run.yaml @@ -37,6 +37,5 @@ jobs: echo "Home: $HOME" - name: Assert egress run: | - curl -sSf --max-time 30 -o /dev/null http://github.com curl -sSf --max-time 30 -o /dev/null https://github.com - echo "Egress reached github.com on :80 and :443" + echo "Egress reached github.com" From fbfa1231d3ba23b4202238e58e9102bc813aa0f7 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Fri, 21 Aug 2026 06:12:28 +0000 Subject: [PATCH 19/21] docs(garm-e2e): drop the internal spec reference The spec is not public, so name the concern rather than the document. --- .github/workflows/garm_e2e_test_run.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/garm_e2e_test_run.yaml b/.github/workflows/garm_e2e_test_run.yaml index cb84c649..1cf66574 100644 --- a/.github/workflows/garm_e2e_test_run.yaml +++ b/.github/workflows/garm_e2e_test_run.yaml @@ -4,7 +4,8 @@ # The job a GARM-spawned runner executes. Reaching it at all is most of the # assertion: it only runs if a VM was booted from the published image and its # runner registered and claimed the job. The steps cover what that does not — -# that the runner is usable for real work. Image contents are ISD298's remit. +# that the runner is usable for real work. Validating the image itself is the +# image build's concern, not this workflow's. name: GARM E2E test run From 98d3486e2c37d8c8afeaa2ef5e6a5f299528fe9b Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Fri, 21 Aug 2026 06:23:08 +0000 Subject: [PATCH 20/21] ci(garm-e2e): raise the job timeout to 30 minutes Nothing in the job can run long -- the only unbounded call is a curl capped at 30 seconds -- so the ceiling only matters if the runner itself is degraded, and there is no reason to be strict about it on a contended cloud. --- .github/workflows/garm_e2e_test_run.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/garm_e2e_test_run.yaml b/.github/workflows/garm_e2e_test_run.yaml index 1cf66574..1c7dc9d5 100644 --- a/.github/workflows/garm_e2e_test_run.yaml +++ b/.github/workflows/garm_e2e_test_run.yaml @@ -27,7 +27,7 @@ jobs: # (e.g. self-hosted-linux-amd64-noble-medium) and how GitHub routes a job to a # GARM scale set — by its name as one label. runs-on: "${{ inputs.runner-label }}" - timeout-minutes: 10 + timeout-minutes: 30 steps: - name: Report runner identity run: | From 6c9d43bc99c7bd6e93436aa65e911a7696bc5136 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Fri, 21 Aug 2026 06:24:31 +0000 Subject: [PATCH 21/21] Revert "ci(garm-e2e): raise the job timeout to 30 minutes" This reverts commit 98d3486. timeout-minutes starts when the job begins executing, not when it is queued, so it does not span VM spawn and registration -- the part a contended cloud makes slow. Keep the ceiling at 10 minutes, which is already far more than the job can use, and put the tolerance where it applies: the wait around the dispatch. --- .github/workflows/garm_e2e_test_run.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/garm_e2e_test_run.yaml b/.github/workflows/garm_e2e_test_run.yaml index 1c7dc9d5..1cf66574 100644 --- a/.github/workflows/garm_e2e_test_run.yaml +++ b/.github/workflows/garm_e2e_test_run.yaml @@ -27,7 +27,7 @@ jobs: # (e.g. self-hosted-linux-amd64-noble-medium) and how GitHub routes a job to a # GARM scale set — by its name as one label. runs-on: "${{ inputs.runner-label }}" - timeout-minutes: 30 + timeout-minutes: 10 steps: - name: Report runner identity run: |