diff --git a/.github/workflows/garm_e2e.yaml b/.github/workflows/garm_e2e.yaml new file mode 100644 index 00000000..f25f1200 --- /dev/null +++ b/.github/workflows/garm_e2e.yaml @@ -0,0 +1,213 @@ +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. +# +# GARM end-to-end test workflow. +# +# Manual trigger only, and deliberately not a merge gate. Dispatch it with +# --ref to run a feature branch's version. + +name: GARM E2E + +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + e2e: + name: GARM E2E test + runs-on: self-hosted-linux-amd64-noble-private-endpoint-medium + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7.0.1 + + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - name: Set up uv + uses: astral-sh/setup-uv@v10.0.1 + + - name: Set up tmate session (self-hosted) + if: runner.debug == 1 && runner.environment == 'self-hosted' + uses: canonical/action-tmate@main + with: + detached: true + timeout-minutes: 60 + + # These are all secrets, so the runner masks them in the log without an + # explicit ::add-mask::. The E2E_GITHUB_APP_* trio is a GitHub App of its + # own, separate from the TEST_GITHUB_APP_* one the integration suite uses: + # this one has to manage runner scale sets, which that one must not. + - name: Load tenant and GitHub App configuration + env: + OS_AUTH_URL: ${{ secrets.OS_AUTH_URL }} + OS_PROJECT_NAME: ${{ secrets.OS_PROJECT_NAME }} + OS_USER_DOMAIN_NAME: ${{ secrets.OS_USER_DOMAIN_NAME }} + OS_PROJECT_DOMAIN_NAME: ${{ secrets.OS_PROJECT_DOMAIN_NAME }} + OS_REGION_NAME: ${{ secrets.OS_REGION_NAME }} + OS_NETWORK: ${{ secrets.OS_NETWORK }} + E2E_GITHUB_APP_ID: ${{ secrets.E2E_GITHUB_APP_ID }} + E2E_GITHUB_APP_INSTALLATION_ID: ${{ secrets.E2E_GITHUB_APP_INSTALLATION_ID }} + E2E_GITHUB_APP_PRIVATE_KEY: ${{ secrets.E2E_GITHUB_APP_PRIVATE_KEY }} + run: | + set -euo pipefail + + for KEY in OS_AUTH_URL OS_PROJECT_NAME OS_USER_DOMAIN_NAME \ + OS_PROJECT_DOMAIN_NAME OS_REGION_NAME OS_NETWORK \ + E2E_GITHUB_APP_ID E2E_GITHUB_APP_INSTALLATION_ID \ + E2E_GITHUB_APP_PRIVATE_KEY; do + if [ -z "${!KEY:-}" ]; then + echo "::error::Missing repository secret: $KEY (Settings > Secrets > Actions)." + exit 1 + fi + done + + # Everything but the key is single-line already. + for KEY in OS_AUTH_URL OS_PROJECT_NAME OS_USER_DOMAIN_NAME \ + OS_PROJECT_DOMAIN_NAME OS_REGION_NAME OS_NETWORK \ + E2E_GITHUB_APP_ID E2E_GITHUB_APP_INSTALLATION_ID; do + echo "${KEY}=${!KEY}" >> "$GITHUB_ENV" + done + + # The key travels through two channels that take one KEY=value per line — + # this file, and later opcli's pytest-environment-template, which rejects a + # line without an '='. Normalising here rather than requiring a pre-encoded + # secret keeps the encoding out of everyone's setup instructions. An + # already-encoded value passes through, so both forms work. + case "$E2E_GITHUB_APP_PRIVATE_KEY" in + *-----BEGIN*) + KEY_B64=$(printf '%s' "$E2E_GITHUB_APP_PRIVATE_KEY" | base64 -w0) + ;; + *) + # Already encoded: strip whitespace, since `base64` wraps at 76 columns + # unless told otherwise and a wrapped blob would break the env file. + KEY_B64=$(printf '%s' "$E2E_GITHUB_APP_PRIVATE_KEY" | tr -d '[:space:]') + ;; + esac + # The encoded form is a different string from the secret, so the runner does + # not mask it on its own. + echo "::add-mask::$KEY_B64" + echo "E2E_GITHUB_APP_PRIVATE_KEY=$KEY_B64" >> "$GITHUB_ENV" + + # openstackclient needs this stated explicitly for a v3 auth URL. + echo "OS_IDENTITY_API_VERSION=3" >> "$GITHUB_ENV" + + - name: Fetch tenant credentials from Vault + env: + VAULT_ADDR: ${{ secrets.VAULT_ADDR }} + VAULT_APPROLE_ROLE_ID: ${{ secrets.VAULT_APPROLE_ROLE_ID }} + VAULT_APPROLE_SECRET_ID: ${{ secrets.VAULT_APPROLE_SECRET_ID }} + VAULT_KV_PATH: ${{ secrets.E2E_VAULT_KV_PATH || 'kv/data/garm-e2e/prodstack' }} + # Python rather than curl and jq: the Vault response body never becomes a + # temporary file, so only the two values it yields are handed on — through + # $GITHUB_ENV, like every other secret in this job. It also drops a dependency + # that would otherwise be installed over the private endpoint's egress. + run: | + python3 - <<'SCRIPT' + import json + import os + import sys + import urllib.error + import urllib.request + + + def fail(message): + print(f"::error::{message}") + sys.exit(1) + + + def require(name): + value = os.environ.get(name) + if not value: + fail(f"Missing repository secret: {name} (Settings > Secrets > Actions).") + return value + + + def call(url, token=None, payload=None): + request = urllib.request.Request( + url, data=json.dumps(payload).encode() if payload else None + ) + request.add_header("Content-Type", "application/json") + if token: + request.add_header("X-Vault-Token", token) + try: + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response) + except urllib.error.HTTPError as exc: + # Vault reports what is misconfigured in the body; a bare status code + # would leave this step unable to say why it failed. Only the errors + # key is surfaced, never the whole body, which on a success would be + # the credentials themselves. + try: + errors = json.load(exc).get("errors", []) + except (ValueError, AttributeError): + errors = [] + fail(f"Vault request failed with HTTP {exc.code}: {errors}") + except urllib.error.URLError as exc: + fail(f"Vault at {addr} is unreachable: {exc.reason}") + + + addr = require("VAULT_ADDR").rstrip("/") + # Normalising makes this a different string from the secret whenever + # VAULT_ADDR carries a trailing slash, and masking matches literal + # substrings, so the runner would not mask the derived form. + print(f"::add-mask::{addr}") + kv_path = require("VAULT_KV_PATH").strip("/") + + login = call( + f"{addr}/v1/auth/approle/login", + payload={ + "role_id": require("VAULT_APPROLE_ROLE_ID"), + "secret_id": require("VAULT_APPROLE_SECRET_ID"), + }, + ) + token = login.get("auth", {}).get("client_token") + if not token: + fail("Vault AppRole login returned no client token.") + print(f"::add-mask::{token}") + + # KV v2 nests the fields under data.data; KV v1 puts them directly in data. + # Accept either, so the mount version does not have to be known up front. + body = call(f"{addr}/v1/{kv_path}", token=token).get("data", {}) + secret = body.get("data", body) + + # The Vault secret is keyed on the plain names; openstackclient and the + # tests read them as OS_*. + fields = (("username", "OS_USERNAME"), ("password", "OS_PASSWORD")) + + # Checked before anything is written, so a half-populated environment cannot + # reach a later step. Nothing about the secret is reported: what it contains, + # down to the names of its fields, is not for a CI log. + if not all(secret.get(field) for field, _ in fields): + fail( + "The Vault secret does not provide both a 'username' and a " + "'password' field. Check it manually at the path configured in " + "E2E_VAULT_KV_PATH." + ) + + with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as env_file: + for field, variable in fields: + # Mask before writing: everything downstream may echo the env file. + print(f"::add-mask::{secret[field]}") + env_file.write(f"{variable}={secret[field]}\n") + + print("Tenant credentials fetched and masked.") + SCRIPT + + - name: Assert the credentials authenticate against the tenant + run: | + set -euo pipefail + + uv tool install python-openstackclient + + TOKEN_EXPIRES=$(openstack token issue -f value -c expires) + echo "OpenStack token issued (expires: ${TOKEN_EXPIRES}). Tenant reachable." + + - name: Run the E2E suite + run: | + set -euo pipefail + uv tool install tox --with tox-uv + tox -e garm-e2e diff --git a/AGENTS.md b/AGENTS.md index 7a505949..b191d527 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,7 +8,7 @@ guidance in `.github/instructions/` and the human-facing `CONTRIBUTING.md`. | Path | Contents | | --- | --- | -| `charms/` | Four Juju charms (see below) plus shared integration tests in `charms/tests/integration/`. | +| `charms/` | Four Juju charms (see below) plus shared integration tests in `charms/tests/integration/` and the GARM end-to-end test in `charms/tests/e2e/`. | | `cmd/` | Go application entry points: `planner`, `webhook-gateway`. | | `internal/` | Shared Go packages (`database`, `github`, `planner`, `queue`, `server`, `telemetry`, `webhook`, …) — the application logic the paas charms package and deploy. | | `*-rockcraft.yaml`, `build-*-rock.sh` (repo root) | Rock/image build definitions and their build scripts. | @@ -33,6 +33,7 @@ charms — there are four charms, not two). - **Per-charm Python checks** — from the charm directory, `tox -c tox.toml` (envs `fmt`, `lint`, `complexity`, `static`, `unit`, `coverage-report`; ruff, codespell, pyright, pytest+coverage). CI runs these per charm via `tox -c tox.toml`. - **Integration tests** (root `tox.ini`) — `tox -e -integration` (`garm`, `webhook-gateway`, `planner`, `garm-configurator`) or `tox -e charms-integration` for all. Requires a live Juju model (jubilant + pytest-operator). +- **GARM end-to-end test** (root `tox.ini`) — `tox -e garm-e2e`. Runs against a real OpenStack tenant from `garm_e2e.yaml`, is manually dispatched, and is **not** a merge gate. See `CONTRIBUTING.md` §"GARM E2E". - **`actions/` Python** — `tox -e actions-lint`, `tox -e actions-static`, `tox -e actions-unit`. - **Go** — `go test ./...`. - `charmcraft pack` — build a charm (run from the charm dir; not wired into tox). @@ -88,7 +89,7 @@ For **`garm-configurator`** (plain `ops`): - **DO** fix a missing AAA docstring on any test you move or edit. It's followed unevenly (`planner-operator` and `garm` yes; `garm-configurator` and `webhook-gateway-operator` not yet), so imitating the nearest neighbour is not a reliable guide. -- Integration tests live in the shared `charms/tests/integration/`. +- Integration tests live in the shared `charms/tests/integration/`; the GARM end-to-end test lives in `charms/tests/e2e/`, kept separate so it stays out of the PR test matrix. ## 12-factor divergences from the canonical charm-engineer guidance diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9dc34f6b..05b805d7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -231,3 +231,75 @@ rockcraft.skopeo copy \ oci-archive:webhook-gateway_0.1_amd64.rock \ docker://localhost:32000/webhook-gateway:0.1 ``` + +### GARM E2E + +The GARM end-to-end test (`charms/tests/e2e/`) exercises the full chain on ProdStack: +the charm starts, the configurator delivers config, the GARM API becomes reachable, the +provider authenticates to OpenStack, a VM is created from the runner image, the runner +registers with GitHub, and a dispatched job runs and exits clean. + +> **TODO:** only the workflows and the credential path exist so far. The test currently +> asserts nothing beyond the credentials reaching pytest; the deployment and the +> end-to-end assertions land in a follow-up. + +It is triggered manually and is **not** a merge gate: + +```shell +gh workflow run garm_e2e.yaml --ref +``` + +`--ref` selects which branch's version of both the workflow and the test code runs, so +changes to the end-to-end test can be exercised without merging them first. + +#### Required secrets + +Infrastructure details are secrets, not variables — endpoints, project and network names +included. The runner masks secret values in the log automatically, so the workflow does +not register masks for these itself. + +| Name | Description | +| --- | --- | +| `VAULT_ADDR` | Vault server address | +| `VAULT_APPROLE_ROLE_ID` | Vault AppRole role ID | +| `VAULT_APPROLE_SECRET_ID` | Vault AppRole secret ID | +| `OS_AUTH_URL` | Keystone endpoint, e.g. `https://keystone.example.com:5000/v3` | +| `OS_PROJECT_NAME` | OpenStack project/tenant name | +| `OS_USER_DOMAIN_NAME` | OpenStack user domain name | +| `OS_PROJECT_DOMAIN_NAME` | OpenStack project domain name | +| `OS_REGION_NAME` | OpenStack region name | +| `OS_NETWORK` | OpenStack network for runner VMs | +| `E2E_GITHUB_APP_ID` | GitHub App ID | +| `E2E_GITHUB_APP_INSTALLATION_ID` | Installation ID of that App on this repository | +| `E2E_GITHUB_APP_PRIVATE_KEY` | That App's private key. Paste the PEM as issued; base64 is also accepted | +| `E2E_VAULT_KV_PATH` | Optional. Defaults to `kv/data/garm-e2e/prodstack` | + +The OpenStack username and password are **not** repository secrets. They are read at run +time from the Vault KV v2 secret above, which must hold `username` and `password`. + +The `E2E_GITHUB_APP_*` trio is a GitHub App of its own, distinct from the +`TEST_GITHUB_APP_*` one the integration suite uses. The end-to-end test registers and +tears down a runner scale set on this repository, so its App needs `Administration: +read & write` here, which the integration App has no reason to hold. + +Paste the private key into `E2E_GITHUB_APP_PRIVATE_KEY` exactly as GitHub issues it — +the PEM, newlines and all. The workflow reduces it to a single line before it enters the +two `KEY=value` channels that carry it to pytest, neither of which can hold a multi-line +value. A value that is already base64-encoded is accepted unchanged, so an existing +encoded key does not need re-entering. + +#### Credential hygiene + +The test authenticates against a production tenant, so the run logs are part of the +contract: + +- Values read from Vault are `::add-mask::`ed in the same step that reads them, before any + later step runs. Masking registered with the runner scrubs every subsequent log line, + including output from code we do not control; a secret read inside the test process gets + none of that. Repository secrets are masked by the runner already. +- They reach later steps through `$GITHUB_ENV` and pytest through tox's `pass_env` — never + through command line arguments, which would expose them in `ps` and in pytest's header. +- Steps handling credentials use `set -euo pipefail` and never `set -x`. +- Assertions report the *name* of a missing setting, never its value, and use + `pytest.fail` rather than `assert` so that assertion rewriting cannot introspect + `os.environ` into the failure output. diff --git a/charms/tests/e2e/test_garm_e2e.py b/charms/tests/e2e/test_garm_e2e.py new file mode 100644 index 00000000..0cd545df --- /dev/null +++ b/charms/tests/e2e/test_garm_e2e.py @@ -0,0 +1,56 @@ +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. + +"""GARM end-to-end test. + +TODO: This module is deliberately incomplete. It exists so that the workflows have +something to run; the end-to-end implementation lands in a follow-up. What is here +asserts only that the credentials reach pytest. + +Still to come: deploy postgresql, GARM, a traefik ingress and garm-configurator; assert +the GARM API is reachable and the provider authenticates to OpenStack; assert a VM is +created from the runner image and the runner registers with GitHub; then dispatch +``garm_e2e_test_run.yaml`` at the scale set's label and assert the job is picked up and +exits clean. +""" + +import os + +import pytest + +# Settings the deployment fixtures will need. The OpenStack username and password come +# from Vault, the rest from repository secrets; both routes converge on the job +# environment, which tox forwards to pytest. +REQUIRED_SETTINGS = ( + "OS_AUTH_URL", + "OS_USERNAME", + "OS_PASSWORD", + "OS_PROJECT_NAME", + "OS_USER_DOMAIN_NAME", + "OS_PROJECT_DOMAIN_NAME", + "OS_REGION_NAME", + "OS_NETWORK", + "E2E_GITHUB_APP_ID", + "E2E_GITHUB_APP_INSTALLATION_ID", + "E2E_GITHUB_APP_PRIVATE_KEY", +) + + +@pytest.mark.parametrize("setting", REQUIRED_SETTINGS) +def test_setting_reaches_pytest(setting: str): + """ + arrange: The workflow has resolved the settings, taking the OpenStack username and + password from Vault and the rest from repository secrets, and exported them. + act: Read the setting pytest inherited through tox. + assert: It is present, so the deployment fixtures can authenticate to the tenant and + to GitHub. + Only the name is reported on failure — printing the value would defeat the + masking the workflow applied. + """ + # pytest.fail rather than assert: assertion rewriting would introspect the + # expression and dump the whole of os.environ into the failure output. + if not os.environ.get(setting): + pytest.fail( + f"{setting} did not reach pytest. Check that the workflow exports it and " + f"that tox passes it through in the garm-e2e environment." + ) diff --git a/tox.ini b/tox.ini index 3b320dc5..e6a08579 100644 --- a/tox.ini +++ b/tox.ini @@ -112,6 +112,29 @@ commands = --log-cli-level=INFO \ {posargs:{[vars]tests_path}/integration} +[testenv:garm-e2e] +pass_env = + PYTEST_ADDOPTS + OPCLI_ARTIFACTS_BUILD_YAML + SPREAD_JOB + JUJU_* + KUBECONFIG + OS_* + E2E_* + GITHUB_RUN_ID + GITHUB_REF_NAME +description = Run the GARM end-to-end test against ProdStack +set_env = + PYTHONPATH = {tox_root}/charms +deps = + -r {[vars]tests_path}/integration/requirements.txt +commands = + pytest -v \ + -s \ + --tb native \ + --log-cli-level=INFO \ + {posargs:{[vars]tests_path}/e2e} + [testenv:actions-lint] description = Run formatting and lint checks for Python code under actions/ deps =